mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2024-11-21 16:55:38 +03:00
Code Issues Projects Releases Packages Wiki Activity

[feature] Clean up/uncache remote media (#407)

Browse source 
* Add whereNotEmptyAndNotNull

* Add GetRemoteOlderThanDays

* Add GetRemoteOlderThanDays

* Add PruneRemote to Manager interface

* Start implementing PruneRemote

* add new attachment + status to tests

* fix up and test GetRemoteOlderThan

* fix bad import

* PruneRemote: return number pruned

* add Cached column to mediaattachment

* update + test pruneRemote

* update mediaTest

* use Cached column

* upstep bun to latest version

* embed structs in mediaAttachment

* migrate mediaAttachment to new format

* don't default cached to true

* select only remote media

* update db dependencies

* step bun back to last working version

* update pruneRemote to use Cached field

* fix storage path of test attachments

* add recache logic to manager

* fix trimmed aspect ratio

* test prune and recache

* return errwithcode

* tidy up different paths for emoji vs attachment

* fix incorrect thumbnail type being stored

* expose TransportController to media processor

* implement tee-ing recached content

* add thoughts of dog to test fedi attachments

* test get remote files

* add comment on PruneRemote

* add postData cleanup to recache

* test thumbnail fetching

* add incredible diagram

* go mod tidy

* buffer pipes for recache streaming

* test for client stops reading after 1kb

* add media-remote-cache-days to config

* add cron package

* wrap logrus so it's available to cron

* start and stop cron jobs gracefully
This commit is contained in:
tobi 2022-03-07 11:08:26 +01:00 committed by GitHub
parent 100f1280a6
commit 07727753b9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
424 changed files with 637100 additions and 176498 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
README.md
View file

@ -197,6 +197,7 @@ The following libraries and frameworks are used by GoToSocial, with gratitude
- [nfnt/resize](https://github.com/nfnt/resize); convenient image resizing. [ISC License](https://spdx.org/licenses/ISC.html).
- [oklog/ulid](https://github.com/oklog/ulid); sequential, database-friendly ID generation. [Apache-2.0 License](https://spdx.org/licenses/Apache-2.0.html).
- [ReneKroon/ttlcache](https://github.com/ReneKroon/ttlcache); in-memory caching. [MIT License](https://spdx.org/licenses/MIT.html).
- [robfig/cron](https://github.com/robfig/cron); cron job scheduling. [MIT License](https://spdx.org/licenses/MIT.html).
- [russross/blackfriday](https://github.com/russross/blackfriday); markdown parsing for statuses. [Simplified BSD License](https://spdx.org/licenses/BSD-2-Clause.html).
- [sirupsen/logrus](https://github.com/sirupsen/logrus); logging. [MIT License](https://spdx.org/licenses/MIT.html).
- [spf13/cobra](https://github.com/spf13/cobra); command-line tooling. [Apache-2.0 License](https://spdx.org/licenses/Apache-2.0.html).

1
cmd/gotosocial/flag/server.go
View file

@ -63,6 +63,7 @@ func Media(cmd *cobra.Command, values config.Values) {
cmd.Flags().Int(config.Keys.MediaVideoMaxSize, values.MediaVideoMaxSize, usage.MediaVideoMaxSize)
cmd.Flags().Int(config.Keys.MediaDescriptionMinChars, values.MediaDescriptionMinChars, usage.MediaDescriptionMinChars)
cmd.Flags().Int(config.Keys.MediaDescriptionMaxChars, values.MediaDescriptionMaxChars, usage.MediaDescriptionMaxChars)
cmd.Flags().Int(config.Keys.MediaRemoteCacheDays, values.MediaRemoteCacheDays, usage.MediaRemoteCacheDays)
}
// Storage attaches flags pertaining to storage config.

1
cmd/gotosocial/flag/usage.go
View file

@ -47,6 +47,7 @@ var usage = config.KeyNames{
MediaVideoMaxSize: "Max size of accepted videos in bytes",
MediaDescriptionMinChars: "Min required chars for an image description",
MediaDescriptionMaxChars: "Max permitted chars for an image description",
MediaRemoteCacheDays: "Number of days to locally cache media from remote instances. If set to 0, remote media will be kept indefinitely.",
StorageBackend: "Storage backend to use for media attachments",
StorageLocalBasePath: "Full path to an already-created directory where gts should store/retrieve media files. Subfolders will be created within this dir.",
StatusesMaxChars: "Max permitted characters for posted statuses",

11
docs/configuration/media.md
View file

@ -28,4 +28,15 @@ media-description-min-chars: 0
# Examples: [500, 1000, 1500]
# Default: 500
media-description-max-chars: 500
# Int. Number of days to cache media from remote instances before they are removed from the cache.
# A job will run every day at midnight to clean up any remote media older than the given amount of days.
#
# When remote media is removed from the cache, it is deleted from storage but the database entries for the media
# are kept so that it can be fetched again if requested by a user.
#
# If this is set to 0, then media from remote instances will be cached indefinitely.
# Examples: [30, 60, 7, 0]
# Default: 30
media-remote-cache-days: 30
```

11
example/config.yaml
View file

@ -205,6 +205,17 @@ media-description-min-chars: 0
# Default: 500
media-description-max-chars: 500
# Int. Number of days to cache media from remote instances before they are removed from the cache.
# A job will run every day at midnight to clean up any remote media older than the given amount of days.
#
# When remote media is removed from the cache, it is deleted from storage but the database entries for the media
# are kept so that it can be fetched again if requested by a user.
#
# If this is set to 0, then media from remote instances will be cached indefinitely.
# Examples: [30, 60, 7, 0]
# Default: 30
media-remote-cache-days: 30
##########################
##### STORAGE CONFIG #####
##########################

19
go.mod
View file

@ -18,12 +18,13 @@ require (
github.com/google/uuid v1.3.0
github.com/gorilla/websocket v1.4.2
github.com/h2non/filetype v1.1.3
github.com/jackc/pgconn v1.10.1
github.com/jackc/pgx/v4 v4.14.1
github.com/jackc/pgconn v1.11.0
github.com/jackc/pgx/v4 v4.15.0
github.com/microcosm-cc/bluemonday v1.0.16
github.com/mitchellh/mapstructure v1.4.3
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646
github.com/oklog/ulid v1.3.1
github.com/robfig/cron/v3 v3.0.1
github.com/russross/blackfriday/v2 v2.1.0
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.2.1
@ -34,7 +35,7 @@ require (
github.com/superseriousbusiness/exif-terminator v0.1.0
github.com/superseriousbusiness/oauth2/v4 v4.3.2-SSB
github.com/tdewolff/minify/v2 v2.9.22
github.com/uptrace/bun v1.0.19
github.com/uptrace/bun v1.0.20
github.com/uptrace/bun/dialect/pgdialect v1.0.19
github.com/uptrace/bun/dialect/sqlitedialect v1.0.19
github.com/wagslane/go-password-validator v0.3.0
@ -42,7 +43,7 @@ require (
golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
golang.org/x/text v0.3.7
gopkg.in/mcuadros/go-syslog.v2 v2.3.0
modernc.org/sqlite v1.14.2
modernc.org/sqlite v1.14.6
mvdan.cc/xurls/v2 v2.3.0
)
@ -84,7 +85,7 @@ require (
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgproto3/v2 v2.2.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
github.com/jackc/pgtype v1.9.1 // indirect
github.com/jackc/pgtype v1.10.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
@ -108,7 +109,7 @@ require (
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
golang.org/x/mod v0.5.1 // indirect
golang.org/x/net v0.0.0-20211209124913-491a49abca63 // indirect
golang.org/x/sys v0.0.0-20211210111614-af8b64212486 // indirect
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
golang.org/x/tools v0.1.8 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/appengine v1.6.7 // indirect
@ -118,9 +119,9 @@ require (
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
lukechampine.com/uint128 v1.1.1 // indirect
modernc.org/cc/v3 v3.35.18 // indirect
modernc.org/ccgo/v3 v3.12.92 // indirect
modernc.org/libc v1.11.101 // indirect
modernc.org/cc/v3 v3.35.22 // indirect
modernc.org/ccgo/v3 v3.15.13 // indirect
modernc.org/libc v1.14.5 // indirect
modernc.org/mathutil v1.4.1 // indirect
modernc.org/memory v1.0.5 // indirect
modernc.org/opt v0.1.1 // indirect

64
go.sum
View file

@ -412,8 +412,8 @@ github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsU
github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o=
github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY=
github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
github.com/jackc/pgconn v1.10.1 h1:DzdIHIjG1AxGwoEEqS+mGsURyjt4enSmqzACXvVzOT8=
github.com/jackc/pgconn v1.10.1/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
github.com/jackc/pgconn v1.11.0 h1:HiHArx4yFbwl91X3qqIHtUFoiIfLNJXCQRsnzkiwwaQ=
github.com/jackc/pgconn v1.11.0/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI=
github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE=
github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8=
github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE=
@ -438,18 +438,18 @@ github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01C
github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc=
github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw=
github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM=
github.com/jackc/pgtype v1.9.1 h1:MJc2s0MFS8C3ok1wQTdQxWuXQcB6+HwAm5x1CzW7mf0=
github.com/jackc/pgtype v1.9.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4=
github.com/jackc/pgtype v1.10.0 h1:ILnBWrRMSXGczYvmkYD6PsYyVFUNLTnIUJHHDLmqk38=
github.com/jackc/pgtype v1.10.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4=
github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y=
github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM=
github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc=
github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs=
github.com/jackc/pgx/v4 v4.14.1 h1:71oo1KAGI6mXhLiTMn6iDFcp3e7+zon/capWjl2OEFU=
github.com/jackc/pgx/v4 v4.14.1/go.mod h1:RgDuE4Z34o7XE92RpLsvFiOEfrAUT0Xt2KxvX73W06M=
github.com/jackc/pgx/v4 v4.15.0 h1:B7dTkXsdILD3MF987WGGCcg+tvLW6bZJdEcqVFeU//w=
github.com/jackc/pgx/v4 v4.15.0/go.mod h1:D/zyOyXiaM1TmVWnOM18p0xdDtdakRBa0RsVGI3U3bw=
github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v1.2.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jackc/puddle v1.2.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
@ -518,8 +518,8 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-sqlite3 v1.14.9 h1:10HX2Td0ocZpYEjhilsuo6WWtUqttj2Kb0KtD86/KYA=
github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/mattn/go-sqlite3 v1.14.10 h1:MLn+5bFRlWMGoSRmJour3CL1w/qL96mvipqpwQW/Sfk=
github.com/mattn/go-sqlite3 v1.14.10/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/memcachier/mc v2.0.1+incompatible/go.mod h1:7bkvFE61leUBvXz+yxsOnGBQSZpBSPIMUQSmmSHvuXc=
github.com/microcosm-cc/bluemonday v1.0.16 h1:kHmAq2t7WPWLjiGvzKa5o3HzSfahUKiOq7fAPUiMNIc=
@ -590,6 +590,8 @@ github.com/quasoft/memstore v0.0.0-20191010062613-2bce066d2b0b h1:aUNXCGgukb4gtY
github.com/quasoft/memstore v0.0.0-20191010062613-2bce066d2b0b/go.mod h1:wTPjTepVu7uJBYgZ0SdWHQlIas582j6cn2jgk4DDdlg=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
@ -689,8 +691,9 @@ github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ=
github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw=
github.com/uptrace/bun v1.0.19 h1:Jl5GbcnLqeo4mHLVbF+LvzVXiXTrwQOdIVvuMO+5zXI=
github.com/uptrace/bun v1.0.19/go.mod h1:Uv7z0z+7dXnUS9P5hMF0hdiM/4M+xOUHQCrZpyDrpRc=
github.com/uptrace/bun v1.0.20 h1:/T4p9C9tEN75U3cFMBr5njlP+rBfs4An8BmlQQPbcfE=
github.com/uptrace/bun v1.0.20/go.mod h1:Uv7z0z+7dXnUS9P5hMF0hdiM/4M+xOUHQCrZpyDrpRc=
github.com/uptrace/bun/dialect/pgdialect v1.0.19 h1:8vdHpXRq3rbNDRn13llLxJ52wa+aBBn7QI0Q7IyW/d8=
github.com/uptrace/bun/dialect/pgdialect v1.0.19/go.mod h1:VMmro4cCcjC7xAIeWYvRHv8V2nF+q2Tm36WK+poCeuo=
github.com/uptrace/bun/dialect/sqlitedialect v1.0.19 h1:Oy0scO7sXbVBk9k6xc2yFF/X1maW5UU6uqO6coX/5/8=
@ -970,8 +973,8 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20211123173158-ef496fb156ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20211210111614-af8b64212486 h1:5hpz5aRr+W1erYCL5JRhSUBJRph7l9XkNveoExlrKYk=
golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@ -1261,8 +1264,10 @@ modernc.org/cc/v3 v3.35.10/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g
modernc.org/cc/v3 v3.35.15/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
modernc.org/cc/v3 v3.35.16/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
modernc.org/cc/v3 v3.35.17/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
modernc.org/cc/v3 v3.35.18 h1:rMZhRcWrba0y3nVmdiQ7kxAgOOSq2m2f2VzjHLgEs6U=
modernc.org/cc/v3 v3.35.18/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
modernc.org/cc/v3 v3.35.20/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
modernc.org/cc/v3 v3.35.22 h1:BzShpwCAP7TWzFppM4k2t03RhXhgYqaibROWkrWq7lE=
modernc.org/cc/v3 v3.35.22/go.mod h1:iPJg1pkwXqAV16SNgFBVYmggfMg6xhs+2oiO0vclK3g=
modernc.org/ccgo/v3 v3.9.5/go.mod h1:umuo2EP2oDSBnD3ckjaVUXMrmeAw8C8OSICVa0iFf60=
modernc.org/ccgo/v3 v3.10.0/go.mod h1:c0yBmkRFi7uW4J7fwx/JiijwOjeAeR2NoSaRVFPmjMw=
modernc.org/ccgo/v3 v3.11.0/go.mod h1:dGNposbDp9TOZ/1KBxghxtUp/bzErD0/0QW4hhSaBMI=
@ -1292,19 +1297,24 @@ modernc.org/ccgo/v3 v3.12.55/go.mod h1:rsXiIyJi9psOwiBkplOaHye5L4MOOaCjHg1Fxkj7I
modernc.org/ccgo/v3 v3.12.56/go.mod h1:ljeFks3faDseCkr60JMpeDb2GSO3TKAmrzm7q9YOcMU=
modernc.org/ccgo/v3 v3.12.57/go.mod h1:hNSF4DNVgBl8wYHpMvPqQWDQx8luqxDnNGCMM4NFNMc=
modernc.org/ccgo/v3 v3.12.60/go.mod h1:k/Nn0zdO1xHVWjPYVshDeWKqbRWIfif5dtsIOCUVMqM=
modernc.org/ccgo/v3 v3.12.65/go.mod h1:D6hQtKxPNZiY6wDBtehSGKFKmyXn53F8nGTpH+POmS4=
modernc.org/ccgo/v3 v3.12.66/go.mod h1:jUuxlCFZTUZLMV08s7B1ekHX5+LIAurKTTaugUr/EhQ=
modernc.org/ccgo/v3 v3.12.67/go.mod h1:Bll3KwKvGROizP2Xj17GEGOTrlvB1XcVaBrC90ORO84=
modernc.org/ccgo/v3 v3.12.73/go.mod h1:hngkB+nUUqzOf3iqsM48Gf1FZhY599qzVg1iX+BT3cQ=
modernc.org/ccgo/v3 v3.12.81/go.mod h1:p2A1duHoBBg1mFtYvnhAnQyI6vL0uw5PGYLSIgF6rYY=
modernc.org/ccgo/v3 v3.12.82/go.mod h1:ApbflUfa5BKadjHynCficldU1ghjen84tuM5jRynB7w=
modernc.org/ccgo/v3 v3.12.84/go.mod h1:ApbflUfa5BKadjHynCficldU1ghjen84tuM5jRynB7w=
modernc.org/ccgo/v3 v3.12.86/go.mod h1:dN7S26DLTgVSni1PVA3KxxHTcykyDurf3OgUzNqTSrU=
modernc.org/ccgo/v3 v3.12.90/go.mod h1:obhSc3CdivCRpYZmrvO88TXlW0NvoSVvdh/ccRjJYko=
modernc.org/ccgo/v3 v3.12.92 h1:QT2o1iX5IMsKo0MQ9jKXKb5th+J1pDECtn0BXNB85bE=
modernc.org/ccgo/v3 v3.12.92/go.mod h1:5yDdN7ti9KWPi5bRVWPl8UNhpEAtCjuEE7ayQnzzqHA=
modernc.org/ccorpus v1.11.1 h1:K0qPfpVG1MJh5BYazccnmhywH4zHuOgJXgbjzyp6dWA=
modernc.org/ccgo/v3 v3.13.1/go.mod h1:aBYVOUfIlcSnrsRVU8VRS35y2DIfpgkmVkYZ0tpIXi4=
modernc.org/ccgo/v3 v3.15.1/go.mod h1:md59wBwDT2LznX/OTCPoVS6KIsdRgY8xqQwBV+hkTH0=
modernc.org/ccgo/v3 v3.15.9/go.mod h1:md59wBwDT2LznX/OTCPoVS6KIsdRgY8xqQwBV+hkTH0=
modernc.org/ccgo/v3 v3.15.10/go.mod h1:wQKxoFn0ynxMuCLfFD09c8XPUCc8obfchoVR9Cn0fI8=
modernc.org/ccgo/v3 v3.15.12/go.mod h1:VFePOWoCd8uDGRJpq/zfJ29D0EVzMSyID8LCMWYbX6I=
modernc.org/ccgo/v3 v3.15.13 h1:hqlCzNJTXLrhS70y1PqWckrF9x1btSQRC7JFuQcBg5c=
modernc.org/ccgo/v3 v3.15.13/go.mod h1:QHtvdpeODlXjdK3tsbpyK+7U9JV4PQsrPGIbtmc0KfY=
modernc.org/ccorpus v1.11.1/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
modernc.org/ccorpus v1.11.4 h1:YOmQBBzE8GC/puUx76D5j/gJYIZQsydrh6VMJVfXF0M=
modernc.org/ccorpus v1.11.4/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ=
modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM=
modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM=
modernc.org/libc v1.9.8/go.mod h1:U1eq8YWr/Kc1RWCMFUWEdkTg8OTcfLw2kY8EDwl039w=
@ -1336,7 +1346,6 @@ modernc.org/libc v1.11.54/go.mod h1:S/FVnskbzVUrjfBqlGFIPA5m7UwB3n9fojHhCNfSsnw=
modernc.org/libc v1.11.55/go.mod h1:j2A5YBRm6HjNkoSs/fzZrSxCuwWqcMYTDPLNx0URn3M=
modernc.org/libc v1.11.56/go.mod h1:pakHkg5JdMLt2OgRadpPOTnyRXm/uzu+Yyg/LSLdi18=
modernc.org/libc v1.11.58/go.mod h1:ns94Rxv0OWyoQrDqMFfWwka2BcaF6/61CqJRK9LP7S8=
modernc.org/libc v1.11.70/go.mod h1:DUOmMYe+IvKi9n6Mycyx3DbjfzSKrdr/0Vgt3j7P5gw=
modernc.org/libc v1.11.71/go.mod h1:DUOmMYe+IvKi9n6Mycyx3DbjfzSKrdr/0Vgt3j7P5gw=
modernc.org/libc v1.11.75/go.mod h1:dGRVugT6edz361wmD9gk6ax1AbDSe0x5vji0dGJiPT0=
modernc.org/libc v1.11.82/go.mod h1:NF+Ek1BOl2jeC7lw3a7Jj5PWyHPwWD4aq3wVKxqV1fI=
@ -1344,8 +1353,13 @@ modernc.org/libc v1.11.86/go.mod h1:ePuYgoQLmvxdNT06RpGnaDKJmDNEkV7ZPKI2jnsvZoE=
modernc.org/libc v1.11.87/go.mod h1:Qvd5iXTeLhI5PS0XSyqMY99282y+3euapQFxM7jYnpY=
modernc.org/libc v1.11.88/go.mod h1:h3oIVe8dxmTcchcFuCcJ4nAWaoiwzKCdv82MM0oiIdQ=
modernc.org/libc v1.11.98/go.mod h1:ynK5sbjsU77AP+nn61+k+wxUGRx9rOFcIqWYYMaDZ4c=
modernc.org/libc v1.11.101 h1:mFp54enlBZiH0d26tRFoq2WL8KsjV1sefpOy4+gyTDQ=
modernc.org/libc v1.11.101/go.mod h1:wLLYgEiY2D17NbBOEp+mIJJJBGSiy7fLL4ZrGGZ+8jI=
modernc.org/libc v1.12.0/go.mod h1:2MH3DaF/gCU8i/UBiVE1VFRos4o523M7zipmwH8SIgQ=
modernc.org/libc v1.14.1/go.mod h1:npFeGWjmZTjFeWALQLrvklVmAxv4m80jnG3+xI8FdJk=
modernc.org/libc v1.14.2/go.mod h1:MX1GBLnRLNdvmK9azU9LCxZ5lMyhrbEMK8rG3X/Fe34=
modernc.org/libc v1.14.3/go.mod h1:GPIvQVOVPizzlqyRX3l756/3ppsAgg1QgPxjr5Q4agQ=
modernc.org/libc v1.14.5 h1:DAHvwGoVRDZs5iJXnX9RJrgXSsorupCWmJ2ac964Owk=
modernc.org/libc v1.14.5/go.mod h1:2PJHINagVxO4QW/5OQdRrvMYo+bm5ClpUFfyXCYl9ak=
modernc.org/mathutil v1.1.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/mathutil v1.4.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
@ -1356,16 +1370,16 @@ modernc.org/memory v1.0.5 h1:XRch8trV7GgvTec2i7jc33YlUI0RKVDBvZ5eZ5m8y14=
modernc.org/memory v1.0.5/go.mod h1:B7OYswTRnfGg+4tDH1t1OeUNnsy2viGTdME4tzd+IjM=
modernc.org/opt v0.1.1 h1:/0RX92k9vwVeDXj+Xn23DKp2VJubL7k8qNffND6qn3A=
modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
modernc.org/sqlite v1.14.2 h1:ohsW2+e+Qe2To1W6GNezzKGwjXwSax6R+CrhRxVaFbE=
modernc.org/sqlite v1.14.2/go.mod h1:yqfn85u8wVOE6ub5UT8VI9JjhrwBUUCNyTACN0h6Sx8=
modernc.org/sqlite v1.14.6 h1:Jt5P3k80EtDBWaq1beAxnWW+5MdHXbZITujnRS7+zWg=
modernc.org/sqlite v1.14.6/go.mod h1:yiCvMv3HblGmzENNIaNtFhfaNIwcla4u2JQEwJPzfEc=
modernc.org/strutil v1.1.1 h1:xv+J1BXY3Opl2ALrBwyfEikFAj8pmqcpnfmuwUwcozs=
modernc.org/strutil v1.1.1/go.mod h1:DE+MQQ/hjKBZS2zNInV5hhcipt5rLPWkmpbGeW5mmdw=
modernc.org/tcl v1.8.13 h1:V0sTNBw0Re86PvXZxuCub3oO9WrSTqALgrwNZNvLFGw=
modernc.org/tcl v1.8.13/go.mod h1:V+q/Ef0IJaNUSECieLU4o+8IScapxnMyFV6i/7uQlAY=
modernc.org/tcl v1.11.0 h1:B/zzEYjINeaki38KcIqdQRQx7W3WE7TkrlTwGnbm2II=
modernc.org/tcl v1.11.0/go.mod h1:zsTUpbQ+NxQEjOjCUlImDLPv1sG8Ww0qp66ZvyOxCgw=
modernc.org/token v1.0.0 h1:a0jaWiNMDhDUtqOj09wvjWWAqd3q7WpBulmL9H2egsk=
modernc.org/token v1.0.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
modernc.org/z v1.2.19 h1:BGyRFWhDVn5LFS5OcX4Yd/MlpRTOc7hOPTdcIpCiUao=
modernc.org/z v1.2.19/go.mod h1:+ZpP0pc4zz97eukOzW3xagV/lS82IpPN9NGG5pNF9vY=
modernc.org/z v1.3.0 h1:4RWULo1Nvaq5ZBhbLe74u8p6tV4Mmm0ZrPBXYPm/xjM=
modernc.org/z v1.3.0/go.mod h1:+mvgLH814oDjtATDdT3rs84JnUIpkvAF5B8AVkNlE2g=
mvdan.cc/xurls/v2 v2.3.0 h1:59Olnbt67UKpxF1EwVBopJvkSUBmgtb468E4GVWIZ1I=
mvdan.cc/xurls/v2 v2.3.0/go.mod h1:AjuTy7gEiUArFMjgBBDU4SMxlfUYsRokpJQgNWOt3e4=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

10
internal/api/client/fileserver/servefile.go
View file

@ -79,15 +79,15 @@ func (m *FileServer) ServeFile(c *gin.Context) {
return
}
content, err := m.processor.FileGet(c.Request.Context(), authed, &model.GetContentRequestForm{
content, errWithCode := m.processor.FileGet(c.Request.Context(), authed, &model.GetContentRequestForm{
AccountID: accountID,
MediaType: mediaType,
MediaSize: mediaSize,
FileName: fileName,
})
if err != nil {
l.Debug(err)
c.String(http.StatusNotFound, "404 page not found")
if errWithCode != nil {
l.Errorf(errWithCode.Error())
c.JSON(errWithCode.Code(), gin.H{"error": errWithCode.Safe()})
return
}
@ -104,7 +104,7 @@ func (m *FileServer) ServeFile(c *gin.Context) {
// This is mostly needed because when sharing a link to a gts-hosted file on something like mastodon, the masto servers will
// attempt to look up the content to provide a preview of the link, and they ask for text/html.
format, err := api.NegotiateAccept(c, api.Offer(content.ContentType))
if err != nil {
if errWithCode != nil {
c.JSON(http.StatusNotAcceptable, gin.H{"error": err.Error()})
return
}

1
internal/config/defaults.go
View file

@ -53,6 +53,7 @@ var Defaults = Values{
MediaVideoMaxSize: 10485760, // 10mb
MediaDescriptionMinChars: 0,
MediaDescriptionMaxChars: 500,
MediaRemoteCacheDays: 30,
StorageBackend: "local",
StorageLocalBasePath: "/gotosocial/storage",

2
internal/config/keys.go
View file

@ -58,6 +58,7 @@ type KeyNames struct {
MediaVideoMaxSize string
MediaDescriptionMinChars string
MediaDescriptionMaxChars string
MediaRemoteCacheDays string
// storage
StorageBackend string
@ -138,6 +139,7 @@ var Keys = KeyNames{
MediaVideoMaxSize: "media-video-max-size",
MediaDescriptionMinChars: "media-description-min-chars",
MediaDescriptionMaxChars: "media-description-max-chars",
MediaRemoteCacheDays: "media-remote-cache-days",
StorageBackend: "storage-backend",
StorageLocalBasePath: "storage-local-base-path",

1
internal/config/values.go
View file

@ -51,6 +51,7 @@ type Values struct {
MediaVideoMaxSize int
MediaDescriptionMinChars int
MediaDescriptionMaxChars int
MediaRemoteCacheDays int
StorageBackend string
StorageLocalBasePath string

2
internal/db/bundb/basic_test.go
View file

@ -43,7 +43,7 @@ func (suite *BasicTestSuite) TestGetAllStatuses() {
s := []*gtsmodel.Status{}
err := suite.db.GetAll(context.Background(), &s)
suite.NoError(err)
suite.Len(s, 14)
suite.Len(s, 15)
}
func (suite *BasicTestSuite) TestGetAllNotNull() {

3
internal/db/bundb/bundb_test.go
View file

@ -55,9 +55,8 @@ func (suite *BunDBStandardTestSuite) SetupSuite() {
}
func (suite *BunDBStandardTestSuite) SetupTest() {
testrig.InitTestLog()
testrig.InitTestConfig()
testrig.InitTestLog()
suite.db = testrig.NewTestDB()
testrig.StandardDBSetup(suite.db, suite.testAccounts)
}

24
internal/db/bundb/media.go
View file

@ -20,6 +20,7 @@ package bundb
import (
"context"
"time"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
@ -48,3 +49,26 @@ func (m *mediaDB) GetAttachmentByID(ctx context.Context, id string) (*gtsmodel.M
}
return attachment, nil
}
func (m *mediaDB) GetRemoteOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.MediaAttachment, db.Error) {
attachments := []*gtsmodel.MediaAttachment{}
q := m.conn.
NewSelect().
Model(&attachments).
Where("media_attachment.cached = true").
Where("media_attachment.avatar = false").
Where("media_attachment.header = false").
Where("media_attachment.created_at < ?", olderThan).
WhereGroup(" AND ", whereNotEmptyAndNotNull("media_attachment.remote_url")).
Order("media_attachment.created_at DESC")
if limit != 0 {
q = q.Limit(limit)
}
if err := q.Scan(ctx); err != nil {
return nil, m.conn.ProcessError(err)
}
return attachments, nil
}

48
internal/db/bundb/media_test.go Normal file
View file

@ -0,0 +1,48 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package bundb_test
import (
"context"
"testing"
"time"
"github.com/stretchr/testify/suite"
)
type MediaTestSuite struct {
BunDBStandardTestSuite
}
func (suite *MediaTestSuite) TestGetAttachmentByID() {
testAttachment := suite.testAttachments["admin_account_status_1_attachment_1"]
attachment, err := suite.db.GetAttachmentByID(context.Background(), testAttachment.ID)
suite.NoError(err)
suite.NotNil(attachment)
}
func (suite *MediaTestSuite) TestGetOlder() {
attachments, err := suite.db.GetRemoteOlderThan(context.Background(), time.Now(), 20)
suite.NoError(err)
suite.Len(attachments, 1)
}
func TestMediaTestSuite(t *testing.T) {
suite.Run(t, new(MediaTestSuite))
}

172
internal/db/bundb/migrations/20220214175650_media_cleanup.go Normal file
View file

@ -0,0 +1,172 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package migrations
import (
"context"
"database/sql"
"time"
previousgtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20211113114307_init"
newgtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20220214175650_media_cleanup"
"github.com/uptrace/bun"
)
func init() {
const batchSize = 100
up := func(ctx context.Context, db *bun.DB) error {
// we need to migrate media attachments into a new table
// see section 6 here: https://www.sqlite.org/lang_altertable.html
return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
// create the new media attachments table
if _, err := tx.
NewCreateTable().
ModelTableExpr("new_media_attachments").
Model(&newgtsmodel.MediaAttachment{}).
IfNotExists().
Exec(ctx); err != nil {
return err
}
offset := time.Now()
// migrate existing media attachments into new table
migrateLoop:
for {
oldAttachments := []*previousgtsmodel.MediaAttachment{}
err := tx.
NewSelect().
Model(&oldAttachments).
// subtract a millisecond from the offset just to make sure we're not getting double entries (this happens sometimes)
Where("media_attachment.created_at < ?", offset.Add(-1*time.Millisecond)).
Order("media_attachment.created_at DESC").
Limit(batchSize).
Scan(ctx)
if err != nil && err != sql.ErrNoRows {
// there's been a real error
return err
}
if err == sql.ErrNoRows || len(oldAttachments) == 0 {
// we're finished migrating
break migrateLoop
}
// update the offset to the createdAt time of the oldest media attachment in the slice
offset = oldAttachments[len(oldAttachments)-1].CreatedAt
// for every old attachment, we need to make a new attachment out of it by taking the same values
newAttachments := []*newgtsmodel.MediaAttachment{}
for _, old := range oldAttachments {
new := &newgtsmodel.MediaAttachment{
ID: old.ID,
CreatedAt: old.CreatedAt,
UpdatedAt: old.UpdatedAt,
StatusID: old.StatusID,
URL: old.URL,
RemoteURL: old.RemoteURL,
Type: newgtsmodel.FileType(old.Type),
FileMeta: newgtsmodel.FileMeta{
Original: newgtsmodel.Original{
Width: old.FileMeta.Original.Width,
Height: old.FileMeta.Original.Height,
Size: old.FileMeta.Original.Size,
Aspect: old.FileMeta.Original.Aspect,
},
Small: newgtsmodel.Small{
Width: old.FileMeta.Small.Width,
Height: old.FileMeta.Small.Height,
Size: old.FileMeta.Small.Size,
Aspect: old.FileMeta.Small.Aspect,
},
Focus: newgtsmodel.Focus{
X: old.FileMeta.Focus.X,
Y: old.FileMeta.Focus.Y,
},
},
AccountID: old.AccountID,
Description: old.Description,
ScheduledStatusID: old.ScheduledStatusID,
Blurhash: old.Blurhash,
Processing: newgtsmodel.ProcessingStatus(old.Processing),
File: newgtsmodel.File{
Path: old.File.Path,
ContentType: old.File.ContentType,
FileSize: old.File.FileSize,
UpdatedAt: old.File.UpdatedAt,
},
Thumbnail: newgtsmodel.Thumbnail{
Path: old.Thumbnail.Path,
ContentType: old.Thumbnail.ContentType,
FileSize: old.Thumbnail.FileSize,
UpdatedAt: old.Thumbnail.UpdatedAt,
URL: old.Thumbnail.URL,
RemoteURL: old.Thumbnail.RemoteURL,
},
Avatar: old.Avatar,
Header: old.Header,
Cached: true,
}
newAttachments = append(newAttachments, new)
}
// insert this batch of new attachments, and then continue the loop
if _, err := tx.
NewInsert().
Model(&newAttachments).
ModelTableExpr("new_media_attachments").
Exec(ctx); err != nil {
return err
}
}
// we have all the data we need from the old table, so we can safely drop it now
if _, err := tx.NewDropTable().Model(&previousgtsmodel.MediaAttachment{}).Exec(ctx); err != nil {
return err
}
// rename the new table to the same name as the old table was
if _, err := tx.QueryContext(ctx, "ALTER TABLE new_media_attachments RENAME TO media_attachments;"); err != nil {
return err
}
// add an index to the new table
if _, err := tx.
NewCreateIndex().
Model(&newgtsmodel.MediaAttachment{}).
Index("media_attachments_id_idx").
Column("id").
Exec(ctx); err != nil {
return err
}
return nil
})
}
down := func(ctx context.Context, db *bun.DB) error {
return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error {
return nil
})
}
if err := Migrations.Register(up, down); err != nil {
panic(err)
}
}

117
internal/db/bundb/migrations/20220214175650_media_cleanup/account.go Normal file
View file

@ -0,0 +1,117 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
// Package gtsmodel contains types used *internally* by GoToSocial and added/removed/selected from the database.
// These types should never be serialized and/or sent out via public APIs, as they contain sensitive information.
// The annotation used on these structs is for handling them via the bun-db ORM.
// See here for more info on bun model annotations: https://bun.uptrace.dev/guide/models.html
package gtsmodel
import (
"crypto/rsa"
"time"
)
// Account represents either a local or a remote fediverse account, gotosocial or otherwise (mastodon, pleroma, etc).
type Account struct {
ID string `validate:"required,ulid" bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
CreatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created
UpdatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated
Username string `validate:"required" bun:",nullzero,notnull,unique:userdomain"` // Username of the account, should just be a string of [a-zA-Z0-9_]. Can be added to domain to create the full username in the form ``[username]@[domain]`` eg., ``user_96@example.org``. Username and domain should be unique *with* each other
Domain string `validate:"omitempty,fqdn" bun:",nullzero,unique:userdomain"` // Domain of the account, will be null if this is a local account, otherwise something like ``example.org``. Should be unique with username.
AvatarMediaAttachmentID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // Database ID of the media attachment, if present
AvatarMediaAttachment *MediaAttachment `validate:"-" bun:"rel:belongs-to"` // MediaAttachment corresponding to avatarMediaAttachmentID
AvatarRemoteURL string `validate:"omitempty,url" bun:",nullzero"` // For a non-local account, where can the header be fetched?
HeaderMediaAttachmentID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // Database ID of the media attachment, if present
HeaderMediaAttachment *MediaAttachment `validate:"-" bun:"rel:belongs-to"` // MediaAttachment corresponding to headerMediaAttachmentID
HeaderRemoteURL string `validate:"omitempty,url" bun:",nullzero"` // For a non-local account, where can the header be fetched?
DisplayName string `validate:"-" bun:""` // DisplayName for this account. Can be empty, then just the Username will be used for display purposes.
Fields []Field `validate:"-"` // a key/value map of fields that this account has added to their profile
Note string `validate:"-" bun:""` // A note that this account has on their profile (ie., the account's bio/description of themselves)
Memorial bool `validate:"-" bun:",default:false"` // Is this a memorial account, ie., has the user passed away?
AlsoKnownAs string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // This account is associated with x account id (TODO: migrate to be AlsoKnownAsID)
MovedToAccountID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // This account has moved this account id in the database
Bot bool `validate:"-" bun:",default:false"` // Does this account identify itself as a bot?
Reason string `validate:"-" bun:""` // What reason was given for signing up when this account was created?
Locked bool `validate:"-" bun:",default:true"` // Does this account need an approval for new followers?
Discoverable bool `validate:"-" bun:",default:false"` // Should this account be shown in the instance's profile directory?
Privacy Visibility `validate:"required_without=Domain,omitempty,oneof=public unlocked followers_only mutuals_only direct" bun:",nullzero"` // Default post privacy for this account
Sensitive bool `validate:"-" bun:",default:false"` // Set posts from this account to sensitive by default?
Language string `validate:"omitempty,bcp47_language_tag" bun:",nullzero,notnull,default:'en'"` // What language does this account post in?
URI string `validate:"required,url" bun:",nullzero,notnull,unique"` // ActivityPub URI for this account.
URL string `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"` // Web URL for this account's profile
LastWebfingeredAt time.Time `validate:"required_with=Domain" bun:"type:timestamptz,nullzero"` // Last time this account was refreshed/located with webfinger.
InboxURI string `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"` // Address of this account's ActivityPub inbox, for sending activity to
OutboxURI string `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"` // Address of this account's activitypub outbox
FollowingURI string `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"` // URI for getting the following list of this account
FollowersURI string `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"` // URI for getting the followers list of this account
FeaturedCollectionURI string `validate:"required_without=Domain,omitempty,url" bun:",nullzero,unique"` // URL for getting the featured collection list of this account
ActorType string `validate:"oneof=Application Group Organization Person Service" bun:",nullzero,notnull"` // What type of activitypub actor is this account?
PrivateKey *rsa.PrivateKey `validate:"required_without=Domain"` // Privatekey for validating activitypub requests, will only be defined for local accounts
PublicKey *rsa.PublicKey `validate:"required"` // Publickey for encoding activitypub requests, will be defined for both local and remote accounts
PublicKeyURI string `validate:"required,url" bun:",nullzero,notnull,unique"` // Web-reachable location of this account's public key
SensitizedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero"` // When was this account set to have all its media shown as sensitive?
SilencedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero"` // When was this account silenced (eg., statuses only visible to followers, not public)?
SuspendedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero"` // When was this account suspended (eg., don't allow it to log in/post, don't accept media/posts from this account)
HideCollections bool `validate:"-" bun:",default:false"` // Hide this account's collections
SuspensionOrigin string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // id of the database entry that caused this account to become suspended -- can be an account ID or a domain block ID
}
// Field represents a key value field on an account, for things like pronouns, website, etc.
// VerifiedAt is optional, to be used only if Value is a URL to a webpage that contains the
// username of the user.
type Field struct {
Name string `validate:"required"` // Name of this field.
Value string `validate:"required"` // Value of this field.
VerifiedAt time.Time `validate:"-" bun:",nullzero"` // This field was verified at (optional).
}
// Relationship describes a requester's relationship with another account.
type Relationship struct {
ID string // The account id.
Following bool // Are you following this user?
ShowingReblogs bool // Are you receiving this user's boosts in your home timeline?
Notifying bool // Have you enabled notifications for this user?
FollowedBy bool // Are you followed by this user?
Blocking bool // Are you blocking this user?
BlockedBy bool // Is this user blocking you?
Muting bool // Are you muting this user?
MutingNotifications bool // Are you muting notifications from this user?
Requested bool // Do you have a pending follow request for this user?
DomainBlocking bool // Are you blocking this user's domain?
Endorsed bool // Are you featuring this user on your profile?
Note string // Your note on this account.
}
// Visibility represents the visibility granularity of a status.
type Visibility string
const (
// VisibilityPublic means this status will be visible to everyone on all timelines.
VisibilityPublic Visibility = "public"
// VisibilityUnlocked means this status will be visible to everyone, but will only show on home timeline to followers, and in lists.
VisibilityUnlocked Visibility = "unlocked"
// VisibilityFollowersOnly means this status is viewable to followers only.
VisibilityFollowersOnly Visibility = "followers_only"
// VisibilityMutualsOnly means this status is visible to mutual followers only.
VisibilityMutualsOnly Visibility = "mutuals_only"
// VisibilityDirect means this status is visible only to mentioned recipients.
VisibilityDirect Visibility = "direct"
// VisibilityDefault is used when no other setting can be found.
VisibilityDefault Visibility = VisibilityUnlocked
)

116
internal/db/bundb/migrations/20220214175650_media_cleanup/mediaattachment.go Normal file
View file

@ -0,0 +1,116 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gtsmodel
import (
"time"
)
type MediaAttachment struct {
ID string `validate:"required,ulid" bun:"type:CHAR(26),pk,nullzero,notnull,unique"` // id of this item in the database
CreatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item created
UpdatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // when was item last updated
StatusID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // ID of the status to which this is attached
URL string `validate:"required_without=RemoteURL,omitempty,url" bun:",nullzero"` // Where can the attachment be retrieved on *this* server
RemoteURL string `validate:"required_without=URL,omitempty,url" bun:",nullzero"` // Where can the attachment be retrieved on a remote server (empty for local media)
Type FileType `validate:"oneof=Image Gif Audio Video Unknown" bun:",nullzero,notnull"` // Type of file (image/gif/audio/video)
FileMeta FileMeta `validate:"required" bun:",embed:filemeta_,nullzero,notnull"` // Metadata about the file
AccountID string `validate:"required,ulid" bun:"type:CHAR(26),nullzero,notnull"` // To which account does this attachment belong
Account *Account `validate:"-" bun:"rel:has-one"` // Account corresponding to accountID
Description string `validate:"-" bun:""` // Description of the attachment (for screenreaders)
ScheduledStatusID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // To which scheduled status does this attachment belong
Blurhash string `validate:"required_if=Type Image,required_if=Type Gif,required_if=Type Video" bun:",nullzero"` // What is the generated blurhash of this attachment
Processing ProcessingStatus `validate:"oneof=0 1 2 666" bun:",notnull,default:2"` // What is the processing status of this attachment
File File `validate:"required" bun:",embed:file_,notnull,nullzero"` // metadata for the whole file
Thumbnail Thumbnail `validate:"required" bun:",embed:thumbnail_,notnull,nullzero"` // small image thumbnail derived from a larger image, video, or audio file.
Avatar bool `validate:"-" bun:",notnull,default:false"` // Is this attachment being used as an avatar?
Header bool `validate:"-" bun:",notnull,default:false"` // Is this attachment being used as a header?
Cached bool `validate:"-" bun:",notnull"` // Is this attachment currently cached by our instance?
}
// File refers to the metadata for the whole file
type File struct {
Path string `validate:"required,file" bun:",nullzero,notnull"` // Path of the file in storage.
ContentType string `validate:"required" bun:",nullzero,notnull"` // MIME content type of the file.
FileSize int `validate:"required" bun:",notnull"` // File size in bytes
UpdatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // When was the file last updated.
}
// Thumbnail refers to a small image thumbnail derived from a larger image, video, or audio file.
type Thumbnail struct {
Path string `validate:"required,file" bun:",nullzero,notnull"` // Path of the file in storage.
ContentType string `validate:"required" bun:",nullzero,notnull"` // MIME content type of the file.
FileSize int `validate:"required" bun:",notnull"` // File size in bytes
UpdatedAt time.Time `validate:"-" bun:"type:timestamptz,nullzero,notnull,default:current_timestamp"` // When was the file last updated.
URL string `validate:"required_without=RemoteURL,omitempty,url" bun:",nullzero"` // What is the URL of the thumbnail on the local server
RemoteURL string `validate:"required_without=URL,omitempty,url" bun:",nullzero"` // What is the remote URL of the thumbnail (empty for local media)
}
// ProcessingStatus refers to how far along in the processing stage the attachment is.
type ProcessingStatus int
// MediaAttachment processing states.
const (
ProcessingStatusReceived ProcessingStatus = 0 // ProcessingStatusReceived indicates the attachment has been received and is awaiting processing. No thumbnail available yet.
ProcessingStatusProcessing ProcessingStatus = 1 // ProcessingStatusProcessing indicates the attachment is currently being processed. Thumbnail is available but full media is not.
ProcessingStatusProcessed ProcessingStatus = 2 // ProcessingStatusProcessed indicates the attachment has been fully processed and is ready to be served.
ProcessingStatusError ProcessingStatus = 666 // ProcessingStatusError indicates something went wrong processing the attachment and it won't be tried again--these can be deleted.
)
// FileType refers to the file type of the media attaachment.
type FileType string
// MediaAttachment file types.
const (
FileTypeImage FileType = "Image" // FileTypeImage is for jpegs and pngs
FileTypeGif FileType = "Gif" // FileTypeGif is for native gifs and soundless videos that have been converted to gifs
FileTypeAudio FileType = "Audio" // FileTypeAudio is for audio-only files (no video)
FileTypeVideo FileType = "Video" // FileTypeVideo is for files with audio + visual
FileTypeUnknown FileType = "Unknown" // FileTypeUnknown is for unknown file types (surprise surprise!)
)
// FileMeta describes metadata about the actual contents of the file.
type FileMeta struct {
Original Original `validate:"required" bun:"embed:original_"`
Small Small `bun:"embed:small_"`
Focus Focus `bun:"embed:focus_"`
}
// Small can be used for a thumbnail of any media type
type Small struct {
Width int `validate:"required_with=Height Size Aspect"` // width in pixels
Height int `validate:"required_with=Width Size Aspect"` // height in pixels
Size int `validate:"required_with=Width Height Aspect"` // size in pixels (width * height)
Aspect float64 `validate:"required_with=Widhth Height Size"` // aspect ratio (width / height)
}
// Original can be used for original metadata for any media type
type Original struct {
Width int `validate:"required_with=Height Size Aspect"` // width in pixels
Height int `validate:"required_with=Width Size Aspect"` // height in pixels
Size int `validate:"required_with=Width Height Aspect"` // size in pixels (width * height)
Aspect float64 `validate:"required_with=Widhth Height Size"` // aspect ratio (width / height)
}
// Focus describes the 'center' of the image for display purposes.
// X and Y should each be between -1 and 1
type Focus struct {
X float32 `validate:"omitempty,max=1,min=-1"`
Y float32 `validate:"omitempty,max=1,min=-1"`
}

14
internal/db/bundb/util.go
View file

@ -37,6 +37,20 @@ func whereEmptyOrNull(column string) func(*bun.SelectQuery) *bun.SelectQuery {
}
}
// whereNotEmptyAndNotNull is a convenience function to return a bun WhereGroup that specifies
// that the given column should be NEITHER an empty string NOR null.
//
// Use it as follows:
//
// q = q.WhereGroup(" AND ", whereNotEmptyAndNotNull("whatever_column"))
func whereNotEmptyAndNotNull(column string) func(*bun.SelectQuery) *bun.SelectQuery {
return func(q *bun.SelectQuery) *bun.SelectQuery {
return q.
Where("? IS NOT NULL", bun.Ident(column)).
Where("? != ''", bun.Ident(column))
}
}
// updateWhere parses []db.Where and adds it to the given update query.
func updateWhere(q *bun.UpdateQuery, where []db.Where) {
for _, w := range where {

7
internal/db/media.go
View file

@ -20,6 +20,7 @@ package db
import (
"context"
"time"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
@ -28,4 +29,10 @@ import (
type Media interface {
// GetAttachmentByID gets a single attachment by its ID
GetAttachmentByID(ctx context.Context, id string) (*gtsmodel.MediaAttachment, Error)
// GetRemoteOlderThan gets limit n remote media attachments older than the given olderThan time.
// These will be returned in order of attachment.created_at descending (newest to oldest in other words).
//
// The selected media attachments will be those with both a URL and a RemoteURL filled in.
// In other words, media attachments that originated remotely, and that we currently have cached locally.
GetRemoteOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.MediaAttachment, Error)
}

6
internal/federation/federator.go
View file

@ -39,6 +39,8 @@ type Federator interface {
FederatingActor() pub.FederatingActor
// FederatingDB returns the underlying FederatingDB interface.
FederatingDB() federatingdb.DB
// TransportController returns the underlying transport controller.
TransportController() transport.Controller
// AuthenticateFederatedRequest can be used to check the authenticity of incoming http-signed requests for federating resources.
// The given username will be used to create a transport for making outgoing requests. See the implementation for more detailed comments.
@ -107,3 +109,7 @@ func (f *federator) FederatingActor() pub.FederatingActor {
func (f *federator) FederatingDB() federatingdb.DB {
return f.federatingDB
}
func (f *federator) TransportController() transport.Controller {
return f.transportController
}

13
internal/gtsmodel/mediaattachment.go
View file

@ -32,17 +32,18 @@ type MediaAttachment struct {
URL string `validate:"required_without=RemoteURL,omitempty,url" bun:",nullzero"` // Where can the attachment be retrieved on *this* server
RemoteURL string `validate:"required_without=URL,omitempty,url" bun:",nullzero"` // Where can the attachment be retrieved on a remote server (empty for local media)
Type FileType `validate:"oneof=Image Gif Audio Video Unknown" bun:",nullzero,notnull"` // Type of file (image/gif/audio/video)
FileMeta FileMeta `validate:"required" bun:",nullzero,notnull"` // Metadata about the file
FileMeta FileMeta `validate:"required" bun:",embed:filemeta_,nullzero,notnull"` // Metadata about the file
AccountID string `validate:"required,ulid" bun:"type:CHAR(26),nullzero,notnull"` // To which account does this attachment belong
Account *Account `validate:"-" bun:"rel:has-one"` // Account corresponding to accountID
Description string `validate:"-" bun:""` // Description of the attachment (for screenreaders)
ScheduledStatusID string `validate:"omitempty,ulid" bun:"type:CHAR(26),nullzero"` // To which scheduled status does this attachment belong
Blurhash string `validate:"required_if=Type Image,required_if=Type Gif,required_if=Type Video" bun:",nullzero"` // What is the generated blurhash of this attachment
Processing ProcessingStatus `validate:"oneof=0 1 2 666" bun:",notnull,default:2"` // What is the processing status of this attachment
File File `validate:"required" bun:",notnull,nullzero"` // metadata for the whole file
Thumbnail Thumbnail `validate:"required" bun:",notnull,nullzero"` // small image thumbnail derived from a larger image, video, or audio file.
File File `validate:"required" bun:",embed:file_,notnull,nullzero"` // metadata for the whole file
Thumbnail Thumbnail `validate:"required" bun:",embed:thumbnail_,notnull,nullzero"` // small image thumbnail derived from a larger image, video, or audio file.
Avatar bool `validate:"-" bun:",notnull,default:false"` // Is this attachment being used as an avatar?
Header bool `validate:"-" bun:",notnull,default:false"` // Is this attachment being used as a header?
Cached bool `validate:"-" bun:",notnull"` // Is this attachment currently cached by our instance?
}
// File refers to the metadata for the whole file
@ -88,9 +89,9 @@ const (
// FileMeta describes metadata about the actual contents of the file.
type FileMeta struct {
Original Original `validate:"required"`
Small Small
Focus Focus
Original Original `validate:"required" bun:"embed:original_"`
Small Small `bun:"embed:small_"`
Focus Focus `bun:"embed:focus_"`
}
// Small can be used for a thumbnail of any media type

107
internal/media/manager.go
View file

@ -21,11 +21,16 @@ package media
import (
"context"
"errors"
"fmt"
"runtime"
"time"
"codeberg.org/gruf/go-runners"
"codeberg.org/gruf/go-store/kv"
"github.com/robfig/cron/v3"
"github.com/sirupsen/logrus"
"github.com/spf13/viper"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
)
@ -61,6 +66,12 @@ type Manager interface {
//
// ai is optional and can be nil. Any additional information about the emoji provided will be put in the database.
ProcessEmoji(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, shortcode string, id string, uri string, ai *AdditionalEmojiInfo) (*ProcessingEmoji, error)
// RecacheMedia refetches, reprocesses, and recaches an existing attachment that has been uncached via pruneRemote.
RecacheMedia(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, attachmentID string) (*ProcessingMedia, error)
// PruneRemote prunes all remote media cached on this instance that's older than the given amount of days.
// 'Pruning' in this context means removing the locally stored data of the attachment (both thumbnail and full size),
// and setting 'cached' to false on the associated attachment.
PruneRemote(ctx context.Context, olderThanDays int) (int, error)
// NumWorkers returns the total number of workers available to this manager.
NumWorkers() int
// QueueSize returns the total capacity of the queue.
@ -76,11 +87,12 @@ type Manager interface {
}
type manager struct {
db db.DB
storage *kv.KVStore
pool runners.WorkerPool
numWorkers int
queueSize int
db db.DB
storage *kv.KVStore
pool runners.WorkerPool
stopCronJobs func() error
numWorkers int
queueSize int
}
// NewManager returns a media manager with the given db and underlying storage.
@ -97,8 +109,10 @@ type manager struct {
// For a 4 core machine, this will be 2 workers, and a queue length of 20.
// For a single or 2-core machine, the media manager will get 1 worker, and a queue of length 10.
func NewManager(database db.DB, storage *kv.KVStore) (Manager, error) {
numWorkers := runtime.NumCPU() / 2
// configure the worker pool
// make sure we always have at least 1 worker even on single-core machines
numWorkers := runtime.NumCPU() / 2
if numWorkers == 0 {
numWorkers = 1
}
@ -112,11 +126,61 @@ func NewManager(database db.DB, storage *kv.KVStore) (Manager, error) {
queueSize: queueSize,
}
// start the worker pool
if start := m.pool.Start(); !start {
return nil, errors.New("could not start worker pool")
}
logrus.Debugf("started media manager worker pool with %d workers and queue capacity of %d", numWorkers, queueSize)
// start remote cache cleanup cronjob if configured
cacheCleanupDays := viper.GetInt(config.Keys.MediaRemoteCacheDays)
if cacheCleanupDays != 0 {
// we need a way of cancelling running jobs if the media manager is told to stop
pruneCtx, pruneCancel := context.WithCancel(context.Background())
// create a new cron instance and add a function to it
c := cron.New(cron.WithLogger(&logrusWrapper{}))
pruneFunc := func() {
begin := time.Now()
pruned, err := m.PruneRemote(pruneCtx, cacheCleanupDays)
if err != nil {
logrus.Errorf("media manager: error pruning remote cache: %s", err)
return
}
logrus.Infof("media manager: pruned %d remote cache entries in %s", pruned, time.Since(begin))
}
// run every night
entryID, err := c.AddFunc("@midnight", pruneFunc)
if err != nil {
pruneCancel()
return nil, fmt.Errorf("error starting media manager remote cache cleanup job: %s", err)
}
// since we're running a cron job, we should define how the manager should stop them
m.stopCronJobs = func() error {
// try to stop any jobs gracefully by waiting til they're finished
cronCtx := c.Stop()
select {
case <-cronCtx.Done():
logrus.Infof("media manager: cron finished jobs and stopped gracefully")
case <-time.After(1 * time.Minute):
logrus.Infof("media manager: cron didn't stop after 60 seconds, will force close")
break
}
// whether the job is finished neatly or we had to wait a minute, cancel the context on the prune job
pruneCancel()
return nil
}
// now start all the cron stuff we've lined up
c.Start()
logrus.Infof("started media manager remote cache cleanup job: will run next at %s", c.Entry(entryID).Next)
}
return m, nil
}
@ -168,6 +232,30 @@ func (m *manager) ProcessEmoji(ctx context.Context, data DataFunc, postData Post
return processingEmoji, nil
}
func (m *manager) RecacheMedia(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, attachmentID string) (*ProcessingMedia, error) {
processingRecache, err := m.preProcessRecache(ctx, data, postData, attachmentID)
if err != nil {
return nil, err
}
logrus.Tracef("RecacheMedia: about to enqueue recache with attachmentID %s, queue length is %d", processingRecache.AttachmentID(), m.pool.Queue())
m.pool.Enqueue(func(innerCtx context.Context) {
select {
case <-innerCtx.Done():
// if the inner context is done that means the worker pool is closing, so we should just return
return
default:
// start loading the media already for the caller's convenience
if _, err := processingRecache.LoadAttachment(innerCtx); err != nil {
logrus.Errorf("RecacheMedia: error processing recache with attachmentID %s: %s", processingRecache.AttachmentID(), err)
}
}
})
logrus.Tracef("RecacheMedia: succesfully queued recache with attachmentID %s, queue length is %d", processingRecache.AttachmentID(), m.pool.Queue())
return processingRecache, nil
}
func (m *manager) NumWorkers() int {
return m.numWorkers
}
@ -186,10 +274,15 @@ func (m *manager) ActiveWorkers() int {
func (m *manager) Stop() error {
logrus.Info("stopping media manager worker pool")
stopped := m.pool.Stop()
if !stopped {
return errors.New("could not stop media manager worker pool")
}
if m.stopCronJobs != nil { // only defined if cron jobs are actually running
logrus.Info("stopping media manager cache cleanup jobs")
return m.stopCronJobs()
}
return nil
}

2
internal/media/manager_test.go
View file

@ -31,7 +31,7 @@ import (
"codeberg.org/gruf/go-store/kv"
"codeberg.org/gruf/go-store/storage"
"github.com/stretchr/testify/suite"
gtsmodel "github.com/superseriousbusiness/gotosocial/internal/db/bundb/migrations/20211113114307_init"
gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
)

9
internal/media/media_test.go
View file

@ -22,6 +22,7 @@ import (
"codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/db"
gtsmodel "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@ -29,9 +30,10 @@ import (
type MediaStandardTestSuite struct {
suite.Suite
db db.DB
storage *kv.KVStore
manager media.Manager
db db.DB
storage *kv.KVStore
manager media.Manager
testAttachments map[string]*gtsmodel.MediaAttachment
}
func (suite *MediaStandardTestSuite) SetupSuite() {
@ -45,6 +47,7 @@ func (suite *MediaStandardTestSuite) SetupSuite() {
func (suite *MediaStandardTestSuite) SetupTest() {
testrig.StandardStorageSetup(suite.storage, "../../testrig/media")
testrig.StandardDBSetup(suite.db, nil)
suite.testAttachments = testrig.NewTestAttachments()
suite.manager = testrig.NewTestMediaManager(suite.db, suite.storage)
}

38
internal/media/processingmedia.go
View file

@ -66,6 +66,9 @@ type ProcessingMedia struct {
// track whether this media has already been put in the databse
insertedInDB bool
// true if this is a recache, false if it's brand new media
recache bool
}
// AttachmentID returns the ID of the underlying media attachment without blocking processing.
@ -93,8 +96,16 @@ func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAt
// store the result in the database before returning it
if !p.insertedInDB {
if err := p.database.Put(ctx, p.attachment); err != nil {
return nil, err
if p.recache {
// if it's a recache we should only need to update
if err := p.database.UpdateByPrimaryKey(ctx, p.attachment); err != nil {
return nil, err
}
} else {
// otherwise we need to really PUT it
if err := p.database.Put(ctx, p.attachment); err != nil {
return nil, err
}
}
p.insertedInDB = true
}
@ -305,6 +316,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
if err := p.storage.PutStream(p.attachment.File.Path, clean); err != nil {
return fmt.Errorf("store: error storing stream: %s", err)
}
p.attachment.Cached = true
// if the original reader is a readcloser, close it since we're done with it now
if rc, ok := reader.(io.ReadCloser); ok {
@ -360,6 +372,7 @@ func (m *manager) preProcessMedia(ctx context.Context, data DataFunc, postData P
Thumbnail: thumbnail,
Avatar: false,
Header: false,
Cached: false,
}
// check if we have additional info to add to the attachment,
@ -418,3 +431,24 @@ func (m *manager) preProcessMedia(ctx context.Context, data DataFunc, postData P
return processingMedia, nil
}
func (m *manager) preProcessRecache(ctx context.Context, data DataFunc, postData PostDataCallbackFunc, attachmentID string) (*ProcessingMedia, error) {
// get the existing attachment
attachment, err := m.db.GetAttachmentByID(ctx, attachmentID)
if err != nil {
return nil, err
}
processingMedia := &ProcessingMedia{
attachment: attachment,
data: data,
postData: postData,
thumbState: int32(received),
fullSizeState: int32(received),
database: m.db,
storage: m.storage,
recache: true, // indicate it's a recache
}
return processingMedia, nil
}

96
internal/media/pruneremote.go Normal file
View file

@ -0,0 +1,96 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package media
import (
"context"
"fmt"
"time"
"codeberg.org/gruf/go-store/storage"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
// amount of media attachments to select at a time from the db when pruning
const selectPruneLimit = 20
func (m *manager) PruneRemote(ctx context.Context, olderThanDays int) (int, error) {
var totalPruned int
// convert days into a duration string
olderThanHoursString := fmt.Sprintf("%dh", olderThanDays*24)
// parse the duration string into a duration
olderThanHours, err := time.ParseDuration(olderThanHoursString)
if err != nil {
return totalPruned, fmt.Errorf("PruneRemote: %d", err)
}
// 'subtract' that from the time now to give our threshold
olderThan := time.Now().Add(-olderThanHours)
logrus.Infof("PruneRemote: pruning media older than %s", olderThan)
// select 20 attachments at a time and prune them
for attachments, err := m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit) {
// use the age of the oldest attachment (the last one in the slice) as the next 'older than' value
l := len(attachments)
logrus.Tracef("PruneRemote: got %d attachments older than %s", l, olderThan)
olderThan = attachments[l-1].CreatedAt
// prune each attachment
for _, attachment := range attachments {
if err := m.PruneOne(ctx, attachment); err != nil {
return totalPruned, err
}
totalPruned++
}
}
// make sure we don't have a real error when we leave the loop
if err != nil && err != db.ErrNoEntries {
return totalPruned, err
}
logrus.Infof("PruneRemote: finished pruning remote media: pruned %d entries", totalPruned)
return totalPruned, nil
}
func (m *manager) PruneOne(ctx context.Context, attachment *gtsmodel.MediaAttachment) error {
if attachment.File.Path != "" {
// delete the full size attachment from storage
logrus.Tracef("PruneOne: deleting %s", attachment.File.Path)
if err := m.storage.Delete(attachment.File.Path); err != nil && err != storage.ErrNotFound {
return err
}
attachment.Cached = false
}
if attachment.Thumbnail.Path != "" {
// delete the thumbnail from storage
logrus.Tracef("PruneOne: deleting %s", attachment.Thumbnail.Path)
if err := m.storage.Delete(attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
return err
}
attachment.Cached = false
}
// update the attachment to reflect that we no longer have it cached
return m.db.UpdateByPrimaryKey(ctx, attachment)
}

111
internal/media/pruneremote_test.go Normal file
View file

@ -0,0 +1,111 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package media_test
import (
"bytes"
"context"
"io"
"os"
"testing"
"codeberg.org/gruf/go-store/storage"
"github.com/stretchr/testify/suite"
)
type PruneRemoteTestSuite struct {
MediaStandardTestSuite
}
func (suite *PruneRemoteTestSuite) TestPruneRemote() {
testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
suite.True(testAttachment.Cached)
totalPruned, err := suite.manager.PruneRemote(context.Background(), 1)
suite.NoError(err)
suite.Equal(1, totalPruned)
prunedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testAttachment.ID)
suite.NoError(err)
// the media should no longer be cached
suite.False(prunedAttachment.Cached)
}
func (suite *PruneRemoteTestSuite) TestPruneRemoteTwice() {
totalPruned, err := suite.manager.PruneRemote(context.Background(), 1)
suite.NoError(err)
suite.Equal(1, totalPruned)
// final prune should prune nothing, since the first prune already happened
totalPrunedAgain, err := suite.manager.PruneRemote(context.Background(), 1)
suite.NoError(err)
suite.Equal(0, totalPrunedAgain)
}
func (suite *PruneRemoteTestSuite) TestPruneAndRecache() {
ctx := context.Background()
testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
totalPruned, err := suite.manager.PruneRemote(ctx, 1)
suite.NoError(err)
suite.Equal(1, totalPruned)
// media should no longer be stored
_, err = suite.storage.Get(testAttachment.File.Path)
suite.Error(err)
suite.ErrorIs(err, storage.ErrNotFound)
_, err = suite.storage.Get(testAttachment.Thumbnail.Path)
suite.Error(err)
suite.ErrorIs(err, storage.ErrNotFound)
// now recache the image....
data := func(_ context.Context) (io.Reader, int, error) {
// load bytes from a test image
b, err := os.ReadFile("../../testrig/media/thoughtsofdog-original.jpeg")
if err != nil {
panic(err)
}
return bytes.NewBuffer(b), len(b), nil
}
processingRecache, err := suite.manager.RecacheMedia(ctx, data, nil, testAttachment.ID)
suite.NoError(err)
// synchronously load the recached attachment
recachedAttachment, err := processingRecache.LoadAttachment(ctx)
suite.NoError(err)
suite.NotNil(recachedAttachment)
// recachedAttachment should be basically the same as the old attachment
suite.True(recachedAttachment.Cached)
suite.Equal(testAttachment.ID, recachedAttachment.ID)
suite.Equal(testAttachment.File.Path, recachedAttachment.File.Path) // file should be stored in the same place
suite.Equal(testAttachment.Thumbnail.Path, recachedAttachment.Thumbnail.Path) // as should the thumbnail
suite.EqualValues(testAttachment.FileMeta, recachedAttachment.FileMeta) // and the filemeta should be the same
// recached files should be back in storage
_, err = suite.storage.Get(recachedAttachment.File.Path)
suite.NoError(err)
_, err = suite.storage.Get(recachedAttachment.Thumbnail.Path)
suite.NoError(err)
}
func TestPruneRemoteTestSuite(t *testing.T) {
suite.Run(t, &PruneRemoteTestSuite{})
}

15
internal/media/util.go
View file

@ -23,6 +23,7 @@ import (
"fmt"
"github.com/h2non/filetype"
"github.com/sirupsen/logrus"
)
// parseContentType parses the MIME content type from a file, returning it as a string in the form (eg., "image/jpeg").
@ -103,3 +104,17 @@ func ParseMediaSize(s string) (Size, error) {
}
return "", fmt.Errorf("%s not a recognized MediaSize", s)
}
// logrusWrapper is just a util for passing the logrus logger into the cron logging system.
type logrusWrapper struct {
}
// Info logs routine messages about cron's operation.
func (l *logrusWrapper) Info(msg string, keysAndValues ...interface{}) {
logrus.Info("media manager cron logger: ", msg, keysAndValues)
}
// Error logs an error condition.
func (l *logrusWrapper) Error(err error, msg string, keysAndValues ...interface{}) {
logrus.Error("media manager cron logger: ", err, msg, keysAndValues)
}

2
internal/processing/media.go
View file

@ -38,6 +38,6 @@ func (p *processor) MediaUpdate(ctx context.Context, authed *oauth.Auth, mediaAt
return p.mediaProcessor.Update(ctx, authed.Account, mediaAttachmentID, form)
}
func (p *processor) FileGet(ctx context.Context, authed *oauth.Auth, form *apimodel.GetContentRequestForm) (*apimodel.Content, error) {
func (p *processor) FileGet(ctx context.Context, authed *oauth.Auth, form *apimodel.GetContentRequestForm) (*apimodel.Content, gtserror.WithCode) {
return p.mediaProcessor.GetFile(ctx, authed.Account, form)
}

225
internal/processing/media/getfile.go
View file

@ -19,8 +19,11 @@
package media
import (
"bufio"
"context"
"fmt"
"io"
"net/url"
"strings"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
@ -29,7 +32,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/media"
)
func (p *processor) GetFile(ctx context.Context, account *gtsmodel.Account, form *apimodel.GetContentRequestForm) (*apimodel.Content, error) {
func (p *processor) GetFile(ctx context.Context, account *gtsmodel.Account, form *apimodel.GetContentRequestForm) (*apimodel.Content, gtserror.WithCode) {
// parse the form fields
mediaSize, err := media.ParseMediaSize(form.MediaSize)
if err != nil {
@ -46,74 +49,208 @@ func (p *processor) GetFile(ctx context.Context, account *gtsmodel.Account, form
return nil, gtserror.NewErrorNotFound(fmt.Errorf("file name %s not parseable", form.FileName))
}
wantedMediaID := spl[0]
expectedAccountID := form.AccountID
// get the account that owns the media and make sure it's not suspended
acct, err := p.db.GetAccountByID(ctx, form.AccountID)
acct, err := p.db.GetAccountByID(ctx, expectedAccountID)
if err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("account with id %s could not be selected from the db: %s", form.AccountID, err))
return nil, gtserror.NewErrorNotFound(fmt.Errorf("account with id %s could not be selected from the db: %s", expectedAccountID, err))
}
if !acct.SuspendedAt.IsZero() {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("account with id %s is suspended", form.AccountID))
return nil, gtserror.NewErrorNotFound(fmt.Errorf("account with id %s is suspended", expectedAccountID))
}
// make sure the requesting account and the media account don't block each other
if account != nil {
blocked, err := p.db.IsBlocked(ctx, account.ID, form.AccountID, true)
blocked, err := p.db.IsBlocked(ctx, account.ID, expectedAccountID, true)
if err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("block status could not be established between accounts %s and %s: %s", form.AccountID, account.ID, err))
return nil, gtserror.NewErrorNotFound(fmt.Errorf("block status could not be established between accounts %s and %s: %s", expectedAccountID, account.ID, err))
}
if blocked {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("block exists between accounts %s and %s", form.AccountID, account.ID))
return nil, gtserror.NewErrorNotFound(fmt.Errorf("block exists between accounts %s and %s", expectedAccountID, account.ID))
}
}
// the way we store emojis is a little different from the way we store other attachments,
// so we need to take different steps depending on the media type being requested
content := &apimodel.Content{}
var storagePath string
switch mediaType {
case media.TypeEmoji:
e := &gtsmodel.Emoji{}
if err := p.db.GetByID(ctx, wantedMediaID, e); err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s could not be taken from the db: %s", wantedMediaID, err))
}
if e.Disabled {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s has been disabled", wantedMediaID))
}
switch mediaSize {
case media.SizeOriginal:
content.ContentType = e.ImageContentType
content.ContentLength = int64(e.ImageFileSize)
storagePath = e.ImagePath
case media.SizeStatic:
content.ContentType = e.ImageStaticContentType
content.ContentLength = int64(e.ImageStaticFileSize)
storagePath = e.ImageStaticPath
default:
return nil, gtserror.NewErrorNotFound(fmt.Errorf("media size %s not recognized for emoji", mediaSize))
}
return p.getEmojiContent(ctx, wantedMediaID, mediaSize)
case media.TypeAttachment, media.TypeHeader, media.TypeAvatar:
a, err := p.db.GetAttachmentByID(ctx, wantedMediaID)
if err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s could not be taken from the db: %s", wantedMediaID, err))
return p.getAttachmentContent(ctx, account, wantedMediaID, expectedAccountID, mediaSize)
default:
return nil, gtserror.NewErrorNotFound(fmt.Errorf("media type %s not recognized", mediaType))
}
}
func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount *gtsmodel.Account, wantedMediaID string, expectedAccountID string, mediaSize media.Size) (*apimodel.Content, gtserror.WithCode) {
attachmentContent := &apimodel.Content{}
var storagePath string
// retrieve attachment from the database and do basic checks on it
a, err := p.db.GetAttachmentByID(ctx, wantedMediaID)
if err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s could not be taken from the db: %s", wantedMediaID, err))
}
if a.AccountID != expectedAccountID {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s is not owned by %s", wantedMediaID, expectedAccountID))
}
// get file information from the attachment depending on the requested media size
switch mediaSize {
case media.SizeOriginal:
attachmentContent.ContentType = a.File.ContentType
attachmentContent.ContentLength = int64(a.File.FileSize)
storagePath = a.File.Path
case media.SizeSmall:
attachmentContent.ContentType = a.Thumbnail.ContentType
attachmentContent.ContentLength = int64(a.Thumbnail.FileSize)
storagePath = a.Thumbnail.Path
default:
return nil, gtserror.NewErrorNotFound(fmt.Errorf("media size %s not recognized for attachment", mediaSize))
}
// if we have the media cached on our server already, we can now simply return it from storage
if a.Cached {
return p.streamFromStorage(storagePath, attachmentContent)
}
// if we don't have it cached, then we can assume two things:
// 1. this is remote media, since local media should never be uncached
// 2. we need to fetch it again using a transport and the media manager
remoteMediaIRI, err := url.Parse(a.RemoteURL)
if err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("error parsing remote media iri %s: %s", a.RemoteURL, err))
}
// use an empty string as requestingUsername to use the instance account, unless the request for this
// media has been http signed, then use the requesting account to make the request to remote server
var requestingUsername string
if requestingAccount != nil {
requestingUsername = requestingAccount.Username
}
var data media.DataFunc
var postDataCallback media.PostDataCallbackFunc
if mediaSize == media.SizeSmall {
// if it's the thumbnail that's requested then the user will have to wait a bit while we process the
// large version and derive a thumbnail from it, so use the normal recaching procedure: fetch the media,
// process it, then return the thumbnail data
data = func(innerCtx context.Context) (io.Reader, int, error) {
transport, err := p.transportController.NewTransportForUsername(innerCtx, requestingUsername)
if err != nil {
return nil, 0, err
}
return transport.DereferenceMedia(innerCtx, remoteMediaIRI)
}
if a.AccountID != form.AccountID {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("attachment %s is not owned by %s", wantedMediaID, form.AccountID))
} else {
// if it's the full-sized version being requested, we can cheat a bit by streaming data to the user as
// it's retrieved from the remote server, using tee; this saves the user from having to wait while
// we process the media on our side
//
// this looks a bit like this:
//
// http fetch buffered pipe
// remote server ------------> data function ----------------> api caller
// |
// | tee
// |
// ▼
// instance storage
// Buffer each end of the pipe, so that if the caller drops the connection during the flow, the tee
// reader can continue without having to worry about tee-ing into a closed or blocked pipe.
pipeReader, pipeWriter := io.Pipe()
bufferedWriter := bufio.NewWriterSize(pipeWriter, int(attachmentContent.ContentLength))
bufferedReader := bufio.NewReaderSize(pipeReader, int(attachmentContent.ContentLength))
// the caller will read from the buffered reader, so it doesn't matter if they drop out without reading everything
attachmentContent.Content = bufferedReader
data = func(innerCtx context.Context) (io.Reader, int, error) {
transport, err := p.transportController.NewTransportForUsername(innerCtx, requestingUsername)
if err != nil {
return nil, 0, err
}
readCloser, fileSize, err := transport.DereferenceMedia(innerCtx, remoteMediaIRI)
if err != nil {
return nil, 0, err
}
// everything read from the readCloser by the media manager will be written into the bufferedWriter
teeReader := io.TeeReader(readCloser, bufferedWriter)
return teeReader, fileSize, nil
}
switch mediaSize {
case media.SizeOriginal:
content.ContentType = a.File.ContentType
content.ContentLength = int64(a.File.FileSize)
storagePath = a.File.Path
case media.SizeSmall:
content.ContentType = a.Thumbnail.ContentType
content.ContentLength = int64(a.Thumbnail.FileSize)
storagePath = a.Thumbnail.Path
default:
return nil, gtserror.NewErrorNotFound(fmt.Errorf("media size %s not recognized for attachment", mediaSize))
// close the pipewriter after data has been piped into it, so the reader on the other side doesn't block;
// we don't need to close the reader here because that's the caller's responsibility
postDataCallback = func(innerCtx context.Context) error {
// flush the buffered writer into the buffer of the reader...
if err := bufferedWriter.Flush(); err != nil {
return err
}
// and close the underlying pipe writer
if err := pipeWriter.Close(); err != nil {
return err
}
return nil
}
}
// put the media recached in the queue
processingMedia, err := p.mediaManager.RecacheMedia(ctx, data, postDataCallback, wantedMediaID)
if err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("error recaching media: %s", err))
}
// if it's the thumbnail, stream the processed thumbnail from storage, after waiting for processing to finish
if mediaSize == media.SizeSmall {
// below function call blocks until all processing on the attachment has finished...
if _, err := processingMedia.LoadAttachment(ctx); err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("error loading recached attachment: %s", err))
}
// ... so now we can safely return it
return p.streamFromStorage(storagePath, attachmentContent)
}
return attachmentContent, nil
}
func (p *processor) getEmojiContent(ctx context.Context, wantedEmojiID string, emojiSize media.Size) (*apimodel.Content, gtserror.WithCode) {
emojiContent := &apimodel.Content{}
var storagePath string
e := &gtsmodel.Emoji{}
if err := p.db.GetByID(ctx, wantedEmojiID, e); err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s could not be taken from the db: %s", wantedEmojiID, err))
}
if e.Disabled {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("emoji %s has been disabled", wantedEmojiID))
}
switch emojiSize {
case media.SizeOriginal:
emojiContent.ContentType = e.ImageContentType
emojiContent.ContentLength = int64(e.ImageFileSize)
storagePath = e.ImagePath
case media.SizeStatic:
emojiContent.ContentType = e.ImageStaticContentType
emojiContent.ContentLength = int64(e.ImageStaticFileSize)
storagePath = e.ImageStaticPath
default:
return nil, gtserror.NewErrorNotFound(fmt.Errorf("media size %s not recognized for emoji", emojiSize))
}
return p.streamFromStorage(storagePath, emojiContent)
}
func (p *processor) streamFromStorage(storagePath string, content *apimodel.Content) (*apimodel.Content, gtserror.WithCode) {
reader, err := p.storage.GetStream(storagePath)
if err != nil {
return nil, gtserror.NewErrorNotFound(fmt.Errorf("error retrieving from storage: %s", err))

208
internal/processing/media/getfile_test.go Normal file
View file

@ -0,0 +1,208 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package media_test
import (
"context"
"io"
"path"
"testing"
"time"
"github.com/stretchr/testify/suite"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/media"
)
type GetFileTestSuite struct {
MediaStandardTestSuite
}
func (suite *GetFileTestSuite) TestGetRemoteFileCached() {
ctx := context.Background()
testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
fileName := path.Base(testAttachment.File.Path)
requestingAccount := suite.testAccounts["local_account_1"]
content, errWithCode := suite.mediaProcessor.GetFile(ctx, requestingAccount, &apimodel.GetContentRequestForm{
AccountID: testAttachment.AccountID,
MediaType: string(media.TypeAttachment),
MediaSize: string(media.SizeOriginal),
FileName: fileName,
})
suite.NoError(errWithCode)
suite.NotNil(content)
b, err := io.ReadAll(content.Content)
suite.NoError(err)
if closer, ok := content.Content.(io.Closer); ok {
suite.NoError(closer.Close())
}
suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, b)
suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].ContentType, content.ContentType)
suite.EqualValues(len(suite.testRemoteAttachments[testAttachment.RemoteURL].Data), content.ContentLength)
}
func (suite *GetFileTestSuite) TestGetRemoteFileUncached() {
ctx := context.Background()
// uncache the file from local
testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
testAttachment.Cached = false
err := suite.db.UpdateByPrimaryKey(ctx, testAttachment)
suite.NoError(err)
err = suite.storage.Delete(testAttachment.File.Path)
suite.NoError(err)
err = suite.storage.Delete(testAttachment.Thumbnail.Path)
suite.NoError(err)
// now fetch it
fileName := path.Base(testAttachment.File.Path)
requestingAccount := suite.testAccounts["local_account_1"]
content, errWithCode := suite.mediaProcessor.GetFile(ctx, requestingAccount, &apimodel.GetContentRequestForm{
AccountID: testAttachment.AccountID,
MediaType: string(media.TypeAttachment),
MediaSize: string(media.SizeOriginal),
FileName: fileName,
})
suite.NoError(errWithCode)
suite.NotNil(content)
b, err := io.ReadAll(content.Content)
suite.NoError(err)
if closer, ok := content.Content.(io.Closer); ok {
suite.NoError(closer.Close())
}
suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, b)
suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].ContentType, content.ContentType)
suite.EqualValues(len(suite.testRemoteAttachments[testAttachment.RemoteURL].Data), content.ContentLength)
time.Sleep(2 * time.Second) // wait a few seconds for the media manager to finish doing stuff
// the attachment should be updated in the database
dbAttachment, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID)
suite.NoError(err)
suite.True(dbAttachment.Cached)
// the file should be back in storage at the same path as before
refreshedBytes, err := suite.storage.Get(testAttachment.File.Path)
suite.NoError(err)
suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, refreshedBytes)
}
func (suite *GetFileTestSuite) TestGetRemoteFileUncachedInterrupted() {
ctx := context.Background()
// uncache the file from local
testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
testAttachment.Cached = false
err := suite.db.UpdateByPrimaryKey(ctx, testAttachment)
suite.NoError(err)
err = suite.storage.Delete(testAttachment.File.Path)
suite.NoError(err)
err = suite.storage.Delete(testAttachment.Thumbnail.Path)
suite.NoError(err)
// now fetch it
fileName := path.Base(testAttachment.File.Path)
requestingAccount := suite.testAccounts["local_account_1"]
content, errWithCode := suite.mediaProcessor.GetFile(ctx, requestingAccount, &apimodel.GetContentRequestForm{
AccountID: testAttachment.AccountID,
MediaType: string(media.TypeAttachment),
MediaSize: string(media.SizeOriginal),
FileName: fileName,
})
suite.NoError(errWithCode)
suite.NotNil(content)
// only read the first kilobyte and then stop
b := make([]byte, 1024)
_, err = content.Content.Read(b)
suite.NoError(err)
// close the reader
if closer, ok := content.Content.(io.Closer); ok {
suite.NoError(closer.Close())
}
time.Sleep(2 * time.Second) // wait a few seconds for the media manager to finish doing stuff
// the attachment should still be updated in the database even though the caller hung up
dbAttachment, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID)
suite.NoError(err)
suite.True(dbAttachment.Cached)
// the file should be back in storage at the same path as before
refreshedBytes, err := suite.storage.Get(testAttachment.File.Path)
suite.NoError(err)
suite.Equal(suite.testRemoteAttachments[testAttachment.RemoteURL].Data, refreshedBytes)
}
func (suite *GetFileTestSuite) TestGetRemoteFileThumbnailUncached() {
ctx := context.Background()
testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
// fetch the existing thumbnail bytes from storage first
thumbnailBytes, err := suite.storage.Get(testAttachment.Thumbnail.Path)
suite.NoError(err)
// uncache the file from local
testAttachment.Cached = false
err = suite.db.UpdateByPrimaryKey(ctx, testAttachment)
suite.NoError(err)
err = suite.storage.Delete(testAttachment.File.Path)
suite.NoError(err)
err = suite.storage.Delete(testAttachment.Thumbnail.Path)
suite.NoError(err)
// now fetch the thumbnail
fileName := path.Base(testAttachment.File.Path)
requestingAccount := suite.testAccounts["local_account_1"]
content, errWithCode := suite.mediaProcessor.GetFile(ctx, requestingAccount, &apimodel.GetContentRequestForm{
AccountID: testAttachment.AccountID,
MediaType: string(media.TypeAttachment),
MediaSize: string(media.SizeSmall),
FileName: fileName,
})
suite.NoError(errWithCode)
suite.NotNil(content)
b, err := io.ReadAll(content.Content)
suite.NoError(err)
if closer, ok := content.Content.(io.Closer); ok {
suite.NoError(closer.Close())
}
suite.Equal(thumbnailBytes, b)
suite.Equal("image/jpeg", content.ContentType)
suite.EqualValues(testAttachment.Thumbnail.FileSize, content.ContentLength)
}
func TestGetFileTestSuite(t *testing.T) {
suite.Run(t, &GetFileTestSuite{})
}

24
internal/processing/media/media.go
View file

@ -27,6 +27,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
)
@ -36,24 +37,27 @@ type Processor interface {
Create(ctx context.Context, account *gtsmodel.Account, form *apimodel.AttachmentRequest) (*apimodel.Attachment, error)
// Delete deletes the media attachment with the given ID, including all files pertaining to that attachment.
Delete(ctx context.Context, mediaAttachmentID string) gtserror.WithCode
GetFile(ctx context.Context, account *gtsmodel.Account, form *apimodel.GetContentRequestForm) (*apimodel.Content, error)
// GetFile retrieves a file from storage and streams it back to the caller via an io.reader embedded in *apimodel.Content.
GetFile(ctx context.Context, account *gtsmodel.Account, form *apimodel.GetContentRequestForm) (*apimodel.Content, gtserror.WithCode)
GetMedia(ctx context.Context, account *gtsmodel.Account, mediaAttachmentID string) (*apimodel.Attachment, gtserror.WithCode)
Update(ctx context.Context, account *gtsmodel.Account, mediaAttachmentID string, form *apimodel.AttachmentUpdateRequest) (*apimodel.Attachment, gtserror.WithCode)
}
type processor struct {
tc typeutils.TypeConverter
mediaManager media.Manager
storage *kv.KVStore
db db.DB
tc typeutils.TypeConverter
mediaManager media.Manager
transportController transport.Controller
storage *kv.KVStore
db db.DB
}
// New returns a new media processor.
func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, storage *kv.KVStore) Processor {
func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, transportController transport.Controller, storage *kv.KVStore) Processor {
return &processor{
tc: tc,
mediaManager: mediaManager,
storage: storage,
db: db,
tc: tc,
mediaManager: mediaManager,
transportController: transportController,
storage: storage,
db: db,
}
}

125
internal/processing/media/media_test.go Normal file
View file

@ -0,0 +1,125 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package media_test
import (
"bytes"
"io"
"net/http"
"codeberg.org/gruf/go-store/kv"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/media"
mediaprocessing "github.com/superseriousbusiness/gotosocial/internal/processing/media"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/internal/typeutils"
"github.com/superseriousbusiness/gotosocial/testrig"
)
type MediaStandardTestSuite struct {
// standard suite interfaces
suite.Suite
db db.DB
tc typeutils.TypeConverter
storage *kv.KVStore
mediaManager media.Manager
transportController transport.Controller
// standard suite models
testTokens map[string]*gtsmodel.Token
testClients map[string]*gtsmodel.Client
testApplications map[string]*gtsmodel.Application
testUsers map[string]*gtsmodel.User
testAccounts map[string]*gtsmodel.Account
testAttachments map[string]*gtsmodel.MediaAttachment
testStatuses map[string]*gtsmodel.Status
testRemoteAttachments map[string]testrig.RemoteAttachmentFile
// module being tested
mediaProcessor mediaprocessing.Processor
}
func (suite *MediaStandardTestSuite) SetupSuite() {
suite.testTokens = testrig.NewTestTokens()
suite.testClients = testrig.NewTestClients()
suite.testApplications = testrig.NewTestApplications()
suite.testUsers = testrig.NewTestUsers()
suite.testAccounts = testrig.NewTestAccounts()
suite.testAttachments = testrig.NewTestAttachments()
suite.testStatuses = testrig.NewTestStatuses()
suite.testRemoteAttachments = testrig.NewTestFediAttachments("../../../testrig/media")
}
func (suite *MediaStandardTestSuite) SetupTest() {
testrig.InitTestConfig()
testrig.InitTestLog()
suite.db = testrig.NewTestDB()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.storage = testrig.NewTestStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.transportController = suite.mockTransportController()
suite.mediaProcessor = mediaprocessing.New(suite.db, suite.tc, suite.mediaManager, suite.transportController, suite.storage)
testrig.StandardDBSetup(suite.db, nil)
testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")
}
func (suite *MediaStandardTestSuite) TearDownTest() {
testrig.StandardDBTeardown(suite.db)
testrig.StandardStorageTeardown(suite.storage)
}
func (suite *MediaStandardTestSuite) mockTransportController() transport.Controller {
do := func(req *http.Request) (*http.Response, error) {
logrus.Debugf("received request for %s", req.URL)
responseBytes := []byte{}
responseType := ""
responseLength := 0
if attachment, ok := suite.testRemoteAttachments[req.URL.String()]; ok {
responseBytes = attachment.Data
responseType = attachment.ContentType
}
if len(responseBytes) != 0 {
// we found something, so print what we're going to return
logrus.Debugf("returning response %s", string(responseBytes))
}
responseLength = len(responseBytes)
reader := bytes.NewReader(responseBytes)
readCloser := io.NopCloser(reader)
response := &http.Response{
StatusCode: 200,
Body: readCloser,
ContentLength: int64(responseLength),
Header: http.Header{
"content-type": {responseType},
},
}
return response, nil
}
mockClient := testrig.NewMockHTTPClient(do)
return testrig.NewTestTransportController(mockClient, suite.db)
}

4
internal/processing/processor.go
View file

@ -115,7 +115,7 @@ type Processor interface {
BlocksGet(ctx context.Context, authed *oauth.Auth, maxID string, sinceID string, limit int) (*apimodel.BlocksResponse, gtserror.WithCode)
// FileGet handles the fetching of a media attachment file via the fileserver.
FileGet(ctx context.Context, authed *oauth.Auth, form *apimodel.GetContentRequestForm) (*apimodel.Content, error)
FileGet(ctx context.Context, authed *oauth.Auth, form *apimodel.GetContentRequestForm) (*apimodel.Content, gtserror.WithCode)
// FollowRequestsGet handles the getting of the authed account's incoming follow requests
FollowRequestsGet(ctx context.Context, auth *oauth.Auth) ([]apimodel.Account, gtserror.WithCode)
@ -270,7 +270,7 @@ func NewProcessor(
streamingProcessor := streaming.New(db, oauthServer)
accountProcessor := account.New(db, tc, mediaManager, oauthServer, fromClientAPI, federator)
adminProcessor := admin.New(db, tc, mediaManager, fromClientAPI)
mediaProcessor := mediaProcessor.New(db, tc, mediaManager, storage)
mediaProcessor := mediaProcessor.New(db, tc, mediaManager, federator.TransportController(), storage)
userProcessor := user.New(db, emailSender)
federationProcessor := federationProcessor.New(db, tc, federator, fromFederator)
filter := visibility.NewFilter(db)

2
internal/processing/streaming/notification_test.go
View file

@ -52,7 +52,7 @@ func (suite *NotificationTestSuite) TestStreamNotification() {
suite.NoError(err)
msg := <-openStream.Messages
suite.Equal(`{"id":"01FH57SJCMDWQGEAJ0X08CE3WV","type":"follow","created_at":"2021-10-04T10:52:36+02:00","account":{"id":"01F8MH5ZK5VRH73AKHQM6Y9VNX","username":"foss_satan","acct":"foss_satan@fossbros-anonymous.io","display_name":"big gerald","locked":false,"bot":false,"created_at":"2021-09-26T12:52:36+02:00","note":"i post about like, i dunno, stuff, or whatever!!!!","url":"http://fossbros-anonymous.io/@foss_satan","avatar":"","avatar_static":"","header":"","header_static":"","followers_count":0,"following_count":0,"statuses_count":0,"last_status_at":"","emojis":[],"fields":[]}}`, msg.Payload)
suite.Equal(`{"id":"01FH57SJCMDWQGEAJ0X08CE3WV","type":"follow","created_at":"2021-10-04T10:52:36+02:00","account":{"id":"01F8MH5ZK5VRH73AKHQM6Y9VNX","username":"foss_satan","acct":"foss_satan@fossbros-anonymous.io","display_name":"big gerald","locked":false,"bot":false,"created_at":"2021-09-26T12:52:36+02:00","note":"i post about like, i dunno, stuff, or whatever!!!!","url":"http://fossbros-anonymous.io/@foss_satan","avatar":"","avatar_static":"","header":"","header_static":"","followers_count":0,"following_count":0,"statuses_count":1,"last_status_at":"2021-09-20T10:40:37Z","emojis":[],"fields":[]}}`, msg.Payload)
}
func TestNotificationTestSuite(t *testing.T) {

16
internal/timeline/get_test.go
View file

@ -84,8 +84,8 @@ func (suite *GetTestSuite) TestGetDefault() {
suite.FailNow(err.Error())
}
// we only have 14 statuses in the test suite
suite.Len(statuses, 14)
// we only have 15 statuses in the test suite
suite.Len(statuses, 15)
// statuses should be sorted highest to lowest ID
var highest string
@ -177,8 +177,8 @@ func (suite *GetTestSuite) TestGetMinID() {
suite.FailNow(err.Error())
}
// we should only get 7 statuses back, since we asked for a min ID that excludes some of our entries
suite.Len(statuses, 7)
// we should only get 8 statuses back, since we asked for a min ID that excludes some of our entries
suite.Len(statuses, 8)
// statuses should be sorted highest to lowest ID
var highest string
@ -199,8 +199,8 @@ func (suite *GetTestSuite) TestGetSinceID() {
suite.FailNow(err.Error())
}
// we should only get 7 statuses back, since we asked for a since ID that excludes some of our entries
suite.Len(statuses, 7)
// we should only get 8 statuses back, since we asked for a since ID that excludes some of our entries
suite.Len(statuses, 8)
// statuses should be sorted highest to lowest ID
var highest string
@ -221,8 +221,8 @@ func (suite *GetTestSuite) TestGetSinceIDPrepareNext() {
suite.FailNow(err.Error())
}
// we should only get 7 statuses back, since we asked for a since ID that excludes some of our entries
suite.Len(statuses, 7)
// we should only get 8 statuses back, since we asked for a since ID that excludes some of our entries
suite.Len(statuses, 8)
// statuses should be sorted highest to lowest ID
var highest string

22
test/cliparsing.sh
View file

@ -5,7 +5,7 @@ set -e
echo "STARTING CLI TESTS"
echo "TEST_1 Make sure defaults are set correctly."
TEST_1_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","help":false,"host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
TEST_1_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","help":false,"host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
TEST_1="$(go run ./cmd/gotosocial/... debug config)"
if [ "${TEST_1}" != "${TEST_1_EXPECTED}" ]; then
echo "TEST_1 not equal TEST_1_EXPECTED"
@ -15,7 +15,7 @@ else
fi
echo "TEST_2 Override db-address from default using cli flag."
TEST_2_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","help":false,"host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
TEST_2_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","help":false,"host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
TEST_2="$(go run ./cmd/gotosocial/... --db-address some.db.address debug config)"
if [ "${TEST_2}" != "${TEST_2_EXPECTED}" ]; then
echo "TEST_2 not equal TEST_2_EXPECTED"
@ -25,7 +25,7 @@ else
fi
echo "TEST_3 Override db-address from default using env var."
TEST_3_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","help":false,"host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
TEST_3_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","help":false,"host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
TEST_3="$(GTS_DB_ADDRESS=some.db.address go run ./cmd/gotosocial/... debug config)"
if [ "${TEST_3}" != "${TEST_3_EXPECTED}" ]; then
echo "TEST_3 not equal TEST_3_EXPECTED"
@ -35,7 +35,7 @@ else
fi
echo "TEST_4 Override db-address from default using both env var and cli flag. The cli flag should take priority."
TEST_4_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.other.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","help":false,"host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
TEST_4_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"","db-address":"some.other.db.address","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","help":false,"host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
TEST_4="$(GTS_DB_ADDRESS=some.db.address go run ./cmd/gotosocial/... --db-address some.other.db.address debug config)"
if [ "${TEST_4}" != "${TEST_4_EXPECTED}" ]; then
echo "TEST_4 not equal TEST_4_EXPECTED"
@ -45,7 +45,7 @@ else
fi
echo "TEST_5 Test loading a config file by passing an env var."
TEST_5_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
TEST_5_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
TEST_5="$(GTS_CONFIG_PATH=./test/test.yaml go run ./cmd/gotosocial/... debug config)"
if [ "${TEST_5}" != "${TEST_5_EXPECTED}" ]; then
echo "TEST_5 not equal TEST_5_EXPECTED"
@ -55,7 +55,7 @@ else
fi
echo "TEST_6 Test loading a config file by passing cli flag."
TEST_6_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
TEST_6_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
TEST_6="$(go run ./cmd/gotosocial/... --config-path ./test/test.yaml debug config)"
if [ "${TEST_6}" != "${TEST_6_EXPECTED}" ]; then
echo "TEST_6 not equal TEST_6_EXPECTED"
@ -65,7 +65,7 @@ else
fi
echo "TEST_7 Test loading a config file and overriding one of the variables with a cli flag."
TEST_7_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
TEST_7_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
TEST_7="$(go run ./cmd/gotosocial/... --config-path ./test/test.yaml --account-domain '' debug config)"
if [ "${TEST_7}" != "${TEST_7_EXPECTED}" ]; then
echo "TEST_7 not equal TEST_7_EXPECTED"
@ -75,7 +75,7 @@ else
fi
echo "TEST_8 Test loading a config file and overriding one of the variables with an env var."
TEST_8_EXPECTED='{"account-domain":"peepee","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
TEST_8_EXPECTED='{"account-domain":"peepee","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
TEST_8="$(GTS_ACCOUNT_DOMAIN='peepee' go run ./cmd/gotosocial/... --config-path ./test/test.yaml debug config)"
if [ "${TEST_8}" != "${TEST_8_EXPECTED}" ]; then
echo "TEST_8 not equal TEST_8_EXPECTED"
@ -85,7 +85,7 @@ else
fi
echo "TEST_9 Test loading a config file and overriding one of the variables with both an env var and a cli flag. The cli flag should have priority."
TEST_9_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
TEST_9_EXPECTED='{"account-domain":"","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.yaml","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
TEST_9="$(GTS_ACCOUNT_DOMAIN='peepee' go run ./cmd/gotosocial/... --config-path ./test/test.yaml --account-domain '' debug config)"
if [ "${TEST_9}" != "${TEST_9_EXPECTED}" ]; then
echo "TEST_9 not equal TEST_9_EXPECTED"
@ -95,7 +95,7 @@ else
fi
echo "TEST_10 Test loading a config file from json."
TEST_10_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.json","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
TEST_10_EXPECTED='{"account-domain":"example.org","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test.json","db-address":"127.0.0.1","db-database":"postgres","db-password":"postgres","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"postgres","help":false,"host":"gts.example.org","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-level":"info","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","email","profile","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"someone@example.org","smtp-host":"verycoolemailhost.mail","smtp-password":"smtp-password","smtp-port":8888,"smtp-username":"smtp-username","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","0.0.0.0/0"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
TEST_10="$(go run ./cmd/gotosocial/... --config-path ./test/test.json debug config)"
if [ "${TEST_10}" != "${TEST_10_EXPECTED}" ]; then
echo "TEST_10 not equal TEST_10_EXPECTED"
@ -105,7 +105,7 @@ else
fi
echo "TEST_11 Test loading a partial config file. Default values should be used apart from those set in the config file."
TEST_11_EXPECTED='{"account-domain":"peepee.poopoo","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test2.yaml","db-address":"","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","help":false,"host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-level":"trace","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
TEST_11_EXPECTED='{"account-domain":"peepee.poopoo","accounts-approval-required":true,"accounts-reason-required":true,"accounts-registration-open":true,"application-name":"gotosocial","bind-address":"0.0.0.0","config-path":"./test/test2.yaml","db-address":"","db-database":"gotosocial","db-password":"","db-port":5432,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"postgres","db-user":"","help":false,"host":"","letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":false,"letsencrypt-port":80,"log-level":"trace","media-description-max-chars":500,"media-description-min-chars":0,"media-image-max-size":2097152,"media-remote-cache-days":30,"media-video-max-size":10485760,"oidc-client-id":"","oidc-client-secret":"","oidc-enabled":false,"oidc-idp-name":"","oidc-issuer":"","oidc-scopes":["openid","profile","email","groups"],"oidc-skip-verification":false,"port":8080,"protocol":"https","smtp-from":"GoToSocial","smtp-host":"","smtp-password":"","smtp-port":0,"smtp-username":"","software-version":"","statuses-cw-max-chars":100,"statuses-max-chars":5000,"statuses-media-max-files":6,"statuses-poll-max-options":6,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/gotosocial/storage","syslog-address":"localhost:514","syslog-enabled":false,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32"],"web-asset-base-dir":"./web/assets/","web-template-base-dir":"./web/template/"}'
TEST_11="$(go run ./cmd/gotosocial/... --config-path ./test/test2.yaml debug config)"
if [ "${TEST_11}" != "${TEST_11_EXPECTED}" ]; then
echo "TEST_11 not equal TEST_11_EXPECTED"

1
testrig/config.go
View file

@ -86,6 +86,7 @@ var TestDefaults = config.Values{
MediaVideoMaxSize: 5242880, // 5mb
MediaDescriptionMinChars: 0,
MediaDescriptionMaxChars: 500,
MediaRemoteCacheDays: 30,
StorageBackend: "local",
StorageLocalBasePath: "/gotosocial/storage",

BIN
testrig/media/thoughtsofdog-original.jpeg Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 19 KiB

BIN
testrig/media/thoughtsofdog-small.jpeg Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 20 KiB

115
testrig/testmodels.go
View file

@ -567,13 +567,13 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
Blurhash: "LNJRdVM{00Rj%Mayt7j[4nWBofRj",
Processing: 2,
File: gtsmodel.File{
Path: "/gotosocial/storage/01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg",
Path: "01F8MH17FWEB39HZJ76B6VXSKF/attachment/original/01F8MH6NEM8D7527KZAECTCR76.jpeg",
ContentType: "image/jpeg",
FileSize: 62529,
UpdatedAt: time.Now().Add(-71 * time.Hour),
},
Thumbnail: gtsmodel.Thumbnail{
Path: "/gotosocial/storage/01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpeg",
Path: "01F8MH17FWEB39HZJ76B6VXSKF/attachment/small/01F8MH6NEM8D7527KZAECTCR76.jpeg",
ContentType: "image/jpeg",
FileSize: 6872,
UpdatedAt: time.Now().Add(-71 * time.Hour),
@ -582,6 +582,7 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
},
Avatar: false,
Header: false,
Cached: true,
},
"local_account_1_status_4_attachment_1": {
ID: "01F8MH7TDVANYKWVE8VVKFPJTJ",
@ -615,13 +616,13 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
Blurhash: "LEDara58O=t5EMSOENEN9]}?aK%0",
Processing: 2,
File: gtsmodel.File{
Path: "/gotosocial/storage/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH7TDVANYKWVE8VVKFPJTJ.gif",
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH7TDVANYKWVE8VVKFPJTJ.gif",
ContentType: "image/gif",
FileSize: 1109138,
UpdatedAt: time.Now().Add(-1 * time.Hour),
},
Thumbnail: gtsmodel.Thumbnail{
Path: "/gotosocial/storage/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH7TDVANYKWVE8VVKFPJTJ.jpeg",
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH7TDVANYKWVE8VVKFPJTJ.jpeg",
ContentType: "image/jpeg",
FileSize: 8803,
UpdatedAt: time.Now().Add(-1 * time.Hour),
@ -630,6 +631,7 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
},
Avatar: false,
Header: false,
Cached: true,
},
"local_account_1_unattached_1": {
ID: "01F8MH8RMYQ6MSNY3JM2XT1CQ5",
@ -663,13 +665,13 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
Blurhash: "LSAd]9ogDge-R:M|j=xWIto0xXWX",
Processing: 2,
File: gtsmodel.File{
Path: "/gotosocial/storage/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpeg",
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpeg",
ContentType: "image/jpeg",
FileSize: 27759,
UpdatedAt: time.Now().Add(30 * time.Second),
},
Thumbnail: gtsmodel.Thumbnail{
Path: "/gotosocial/storage/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpeg",
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01F8MH8RMYQ6MSNY3JM2XT1CQ5.jpeg",
ContentType: "image/jpeg",
FileSize: 6177,
UpdatedAt: time.Now().Add(30 * time.Second),
@ -678,6 +680,7 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
},
Avatar: false,
Header: false,
Cached: true,
},
"local_account_1_avatar": {
ID: "01F8MH58A357CV5K7R7TJMSH6S",
@ -711,13 +714,13 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
Blurhash: "LKK9MT,p|YSNDkJ-5rsmvnwcOoe:",
Processing: 2,
File: gtsmodel.File{
Path: "/gotosocial/storage/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg",
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/avatar/original/01F8MH58A357CV5K7R7TJMSH6S.jpeg",
ContentType: "image/jpeg",
FileSize: 457680,
UpdatedAt: time.Now().Add(-47 * time.Hour),
},
Thumbnail: gtsmodel.Thumbnail{
Path: "/gotosocial/storage/01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg",
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/avatar/small/01F8MH58A357CV5K7R7TJMSH6S.jpeg",
ContentType: "image/jpeg",
FileSize: 15374,
UpdatedAt: time.Now().Add(-47 * time.Hour),
@ -726,6 +729,7 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
},
Avatar: true,
Header: false,
Cached: true,
},
"local_account_1_header": {
ID: "01PFPMWK2FF0D9WMHEJHR07C3Q",
@ -759,13 +763,13 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
Blurhash: "L26j{^WCs+R-N}jsxWj@4;WWxDoK",
Processing: 2,
File: gtsmodel.File{
Path: "/gotosocial/storage/01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg",
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/header/original/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg",
ContentType: "image/jpeg",
FileSize: 517226,
UpdatedAt: time.Now().Add(-47 * time.Hour),
},
Thumbnail: gtsmodel.Thumbnail{
Path: "/gotosocial/storage/01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg",
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/header/small/01PFPMWK2FF0D9WMHEJHR07C3Q.jpeg",
ContentType: "image/jpeg",
FileSize: 42308,
UpdatedAt: time.Now().Add(-47 * time.Hour),
@ -774,6 +778,56 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
},
Avatar: false,
Header: true,
Cached: true,
},
"remote_account_1_status_1_attachment_1": {
ID: "01FVW7RXPQ8YJHTEXYPE7Q8ZY0",
StatusID: "01FVW7JHQFSFK166WWKR8CBA6M",
URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
RemoteURL: "http://fossbros-anonymous.io/attachments/original/13bbc3f8-2b5e-46ea-9531-40b4974d9912.jpeg",
CreatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"),
UpdatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"),
Type: gtsmodel.FileTypeImage,
FileMeta: gtsmodel.FileMeta{
Original: gtsmodel.Original{
Width: 472,
Height: 291,
Size: 137352,
Aspect: 1.6219931271477663,
},
Small: gtsmodel.Small{
Width: 472,
Height: 291,
Size: 137352,
Aspect: 1.6219931271477663,
},
Focus: gtsmodel.Focus{
X: 0,
Y: 0,
},
},
AccountID: "01F8MH1H7YV1Z7D2C8K2730QBF",
Description: "tweet from thoughts of dog: i drank. all the water. in my bowl. earlier. but just now. i returned. to the same bowl. and it was. full again.. the bowl. is haunted",
ScheduledStatusID: "",
Blurhash: "LARysgM_IU_3~pD%M_Rj_39FIAt6",
Processing: 2,
File: gtsmodel.File{
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/original/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
ContentType: "image/jpeg",
FileSize: 19310,
UpdatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"),
},
Thumbnail: gtsmodel.Thumbnail{
Path: "01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
ContentType: "image/jpeg",
FileSize: 20395,
UpdatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"),
URL: "http://localhost:8080/fileserver/01F8MH1H7YV1Z7D2C8K2730QBF/attachment/small/01FVW7RXPQ8YJHTEXYPE7Q8ZY0.jpeg",
RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpeg",
},
Avatar: false,
Header: false,
Cached: true,
},
}
}
@ -848,6 +902,10 @@ func newTestStoredAttachments() map[string]filenames {
Original: "team-fortress-original.jpeg",
Small: "team-fortress-small.jpeg",
},
"remote_account_1_status_1_attachment_1": {
Original: "thoughtsofdog-original.jpeg",
Small: "thoughtsofdog-small.jpeg",
},
}
}
@ -1202,6 +1260,33 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
Likeable: true,
ActivityStreamsType: ap.ObjectNote,
},
"remote_account_1_status_1": {
ID: "01FVW7JHQFSFK166WWKR8CBA6M",
URI: "http://fossbros-anonymous.io/users/foss_satan/statuses/01FVW7JHQFSFK166WWKR8CBA6M",
URL: "http://fossbros-anonymous.io/@foss_satan/statuses/01FVW7JHQFSFK166WWKR8CBA6M",
Content: "dark souls status bot: \"thoughts of dog\"",
AttachmentIDs: []string{"01FVW7RXPQ8YJHTEXYPE7Q8ZY0"},
CreatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"),
UpdatedAt: TimeMustParse("2021-09-20T12:40:37+02:00"),
Local: false,
AccountURI: "http://fossbros-anonymous.io/users/foss_satan",
MentionIDs: []string{},
AccountID: "01F8MH5ZK5VRH73AKHQM6Y9VNX",
InReplyToID: "",
InReplyToAccountID: "",
InReplyToURI: "",
BoostOfID: "",
ContentWarning: "",
Visibility: gtsmodel.VisibilityUnlocked,
Sensitive: false,
Language: "en",
CreatedWithApplicationID: "",
Federated: true,
Boostable: true,
Replyable: true,
Likeable: true,
ActivityStreamsType: ap.ObjectNote,
},
}
}
@ -1501,11 +1586,21 @@ func NewTestFediAttachments(relativePath string) map[string]RemoteAttachmentFile
if err != nil {
panic(err)
}
thoughtsOfDogBytes, err := os.ReadFile(fmt.Sprintf("%s/thoughtsofdog-original.jpeg", relativePath))
if err != nil {
panic(err)
}
return map[string]RemoteAttachmentFile{
"https://s3-us-west-2.amazonaws.com/plushcity/media_attachments/files/106/867/380/219/163/828/original/88e8758c5f011439.jpg": {
Data: beeBytes,
ContentType: "image/jpeg",
},
"http://fossbros-anonymous.io/attachments/original/13bbc3f8-2b5e-46ea-9531-40b4974d9912.jpeg": {
Data: thoughtsOfDogBytes,
ContentType: "image/jpeg",
},
}
}

7
vendor/github.com/jackc/pgconn/CHANGELOG.md generated vendored
View file

@ -1,3 +1,10 @@
# 1.11.0 (February 7, 2022)
* Support port in ip from LookupFunc to override config (James Hartig)
* Fix TLS connection timeout (Blake Embrey)
* Add support for read-only, primary, standby, prefer-standby target_session_attributes (Oscar)
* Fix connect when receiving NoticeResponse
# 1.10.1 (November 20, 2021)
* Close without waiting for response (Kei Kamikawa)

90
vendor/github.com/jackc/pgconn/config.go generated vendored
View file

@ -248,21 +248,21 @@ func ParseConfig(connString string) (*Config, error) {
config.LookupFunc = makeDefaultResolver().LookupHost
notRuntimeParams := map[string]struct{}{
"host": struct{}{},
"port": struct{}{},
"database": struct{}{},
"user": struct{}{},
"password": struct{}{},
"passfile": struct{}{},
"connect_timeout": struct{}{},
"sslmode": struct{}{},
"sslkey": struct{}{},
"sslcert": struct{}{},
"sslrootcert": struct{}{},
"target_session_attrs": struct{}{},
"min_read_buffer_size": struct{}{},
"service": struct{}{},
"servicefile": struct{}{},
"host": {},
"port": {},
"database": {},
"user": {},
"password": {},
"passfile": {},
"connect_timeout": {},
"sslmode": {},
"sslkey": {},
"sslcert": {},
"sslrootcert": {},
"target_session_attrs": {},
"min_read_buffer_size": {},
"service": {},
"servicefile": {},
}
for k, v := range settings {
@ -329,10 +329,19 @@ func ParseConfig(connString string) (*Config, error) {
}
}
if settings["target_session_attrs"] == "read-write" {
switch tsa := settings["target_session_attrs"]; tsa {
case "read-write":
config.ValidateConnect = ValidateConnectTargetSessionAttrsReadWrite
} else if settings["target_session_attrs"] != "any" {
return nil, &parseConfigError{connString: connString, msg: fmt.Sprintf("unknown target_session_attrs value: %v", settings["target_session_attrs"])}
case "read-only":
config.ValidateConnect = ValidateConnectTargetSessionAttrsReadOnly
case "primary":
config.ValidateConnect = ValidateConnectTargetSessionAttrsPrimary
case "standby":
config.ValidateConnect = ValidateConnectTargetSessionAttrsStandby
case "any", "prefer-standby":
// do nothing
default:
return nil, &parseConfigError{connString: connString, msg: fmt.Sprintf("unknown target_session_attrs value: %v", tsa)}
}
return config, nil
@ -727,3 +736,48 @@ func ValidateConnectTargetSessionAttrsReadWrite(ctx context.Context, pgConn *PgC
return nil
}
// ValidateConnectTargetSessionAttrsReadOnly is an ValidateConnectFunc that implements libpq compatible
// target_session_attrs=read-only.
func ValidateConnectTargetSessionAttrsReadOnly(ctx context.Context, pgConn *PgConn) error {
result := pgConn.ExecParams(ctx, "show transaction_read_only", nil, nil, nil, nil).Read()
if result.Err != nil {
return result.Err
}
if string(result.Rows[0][0]) != "on" {
return errors.New("connection is not read only")
}
return nil
}
// ValidateConnectTargetSessionAttrsStandby is an ValidateConnectFunc that implements libpq compatible
// target_session_attrs=standby.
func ValidateConnectTargetSessionAttrsStandby(ctx context.Context, pgConn *PgConn) error {
result := pgConn.ExecParams(ctx, "select pg_is_in_recovery()", nil, nil, nil, nil).Read()
if result.Err != nil {
return result.Err
}
if string(result.Rows[0][0]) != "t" {
return errors.New("server is not in hot standby mode")
}
return nil
}
// ValidateConnectTargetSessionAttrsPrimary is an ValidateConnectFunc that implements libpq compatible
// target_session_attrs=primary.
func ValidateConnectTargetSessionAttrsPrimary(ctx context.Context, pgConn *PgConn) error {
result := pgConn.ExecParams(ctx, "select pg_is_in_recovery()", nil, nil, nil, nil).Read()
if result.Err != nil {
return result.Err
}
if string(result.Rows[0][0]) == "t" {
return errors.New("server is in standby mode")
}
return nil
}

82
vendor/github.com/jackc/pgconn/pgconn.go generated vendored
View file

@ -11,6 +11,7 @@ import (
"io"
"math"
"net"
"strconv"
"strings"
"sync"
"time"
@ -44,7 +45,8 @@ type Notification struct {
// DialFunc is a function that can be used to connect to a PostgreSQL server.
type DialFunc func(ctx context.Context, network, addr string) (net.Conn, error)
// LookupFunc is a function that can be used to lookup IPs addrs from host.
// LookupFunc is a function that can be used to lookup IPs addrs from host. Optionally an ip:port combination can be
// returned in order to override the connection string's port.
type LookupFunc func(ctx context.Context, host string) (addrs []string, err error)
// BuildFrontendFunc is a function that can be used to create Frontend implementation for connection.
@ -196,11 +198,24 @@ func expandWithIPs(ctx context.Context, lookupFn LookupFunc, fallbacks []*Fallba
}
for _, ip := range ips {
configs = append(configs, &FallbackConfig{
Host: ip,
Port: fb.Port,
TLSConfig: fb.TLSConfig,
})
splitIP, splitPort, err := net.SplitHostPort(ip)
if err == nil {
port, err := strconv.ParseUint(splitPort, 10, 16)
if err != nil {
return nil, fmt.Errorf("error parsing port (%s) from lookup: %w", splitPort, err)
}
configs = append(configs, &FallbackConfig{
Host: splitIP,
Port: uint16(port),
TLSConfig: fb.TLSConfig,
})
} else {
configs = append(configs, &FallbackConfig{
Host: ip,
Port: fb.Port,
TLSConfig: fb.TLSConfig,
})
}
}
}
@ -215,7 +230,7 @@ func connect(ctx context.Context, config *Config, fallbackConfig *FallbackConfig
var err error
network, address := NetworkAddress(fallbackConfig.Host, fallbackConfig.Port)
pgConn.conn, err = config.DialFunc(ctx, network, address)
netConn, err := config.DialFunc(ctx, network, address)
if err != nil {
var netErr net.Error
if errors.As(err, &netErr) && netErr.Timeout() {
@ -224,24 +239,27 @@ func connect(ctx context.Context, config *Config, fallbackConfig *FallbackConfig
return nil, &connectError{config: config, msg: "dial error", err: err}
}
pgConn.parameterStatuses = make(map[string]string)
pgConn.conn = netConn
pgConn.contextWatcher = newContextWatcher(netConn)
pgConn.contextWatcher.Watch(ctx)
if fallbackConfig.TLSConfig != nil {
if err := pgConn.startTLS(fallbackConfig.TLSConfig); err != nil {
pgConn.conn.Close()
tlsConn, err := startTLS(netConn, fallbackConfig.TLSConfig)
pgConn.contextWatcher.Unwatch() // Always unwatch `netConn` after TLS.
if err != nil {
netConn.Close()
return nil, &connectError{config: config, msg: "tls error", err: err}
}
pgConn.conn = tlsConn
pgConn.contextWatcher = newContextWatcher(tlsConn)
pgConn.contextWatcher.Watch(ctx)
}
pgConn.status = connStatusConnecting
pgConn.contextWatcher = ctxwatch.NewContextWatcher(
func() { pgConn.conn.SetDeadline(time.Date(1, 1, 1, 1, 1, 1, 1, time.UTC)) },
func() { pgConn.conn.SetDeadline(time.Time{}) },
)
pgConn.contextWatcher.Watch(ctx)
defer pgConn.contextWatcher.Unwatch()
pgConn.parameterStatuses = make(map[string]string)
pgConn.status = connStatusConnecting
pgConn.frontend = config.BuildFrontend(pgConn.conn, pgConn.conn)
startupMsg := pgproto3.StartupMessage{
@ -317,7 +335,7 @@ func connect(ctx context.Context, config *Config, fallbackConfig *FallbackConfig
}
}
return pgConn, nil
case *pgproto3.ParameterStatus:
case *pgproto3.ParameterStatus, *pgproto3.NoticeResponse:
// handled by ReceiveMessage
case *pgproto3.ErrorResponse:
pgConn.conn.Close()
@ -329,24 +347,29 @@ func connect(ctx context.Context, config *Config, fallbackConfig *FallbackConfig
}
}
func (pgConn *PgConn) startTLS(tlsConfig *tls.Config) (err error) {
err = binary.Write(pgConn.conn, binary.BigEndian, []int32{8, 80877103})
func newContextWatcher(conn net.Conn) *ctxwatch.ContextWatcher {
return ctxwatch.NewContextWatcher(
func() { conn.SetDeadline(time.Date(1, 1, 1, 1, 1, 1, 1, time.UTC)) },
func() { conn.SetDeadline(time.Time{}) },
)
}
func startTLS(conn net.Conn, tlsConfig *tls.Config) (net.Conn, error) {
err := binary.Write(conn, binary.BigEndian, []int32{8, 80877103})
if err != nil {
return
return nil, err
}
response := make([]byte, 1)
if _, err = io.ReadFull(pgConn.conn, response); err != nil {
return
if _, err = io.ReadFull(conn, response); err != nil {
return nil, err
}
if response[0] != 'S' {
return errors.New("server refused TLS connection")
return nil, errors.New("server refused TLS connection")
}
pgConn.conn = tls.Client(pgConn.conn, tlsConfig)
return nil
return tls.Client(conn, tlsConfig), nil
}
func (pgConn *PgConn) txPasswordMessage(password string) (err error) {
@ -1694,10 +1717,7 @@ func Construct(hc *HijackedConn) (*PgConn, error) {
cleanupDone: make(chan struct{}),
}
pgConn.contextWatcher = ctxwatch.NewContextWatcher(
func() { pgConn.conn.SetDeadline(time.Date(1, 1, 1, 1, 1, 1, 1, time.UTC)) },
func() { pgConn.conn.SetDeadline(time.Time{}) },
)
pgConn.contextWatcher = newContextWatcher(pgConn.conn)
return pgConn, nil
}

7
vendor/github.com/jackc/pgtype/CHANGELOG.md generated vendored
View file

@ -1,3 +1,10 @@
# 1.10.0 (February 7, 2022)
* Normalize UTC timestamps to comply with stdlib (Torkel Rogstad)
* Assign Numeric to *big.Rat (Oleg Lomaka)
* Fix typo in float8 error message (Pinank Solanki)
* Scan type aliases for floating point types (Collin Forsyth)
# 1.9.1 (November 28, 2021)
* Fix: binary timestamp is assumed to be in UTC (restored behavior changed in v1.9.0)

4
vendor/github.com/jackc/pgtype/convert.go generated vendored
View file

@ -337,6 +337,10 @@ func float64AssignTo(srcVal float64, srcStatus Status, dst interface{}) error {
if v := reflect.ValueOf(dst); v.Kind() == reflect.Ptr {
el := v.Elem()
switch el.Kind() {
// if dst is a type alias of a float32 or 64, set dst val
case reflect.Float32, reflect.Float64:
el.SetFloat(srcVal)
return nil
// if dst is a pointer to pointer, strip the pointer and try again
case reflect.Ptr:
if el.IsNil() {

2
vendor/github.com/jackc/pgtype/float8.go generated vendored
View file

@ -204,7 +204,7 @@ func (dst *Float8) DecodeBinary(ci *ConnInfo, src []byte) error {
}
if len(src) != 8 {
return fmt.Errorf("invalid length for float4: %v", len(src))
return fmt.Errorf("invalid length for float8: %v", len(src))
}
n := int64(binary.BigEndian.Uint64(src))

26
vendor/github.com/jackc/pgtype/numeric.go generated vendored
View file

@ -369,6 +369,12 @@ func (src *Numeric) AssignTo(dst interface{}) error {
return fmt.Errorf("%d is greater than maximum value for %T", normalizedInt, *v)
}
*v = normalizedInt.Uint64()
case *big.Rat:
rat, err := src.toBigRat()
if err != nil {
return err
}
v.Set(rat)
default:
if nextDst, retry := GetAssignToDstType(dst); retry {
return src.AssignTo(nextDst)
@ -406,6 +412,26 @@ func (dst *Numeric) toBigInt() (*big.Int, error) {
return num, nil
}
func (dst *Numeric) toBigRat() (*big.Rat, error) {
if dst.NaN {
return nil, fmt.Errorf("%v is not a number", dst)
} else if dst.InfinityModifier == Infinity {
return nil, fmt.Errorf("%v is infinity", dst)
} else if dst.InfinityModifier == NegativeInfinity {
return nil, fmt.Errorf("%v is -infinity", dst)
}
num := new(big.Rat).SetInt(dst.Int)
if dst.Exp > 0 {
mul := new(big.Int).Exp(big10, big.NewInt(int64(dst.Exp)), nil)
num.Mul(num, new(big.Rat).SetInt(mul))
} else if dst.Exp < 0 {
mul := new(big.Int).Exp(big10, big.NewInt(int64(-dst.Exp)), nil)
num.Quo(num, new(big.Rat).SetInt(mul))
}
return num, nil
}
func (src *Numeric) toFloat64() (float64, error) {
if src.NaN {
return math.NaN(), nil

22
vendor/github.com/jackc/pgtype/timestamptz.go generated vendored
View file

@ -124,7 +124,7 @@ func (dst *Timestamptz) DecodeText(ci *ConnInfo, src []byte) error {
return err
}
*dst = Timestamptz{Time: tim, Status: Present}
*dst = Timestamptz{Time: normalizePotentialUTC(tim), Status: Present}
}
return nil
@ -231,6 +231,9 @@ func (src Timestamptz) Value() (driver.Value, error) {
if src.InfinityModifier != None {
return src.InfinityModifier.String(), nil
}
if src.Time.Location().String() == time.UTC.String() {
return src.Time.UTC(), nil
}
return src.Time, nil
case Null:
return nil, nil
@ -289,8 +292,23 @@ func (dst *Timestamptz) UnmarshalJSON(b []byte) error {
return err
}
*dst = Timestamptz{Time: tim, Status: Present}
*dst = Timestamptz{Time: normalizePotentialUTC(tim), Status: Present}
}
return nil
}
// Normalize timestamps in UTC location to behave similarly to how the Golang
// standard library does it: UTC timestamps lack a .loc value.
//
// Reason for this: when comparing two timestamps with reflect.DeepEqual (generally
// speaking not a good idea, but several testing libraries (for example testify)
// does this), their location data needs to be equal for them to be considered
// equal.
func normalizePotentialUTC(timestamp time.Time) time.Time {
if timestamp.Location().String() != time.UTC.String() {
return timestamp
}
return timestamp.UTC()
}

7
vendor/github.com/jackc/pgx/v4/CHANGELOG.md generated vendored
View file

@ -1,3 +1,10 @@
# 4.15.0 (February 7, 2022)
* Upgrade to pgconn v1.11.0
* Upgrade to pgtype v1.10.0
* Upgrade puddle to v1.2.1
* Make BatchResults.Close safe to be called multiple times
# 4.14.1 (November 28, 2021)
* Upgrade pgtype to v1.9.1 (fixes unintentional change to timestamp binary decoding)

33
vendor/github.com/jackc/pgx/v4/batch.go generated vendored
View file

@ -3,6 +3,7 @@ package pgx
import (
"context"
"errors"
"fmt"
"github.com/jackc/pgconn"
)
@ -46,17 +47,18 @@ type BatchResults interface {
// Close closes the batch operation. This must be called before the underlying connection can be used again. Any error
// that occurred during a batch operation may have made it impossible to resyncronize the connection with the server.
// In this case the underlying connection will have been closed.
// In this case the underlying connection will have been closed. Close is safe to call multiple times.
Close() error
}
type batchResults struct {
ctx context.Context
conn *Conn
mrr *pgconn.MultiResultReader
err error
b *Batch
ix int
ctx context.Context
conn *Conn
mrr *pgconn.MultiResultReader
err error
b *Batch
ix int
closed bool
}
// Exec reads the results from the next query in the batch as if the query has been sent with Exec.
@ -64,6 +66,9 @@ func (br *batchResults) Exec() (pgconn.CommandTag, error) {
if br.err != nil {
return nil, br.err
}
if br.closed {
return nil, fmt.Errorf("batch already closed")
}
query, arguments, _ := br.nextQueryAndArgs()
@ -114,6 +119,11 @@ func (br *batchResults) Query() (Rows, error) {
return &connRows{err: br.err, closed: true}, br.err
}
if br.closed {
alreadyClosedErr := fmt.Errorf("batch already closed")
return &connRows{err: alreadyClosedErr, closed: true}, alreadyClosedErr
}
rows := br.conn.getRows(br.ctx, query, arguments)
if !br.mrr.NextResult() {
@ -140,6 +150,10 @@ func (br *batchResults) Query() (Rows, error) {
// QueryFunc reads the results from the next query in the batch as if the query has been sent with Conn.QueryFunc.
func (br *batchResults) QueryFunc(scans []interface{}, f func(QueryFuncRow) error) (pgconn.CommandTag, error) {
if br.closed {
return nil, fmt.Errorf("batch already closed")
}
rows, err := br.Query()
if err != nil {
return nil, err
@ -179,6 +193,11 @@ func (br *batchResults) Close() error {
return br.err
}
if br.closed {
return nil
}
br.closed = true
// log any queries that haven't yet been logged by Exec or Query
for {
query, args, ok := br.nextQueryAndArgs()

7
vendor/github.com/jackc/pgx/v4/rows.go generated vendored
View file

@ -41,10 +41,13 @@ type Rows interface {
// Scan reads the values from the current row into dest values positionally.
// dest can include pointers to core types, values implementing the Scanner
// interface, and nil. nil will skip the value entirely.
// interface, and nil. nil will skip the value entirely. It is an error to
// call Scan without first calling Next() and checking that it returned true.
Scan(dest ...interface{}) error
// Values returns the decoded row values.
// Values returns the decoded row values. As with Scan(), it is an error to
// call Values without first calling Next() and checking that it returned
// true.
Values() ([]interface{}, error)
// RawValues returns the unparsed bytes of the row values. The returned [][]byte is only valid until the next Next

22
vendor/github.com/robfig/cron/v3/.gitignore generated vendored Normal file
View file

@ -0,0 +1,22 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe

1
vendor/github.com/robfig/cron/v3/.travis.yml generated vendored Normal file
View file

@ -0,0 +1 @@
language: go

21
vendor/github.com/robfig/cron/v3/LICENSE generated vendored Normal file
View file

@ -0,0 +1,21 @@
Copyright (C) 2012 Rob Figueiredo
All Rights Reserved.
MIT LICENSE
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

125
vendor/github.com/robfig/cron/v3/README.md generated vendored Normal file
View file

@ -0,0 +1,125 @@
[![GoDoc](http://godoc.org/github.com/robfig/cron?status.png)](http://godoc.org/github.com/robfig/cron)
[![Build Status](https://travis-ci.org/robfig/cron.svg?branch=master)](https://travis-ci.org/robfig/cron)
# cron
Cron V3 has been released!
To download the specific tagged release, run:
go get github.com/robfig/cron/v3@v3.0.0
Import it in your program as:
import "github.com/robfig/cron/v3"
It requires Go 1.11 or later due to usage of Go Modules.
Refer to the documentation here:
http://godoc.org/github.com/robfig/cron
The rest of this document describes the the advances in v3 and a list of
breaking changes for users that wish to upgrade from an earlier version.
## Upgrading to v3 (June 2019)
cron v3 is a major upgrade to the library that addresses all outstanding bugs,
feature requests, and rough edges. It is based on a merge of master which
contains various fixes to issues found over the years and the v2 branch which
contains some backwards-incompatible features like the ability to remove cron
jobs. In addition, v3 adds support for Go Modules, cleans up rough edges like
the timezone support, and fixes a number of bugs.
New features:
- Support for Go modules. Callers must now import this library as
`github.com/robfig/cron/v3`, instead of `gopkg.in/...`
- Fixed bugs:
- 0f01e6b parser: fix combining of Dow and Dom (#70)
- dbf3220 adjust times when rolling the clock forward to handle non-existent midnight (#157)
- eeecf15 spec_test.go: ensure an error is returned on 0 increment (#144)
- 70971dc cron.Entries(): update request for snapshot to include a reply channel (#97)
- 1cba5e6 cron: fix: removing a job causes the next scheduled job to run too late (#206)
- Standard cron spec parsing by default (first field is "minute"), with an easy
way to opt into the seconds field (quartz-compatible). Although, note that the
year field (optional in Quartz) is not supported.
- Extensible, key/value logging via an interface that complies with
the https://github.com/go-logr/logr project.
- The new Chain & JobWrapper types allow you to install "interceptors" to add
cross-cutting behavior like the following:
- Recover any panics from jobs
- Delay a job's execution if the previous run hasn't completed yet
- Skip a job's execution if the previous run hasn't completed yet
- Log each job's invocations
- Notification when jobs are completed
It is backwards incompatible with both v1 and v2. These updates are required:
- The v1 branch accepted an optional seconds field at the beginning of the cron
spec. This is non-standard and has led to a lot of confusion. The new default
parser conforms to the standard as described by [the Cron wikipedia page].
UPDATING: To retain the old behavior, construct your Cron with a custom
parser:
// Seconds field, required
cron.New(cron.WithSeconds())
// Seconds field, optional
cron.New(
cron.WithParser(
cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor))
- The Cron type now accepts functional options on construction rather than the
previous ad-hoc behavior modification mechanisms (setting a field, calling a setter).
UPDATING: Code that sets Cron.ErrorLogger or calls Cron.SetLocation must be
updated to provide those values on construction.
- CRON_TZ is now the recommended way to specify the timezone of a single
schedule, which is sanctioned by the specification. The legacy "TZ=" prefix
will continue to be supported since it is unambiguous and easy to do so.
UPDATING: No update is required.
- By default, cron will no longer recover panics in jobs that it runs.
Recovering can be surprising (see issue #192) and seems to be at odds with
typical behavior of libraries. Relatedly, the `cron.WithPanicLogger` option
has been removed to accommodate the more general JobWrapper type.
UPDATING: To opt into panic recovery and configure the panic logger:
cron.New(cron.WithChain(
cron.Recover(logger), // or use cron.DefaultLogger
))
- In adding support for https://github.com/go-logr/logr, `cron.WithVerboseLogger` was
removed, since it is duplicative with the leveled logging.
UPDATING: Callers should use `WithLogger` and specify a logger that does not
discard `Info` logs. For convenience, one is provided that wraps `*log.Logger`:
cron.New(
cron.WithLogger(cron.VerbosePrintfLogger(logger)))
### Background - Cron spec format
There are two cron spec formats in common usage:
- The "standard" cron format, described on [the Cron wikipedia page] and used by
the cron Linux system utility.
- The cron format used by [the Quartz Scheduler], commonly used for scheduled
jobs in Java software
[the Cron wikipedia page]: https://en.wikipedia.org/wiki/Cron
[the Quartz Scheduler]: http://www.quartz-scheduler.org/documentation/quartz-2.3.0/tutorials/tutorial-lesson-06.html
The original version of this package included an optional "seconds" field, which
made it incompatible with both of these formats. Now, the "standard" format is
the default format accepted, and the Quartz format is opt-in.

92
vendor/github.com/robfig/cron/v3/chain.go generated vendored Normal file
View file

@ -0,0 +1,92 @@
package cron
import (
"fmt"
"runtime"
"sync"
"time"
)
// JobWrapper decorates the given Job with some behavior.
type JobWrapper func(Job) Job
// Chain is a sequence of JobWrappers that decorates submitted jobs with
// cross-cutting behaviors like logging or synchronization.
type Chain struct {
wrappers []JobWrapper
}
// NewChain returns a Chain consisting of the given JobWrappers.
func NewChain(c ...JobWrapper) Chain {
return Chain{c}
}
// Then decorates the given job with all JobWrappers in the chain.
//
// This:
// NewChain(m1, m2, m3).Then(job)
// is equivalent to:
// m1(m2(m3(job)))
func (c Chain) Then(j Job) Job {
for i := range c.wrappers {
j = c.wrappers[len(c.wrappers)-i-1](j)
}
return j
}
// Recover panics in wrapped jobs and log them with the provided logger.
func Recover(logger Logger) JobWrapper {
return func(j Job) Job {
return FuncJob(func() {
defer func() {
if r := recover(); r != nil {
const size = 64 << 10
buf := make([]byte, size)
buf = buf[:runtime.Stack(buf, false)]
err, ok := r.(error)
if !ok {
err = fmt.Errorf("%v", r)
}
logger.Error(err, "panic", "stack", "...\n"+string(buf))
}
}()
j.Run()
})
}
}
// DelayIfStillRunning serializes jobs, delaying subsequent runs until the
// previous one is complete. Jobs running after a delay of more than a minute
// have the delay logged at Info.
func DelayIfStillRunning(logger Logger) JobWrapper {
return func(j Job) Job {
var mu sync.Mutex
return FuncJob(func() {
start := time.Now()
mu.Lock()
defer mu.Unlock()
if dur := time.Since(start); dur > time.Minute {
logger.Info("delay", "duration", dur)
}
j.Run()
})
}
}
// SkipIfStillRunning skips an invocation of the Job if a previous invocation is
// still running. It logs skips to the given logger at Info level.
func SkipIfStillRunning(logger Logger) JobWrapper {
return func(j Job) Job {
var ch = make(chan struct{}, 1)
ch <- struct{}{}
return FuncJob(func() {
select {
case v := <-ch:
j.Run()
ch <- v
default:
logger.Info("skip")
}
})
}
}

27
vendor/github.com/robfig/cron/v3/constantdelay.go generated vendored Normal file
View file

@ -0,0 +1,27 @@
package cron
import "time"
// ConstantDelaySchedule represents a simple recurring duty cycle, e.g. "Every 5 minutes".
// It does not support jobs more frequent than once a second.
type ConstantDelaySchedule struct {
Delay time.Duration
}
// Every returns a crontab Schedule that activates once every duration.
// Delays of less than a second are not supported (will round up to 1 second).
// Any fields less than a Second are truncated.
func Every(duration time.Duration) ConstantDelaySchedule {
if duration < time.Second {
duration = time.Second
}
return ConstantDelaySchedule{
Delay: duration - time.Duration(duration.Nanoseconds())%time.Second,
}
}
// Next returns the next time this should be run.
// This rounds so that the next activation time will be on the second.
func (schedule ConstantDelaySchedule) Next(t time.Time) time.Time {
return t.Add(schedule.Delay - time.Duration(t.Nanosecond())*time.Nanosecond)
}

355
vendor/github.com/robfig/cron/v3/cron.go generated vendored Normal file
View file

@ -0,0 +1,355 @@
package cron
import (
"context"
"sort"
"sync"
"time"
)
// Cron keeps track of any number of entries, invoking the associated func as
// specified by the schedule. It may be started, stopped, and the entries may
// be inspected while running.
type Cron struct {
entries []*Entry
chain Chain
stop chan struct{}
add chan *Entry
remove chan EntryID
snapshot chan chan []Entry
running bool
logger Logger
runningMu sync.Mutex
location *time.Location
parser ScheduleParser
nextID EntryID
jobWaiter sync.WaitGroup
}
// ScheduleParser is an interface for schedule spec parsers that return a Schedule
type ScheduleParser interface {
Parse(spec string) (Schedule, error)
}
// Job is an interface for submitted cron jobs.
type Job interface {
Run()
}
// Schedule describes a job's duty cycle.
type Schedule interface {
// Next returns the next activation time, later than the given time.
// Next is invoked initially, and then each time the job is run.
Next(time.Time) time.Time
}
// EntryID identifies an entry within a Cron instance
type EntryID int
// Entry consists of a schedule and the func to execute on that schedule.
type Entry struct {
// ID is the cron-assigned ID of this entry, which may be used to look up a
// snapshot or remove it.
ID EntryID
// Schedule on which this job should be run.
Schedule Schedule
// Next time the job will run, or the zero time if Cron has not been
// started or this entry's schedule is unsatisfiable
Next time.Time
// Prev is the last time this job was run, or the zero time if never.
Prev time.Time
// WrappedJob is the thing to run when the Schedule is activated.
WrappedJob Job
// Job is the thing that was submitted to cron.
// It is kept around so that user code that needs to get at the job later,
// e.g. via Entries() can do so.
Job Job
}
// Valid returns true if this is not the zero entry.
func (e Entry) Valid() bool { return e.ID != 0 }
// byTime is a wrapper for sorting the entry array by time
// (with zero time at the end).
type byTime []*Entry
func (s byTime) Len() int { return len(s) }
func (s byTime) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
func (s byTime) Less(i, j int) bool {
// Two zero times should return false.
// Otherwise, zero is "greater" than any other time.
// (To sort it at the end of the list.)
if s[i].Next.IsZero() {
return false
}
if s[j].Next.IsZero() {
return true
}
return s[i].Next.Before(s[j].Next)
}
// New returns a new Cron job runner, modified by the given options.
//
// Available Settings
//
// Time Zone
// Description: The time zone in which schedules are interpreted
// Default: time.Local
//
// Parser
// Description: Parser converts cron spec strings into cron.Schedules.
// Default: Accepts this spec: https://en.wikipedia.org/wiki/Cron
//
// Chain
// Description: Wrap submitted jobs to customize behavior.
// Default: A chain that recovers panics and logs them to stderr.
//
// See "cron.With*" to modify the default behavior.
func New(opts ...Option) *Cron {
c := &Cron{
entries: nil,
chain: NewChain(),
add: make(chan *Entry),
stop: make(chan struct{}),
snapshot: make(chan chan []Entry),
remove: make(chan EntryID),
running: false,
runningMu: sync.Mutex{},
logger: DefaultLogger,
location: time.Local,
parser: standardParser,
}
for _, opt := range opts {
opt(c)
}
return c
}
// FuncJob is a wrapper that turns a func() into a cron.Job
type FuncJob func()
func (f FuncJob) Run() { f() }
// AddFunc adds a func to the Cron to be run on the given schedule.
// The spec is parsed using the time zone of this Cron instance as the default.
// An opaque ID is returned that can be used to later remove it.
func (c *Cron) AddFunc(spec string, cmd func()) (EntryID, error) {
return c.AddJob(spec, FuncJob(cmd))
}
// AddJob adds a Job to the Cron to be run on the given schedule.
// The spec is parsed using the time zone of this Cron instance as the default.
// An opaque ID is returned that can be used to later remove it.
func (c *Cron) AddJob(spec string, cmd Job) (EntryID, error) {
schedule, err := c.parser.Parse(spec)
if err != nil {
return 0, err
}
return c.Schedule(schedule, cmd), nil
}
// Schedule adds a Job to the Cron to be run on the given schedule.
// The job is wrapped with the configured Chain.
func (c *Cron) Schedule(schedule Schedule, cmd Job) EntryID {
c.runningMu.Lock()
defer c.runningMu.Unlock()
c.nextID++
entry := &Entry{
ID: c.nextID,
Schedule: schedule,
WrappedJob: c.chain.Then(cmd),
Job: cmd,
}
if !c.running {
c.entries = append(c.entries, entry)
} else {
c.add <- entry
}
return entry.ID
}
// Entries returns a snapshot of the cron entries.
func (c *Cron) Entries() []Entry {
c.runningMu.Lock()
defer c.runningMu.Unlock()
if c.running {
replyChan := make(chan []Entry, 1)
c.snapshot <- replyChan
return <-replyChan
}
return c.entrySnapshot()
}
// Location gets the time zone location
func (c *Cron) Location() *time.Location {
return c.location
}
// Entry returns a snapshot of the given entry, or nil if it couldn't be found.
func (c *Cron) Entry(id EntryID) Entry {
for _, entry := range c.Entries() {
if id == entry.ID {
return entry
}
}
return Entry{}
}
// Remove an entry from being run in the future.
func (c *Cron) Remove(id EntryID) {
c.runningMu.Lock()
defer c.runningMu.Unlock()
if c.running {
c.remove <- id
} else {
c.removeEntry(id)
}
}
// Start the cron scheduler in its own goroutine, or no-op if already started.
func (c *Cron) Start() {
c.runningMu.Lock()
defer c.runningMu.Unlock()
if c.running {
return
}
c.running = true
go c.run()
}
// Run the cron scheduler, or no-op if already running.
func (c *Cron) Run() {
c.runningMu.Lock()
if c.running {
c.runningMu.Unlock()
return
}
c.running = true
c.runningMu.Unlock()
c.run()
}
// run the scheduler.. this is private just due to the need to synchronize
// access to the 'running' state variable.
func (c *Cron) run() {
c.logger.Info("start")
// Figure out the next activation times for each entry.
now := c.now()
for _, entry := range c.entries {
entry.Next = entry.Schedule.Next(now)
c.logger.Info("schedule", "now", now, "entry", entry.ID, "next", entry.Next)
}
for {
// Determine the next entry to run.
sort.Sort(byTime(c.entries))
var timer *time.Timer
if len(c.entries) == 0 || c.entries[0].Next.IsZero() {
// If there are no entries yet, just sleep - it still handles new entries
// and stop requests.
timer = time.NewTimer(100000 * time.Hour)
} else {
timer = time.NewTimer(c.entries[0].Next.Sub(now))
}
for {
select {
case now = <-timer.C:
now = now.In(c.location)
c.logger.Info("wake", "now", now)
// Run every entry whose next time was less than now
for _, e := range c.entries {
if e.Next.After(now) || e.Next.IsZero() {
break
}
c.startJob(e.WrappedJob)
e.Prev = e.Next
e.Next = e.Schedule.Next(now)
c.logger.Info("run", "now", now, "entry", e.ID, "next", e.Next)
}
case newEntry := <-c.add:
timer.Stop()
now = c.now()
newEntry.Next = newEntry.Schedule.Next(now)
c.entries = append(c.entries, newEntry)
c.logger.Info("added", "now", now, "entry", newEntry.ID, "next", newEntry.Next)
case replyChan := <-c.snapshot:
replyChan <- c.entrySnapshot()
continue
case <-c.stop:
timer.Stop()
c.logger.Info("stop")
return
case id := <-c.remove:
timer.Stop()
now = c.now()
c.removeEntry(id)
c.logger.Info("removed", "entry", id)
}
break
}
}
}
// startJob runs the given job in a new goroutine.
func (c *Cron) startJob(j Job) {
c.jobWaiter.Add(1)
go func() {
defer c.jobWaiter.Done()
j.Run()
}()
}
// now returns current time in c location
func (c *Cron) now() time.Time {
return time.Now().In(c.location)
}
// Stop stops the cron scheduler if it is running; otherwise it does nothing.
// A context is returned so the caller can wait for running jobs to complete.
func (c *Cron) Stop() context.Context {
c.runningMu.Lock()
defer c.runningMu.Unlock()
if c.running {
c.stop <- struct{}{}
c.running = false
}
ctx, cancel := context.WithCancel(context.Background())
go func() {
c.jobWaiter.Wait()
cancel()
}()
return ctx
}
// entrySnapshot returns a copy of the current cron entry list.
func (c *Cron) entrySnapshot() []Entry {
var entries = make([]Entry, len(c.entries))
for i, e := range c.entries {
entries[i] = *e
}
return entries
}
func (c *Cron) removeEntry(id EntryID) {
var entries []*Entry
for _, e := range c.entries {
if e.ID != id {
entries = append(entries, e)
}
}
c.entries = entries
}

231
vendor/github.com/robfig/cron/v3/doc.go generated vendored Normal file
View file

@ -0,0 +1,231 @@
/*
Package cron implements a cron spec parser and job runner.
Installation
To download the specific tagged release, run:
go get github.com/robfig/cron/v3@v3.0.0
Import it in your program as:
import "github.com/robfig/cron/v3"
It requires Go 1.11 or later due to usage of Go Modules.
Usage
Callers may register Funcs to be invoked on a given schedule. Cron will run
them in their own goroutines.
c := cron.New()
c.AddFunc("30 * * * *", func() { fmt.Println("Every hour on the half hour") })
c.AddFunc("30 3-6,20-23 * * *", func() { fmt.Println(".. in the range 3-6am, 8-11pm") })
c.AddFunc("CRON_TZ=Asia/Tokyo 30 04 * * *", func() { fmt.Println("Runs at 04:30 Tokyo time every day") })
c.AddFunc("@hourly", func() { fmt.Println("Every hour, starting an hour from now") })
c.AddFunc("@every 1h30m", func() { fmt.Println("Every hour thirty, starting an hour thirty from now") })
c.Start()
..
// Funcs are invoked in their own goroutine, asynchronously.
...
// Funcs may also be added to a running Cron
c.AddFunc("@daily", func() { fmt.Println("Every day") })
..
// Inspect the cron job entries' next and previous run times.
inspect(c.Entries())
..
c.Stop() // Stop the scheduler (does not stop any jobs already running).
CRON Expression Format
A cron expression represents a set of times, using 5 space-separated fields.
Field name | Mandatory? | Allowed values | Allowed special characters
---------- | ---------- | -------------- | --------------------------
Minutes | Yes | 0-59 | * / , -
Hours | Yes | 0-23 | * / , -
Day of month | Yes | 1-31 | * / , - ?
Month | Yes | 1-12 or JAN-DEC | * / , -
Day of week | Yes | 0-6 or SUN-SAT | * / , - ?
Month and Day-of-week field values are case insensitive. "SUN", "Sun", and
"sun" are equally accepted.
The specific interpretation of the format is based on the Cron Wikipedia page:
https://en.wikipedia.org/wiki/Cron
Alternative Formats
Alternative Cron expression formats support other fields like seconds. You can
implement that by creating a custom Parser as follows.
cron.New(
cron.WithParser(
cron.NewParser(
cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor)))
Since adding Seconds is the most common modification to the standard cron spec,
cron provides a builtin function to do that, which is equivalent to the custom
parser you saw earlier, except that its seconds field is REQUIRED:
cron.New(cron.WithSeconds())
That emulates Quartz, the most popular alternative Cron schedule format:
http://www.quartz-scheduler.org/documentation/quartz-2.x/tutorials/crontrigger.html
Special Characters
Asterisk ( * )
The asterisk indicates that the cron expression will match for all values of the
field; e.g., using an asterisk in the 5th field (month) would indicate every
month.
Slash ( / )
Slashes are used to describe increments of ranges. For example 3-59/15 in the
1st field (minutes) would indicate the 3rd minute of the hour and every 15
minutes thereafter. The form "*\/..." is equivalent to the form "first-last/...",
that is, an increment over the largest possible range of the field. The form
"N/..." is accepted as meaning "N-MAX/...", that is, starting at N, use the
increment until the end of that specific range. It does not wrap around.
Comma ( , )
Commas are used to separate items of a list. For example, using "MON,WED,FRI" in
the 5th field (day of week) would mean Mondays, Wednesdays and Fridays.
Hyphen ( - )
Hyphens are used to define ranges. For example, 9-17 would indicate every
hour between 9am and 5pm inclusive.
Question mark ( ? )
Question mark may be used instead of '*' for leaving either day-of-month or
day-of-week blank.
Predefined schedules
You may use one of several pre-defined schedules in place of a cron expression.
Entry | Description | Equivalent To
----- | ----------- | -------------
@yearly (or @annually) | Run once a year, midnight, Jan. 1st | 0 0 1 1 *
@monthly | Run once a month, midnight, first of month | 0 0 1 * *
@weekly | Run once a week, midnight between Sat/Sun | 0 0 * * 0
@daily (or @midnight) | Run once a day, midnight | 0 0 * * *
@hourly | Run once an hour, beginning of hour | 0 * * * *
Intervals
You may also schedule a job to execute at fixed intervals, starting at the time it's added
or cron is run. This is supported by formatting the cron spec like this:
@every <duration>
where "duration" is a string accepted by time.ParseDuration
(http://golang.org/pkg/time/#ParseDuration).
For example, "@every 1h30m10s" would indicate a schedule that activates after
1 hour, 30 minutes, 10 seconds, and then every interval after that.
Note: The interval does not take the job runtime into account. For example,
if a job takes 3 minutes to run, and it is scheduled to run every 5 minutes,
it will have only 2 minutes of idle time between each run.
Time zones
By default, all interpretation and scheduling is done in the machine's local
time zone (time.Local). You can specify a different time zone on construction:
cron.New(
cron.WithLocation(time.UTC))
Individual cron schedules may also override the time zone they are to be
interpreted in by providing an additional space-separated field at the beginning
of the cron spec, of the form "CRON_TZ=Asia/Tokyo".
For example:
# Runs at 6am in time.Local
cron.New().AddFunc("0 6 * * ?", ...)
# Runs at 6am in America/New_York
nyc, _ := time.LoadLocation("America/New_York")
c := cron.New(cron.WithLocation(nyc))
c.AddFunc("0 6 * * ?", ...)
# Runs at 6am in Asia/Tokyo
cron.New().AddFunc("CRON_TZ=Asia/Tokyo 0 6 * * ?", ...)
# Runs at 6am in Asia/Tokyo
c := cron.New(cron.WithLocation(nyc))
c.SetLocation("America/New_York")
c.AddFunc("CRON_TZ=Asia/Tokyo 0 6 * * ?", ...)
The prefix "TZ=(TIME ZONE)" is also supported for legacy compatibility.
Be aware that jobs scheduled during daylight-savings leap-ahead transitions will
not be run!
Job Wrappers
A Cron runner may be configured with a chain of job wrappers to add
cross-cutting functionality to all submitted jobs. For example, they may be used
to achieve the following effects:
- Recover any panics from jobs (activated by default)
- Delay a job's execution if the previous run hasn't completed yet
- Skip a job's execution if the previous run hasn't completed yet
- Log each job's invocations
Install wrappers for all jobs added to a cron using the `cron.WithChain` option:
cron.New(cron.WithChain(
cron.SkipIfStillRunning(logger),
))
Install wrappers for individual jobs by explicitly wrapping them:
job = cron.NewChain(
cron.SkipIfStillRunning(logger),
).Then(job)
Thread safety
Since the Cron service runs concurrently with the calling code, some amount of
care must be taken to ensure proper synchronization.
All cron methods are designed to be correctly synchronized as long as the caller
ensures that invocations have a clear happens-before ordering between them.
Logging
Cron defines a Logger interface that is a subset of the one defined in
github.com/go-logr/logr. It has two logging levels (Info and Error), and
parameters are key/value pairs. This makes it possible for cron logging to plug
into structured logging systems. An adapter, [Verbose]PrintfLogger, is provided
to wrap the standard library *log.Logger.
For additional insight into Cron operations, verbose logging may be activated
which will record job runs, scheduling decisions, and added or removed jobs.
Activate it with a one-off logger as follows:
cron.New(
cron.WithLogger(
cron.VerbosePrintfLogger(log.New(os.Stdout, "cron: ", log.LstdFlags))))
Implementation
Cron entries are stored in an array, sorted by their next activation time. Cron
sleeps until the next job is due to be run.
Upon waking:
- it runs each entry that is active on that second
- it calculates the next run times for the jobs that were run
- it re-sorts the array of entries by next activation time.
- it goes to sleep until the soonest job.
*/
package cron

86
vendor/github.com/robfig/cron/v3/logger.go generated vendored Normal file
View file

@ -0,0 +1,86 @@
package cron
import (
"io/ioutil"
"log"
"os"
"strings"
"time"
)
// DefaultLogger is used by Cron if none is specified.
var DefaultLogger Logger = PrintfLogger(log.New(os.Stdout, "cron: ", log.LstdFlags))
// DiscardLogger can be used by callers to discard all log messages.
var DiscardLogger Logger = PrintfLogger(log.New(ioutil.Discard, "", 0))
// Logger is the interface used in this package for logging, so that any backend
// can be plugged in. It is a subset of the github.com/go-logr/logr interface.
type Logger interface {
// Info logs routine messages about cron's operation.
Info(msg string, keysAndValues ...interface{})
// Error logs an error condition.
Error(err error, msg string, keysAndValues ...interface{})
}
// PrintfLogger wraps a Printf-based logger (such as the standard library "log")
// into an implementation of the Logger interface which logs errors only.
func PrintfLogger(l interface{ Printf(string, ...interface{}) }) Logger {
return printfLogger{l, false}
}
// VerbosePrintfLogger wraps a Printf-based logger (such as the standard library
// "log") into an implementation of the Logger interface which logs everything.
func VerbosePrintfLogger(l interface{ Printf(string, ...interface{}) }) Logger {
return printfLogger{l, true}
}
type printfLogger struct {
logger interface{ Printf(string, ...interface{}) }
logInfo bool
}
func (pl printfLogger) Info(msg string, keysAndValues ...interface{}) {
if pl.logInfo {
keysAndValues = formatTimes(keysAndValues)
pl.logger.Printf(
formatString(len(keysAndValues)),
append([]interface{}{msg}, keysAndValues...)...)
}
}
func (pl printfLogger) Error(err error, msg string, keysAndValues ...interface{}) {
keysAndValues = formatTimes(keysAndValues)
pl.logger.Printf(
formatString(len(keysAndValues)+2),
append([]interface{}{msg, "error", err}, keysAndValues...)...)
}
// formatString returns a logfmt-like format string for the number of
// key/values.
func formatString(numKeysAndValues int) string {
var sb strings.Builder
sb.WriteString("%s")
if numKeysAndValues > 0 {
sb.WriteString(", ")
}
for i := 0; i < numKeysAndValues/2; i++ {
if i > 0 {
sb.WriteString(", ")
}
sb.WriteString("%v=%v")
}
return sb.String()
}
// formatTimes formats any time.Time values as RFC3339.
func formatTimes(keysAndValues []interface{}) []interface{} {
var formattedArgs []interface{}
for _, arg := range keysAndValues {
if t, ok := arg.(time.Time); ok {
arg = t.Format(time.RFC3339)
}
formattedArgs = append(formattedArgs, arg)
}
return formattedArgs
}

45
vendor/github.com/robfig/cron/v3/option.go generated vendored Normal file
View file

@ -0,0 +1,45 @@
package cron
import (
"time"
)
// Option represents a modification to the default behavior of a Cron.
type Option func(*Cron)
// WithLocation overrides the timezone of the cron instance.
func WithLocation(loc *time.Location) Option {
return func(c *Cron) {
c.location = loc
}
}
// WithSeconds overrides the parser used for interpreting job schedules to
// include a seconds field as the first one.
func WithSeconds() Option {
return WithParser(NewParser(
Second | Minute | Hour | Dom | Month | Dow | Descriptor,
))
}
// WithParser overrides the parser used for interpreting job schedules.
func WithParser(p ScheduleParser) Option {
return func(c *Cron) {
c.parser = p
}
}
// WithChain specifies Job wrappers to apply to all jobs added to this cron.
// Refer to the Chain* functions in this package for provided wrappers.
func WithChain(wrappers ...JobWrapper) Option {
return func(c *Cron) {
c.chain = NewChain(wrappers...)
}
}
// WithLogger uses the provided logger.
func WithLogger(logger Logger) Option {
return func(c *Cron) {
c.logger = logger
}
}

434
vendor/github.com/robfig/cron/v3/parser.go generated vendored Normal file
View file

@ -0,0 +1,434 @@
package cron
import (
"fmt"
"math"
"strconv"
"strings"
"time"
)
// Configuration options for creating a parser. Most options specify which
// fields should be included, while others enable features. If a field is not
// included the parser will assume a default value. These options do not change
// the order fields are parse in.
type ParseOption int
const (
Second ParseOption = 1 << iota // Seconds field, default 0
SecondOptional // Optional seconds field, default 0
Minute // Minutes field, default 0
Hour // Hours field, default 0
Dom // Day of month field, default *
Month // Month field, default *
Dow // Day of week field, default *
DowOptional // Optional day of week field, default *
Descriptor // Allow descriptors such as @monthly, @weekly, etc.
)
var places = []ParseOption{
Second,
Minute,
Hour,
Dom,
Month,
Dow,
}
var defaults = []string{
"0",
"0",
"0",
"*",
"*",
"*",
}
// A custom Parser that can be configured.
type Parser struct {
options ParseOption
}
// NewParser creates a Parser with custom options.
//
// It panics if more than one Optional is given, since it would be impossible to
// correctly infer which optional is provided or missing in general.
//
// Examples
//
// // Standard parser without descriptors
// specParser := NewParser(Minute | Hour | Dom | Month | Dow)
// sched, err := specParser.Parse("0 0 15 */3 *")
//
// // Same as above, just excludes time fields
// subsParser := NewParser(Dom | Month | Dow)
// sched, err := specParser.Parse("15 */3 *")
//
// // Same as above, just makes Dow optional
// subsParser := NewParser(Dom | Month | DowOptional)
// sched, err := specParser.Parse("15 */3")
//
func NewParser(options ParseOption) Parser {
optionals := 0
if options&DowOptional > 0 {
optionals++
}
if options&SecondOptional > 0 {
optionals++
}
if optionals > 1 {
panic("multiple optionals may not be configured")
}
return Parser{options}
}
// Parse returns a new crontab schedule representing the given spec.
// It returns a descriptive error if the spec is not valid.
// It accepts crontab specs and features configured by NewParser.
func (p Parser) Parse(spec string) (Schedule, error) {
if len(spec) == 0 {
return nil, fmt.Errorf("empty spec string")
}
// Extract timezone if present
var loc = time.Local
if strings.HasPrefix(spec, "TZ=") || strings.HasPrefix(spec, "CRON_TZ=") {
var err error
i := strings.Index(spec, " ")
eq := strings.Index(spec, "=")
if loc, err = time.LoadLocation(spec[eq+1 : i]); err != nil {
return nil, fmt.Errorf("provided bad location %s: %v", spec[eq+1:i], err)
}
spec = strings.TrimSpace(spec[i:])
}
// Handle named schedules (descriptors), if configured
if strings.HasPrefix(spec, "@") {
if p.options&Descriptor == 0 {
return nil, fmt.Errorf("parser does not accept descriptors: %v", spec)
}
return parseDescriptor(spec, loc)
}
// Split on whitespace.
fields := strings.Fields(spec)
// Validate & fill in any omitted or optional fields
var err error
fields, err = normalizeFields(fields, p.options)
if err != nil {
return nil, err
}
field := func(field string, r bounds) uint64 {
if err != nil {
return 0
}
var bits uint64
bits, err = getField(field, r)
return bits
}
var (
second = field(fields[0], seconds)
minute = field(fields[1], minutes)
hour = field(fields[2], hours)
dayofmonth = field(fields[3], dom)
month = field(fields[4], months)
dayofweek = field(fields[5], dow)
)
if err != nil {
return nil, err
}
return &SpecSchedule{
Second: second,
Minute: minute,
Hour: hour,
Dom: dayofmonth,
Month: month,
Dow: dayofweek,
Location: loc,
}, nil
}
// normalizeFields takes a subset set of the time fields and returns the full set
// with defaults (zeroes) populated for unset fields.
//
// As part of performing this function, it also validates that the provided
// fields are compatible with the configured options.
func normalizeFields(fields []string, options ParseOption) ([]string, error) {
// Validate optionals & add their field to options
optionals := 0
if options&SecondOptional > 0 {
options |= Second
optionals++
}
if options&DowOptional > 0 {
options |= Dow
optionals++
}
if optionals > 1 {
return nil, fmt.Errorf("multiple optionals may not be configured")
}
// Figure out how many fields we need
max := 0
for _, place := range places {
if options&place > 0 {
max++
}
}
min := max - optionals
// Validate number of fields
if count := len(fields); count < min || count > max {
if min == max {
return nil, fmt.Errorf("expected exactly %d fields, found %d: %s", min, count, fields)
}
return nil, fmt.Errorf("expected %d to %d fields, found %d: %s", min, max, count, fields)
}
// Populate the optional field if not provided
if min < max && len(fields) == min {
switch {
case options&DowOptional > 0:
fields = append(fields, defaults[5]) // TODO: improve access to default
case options&SecondOptional > 0:
fields = append([]string{defaults[0]}, fields...)
default:
return nil, fmt.Errorf("unknown optional field")
}
}
// Populate all fields not part of options with their defaults
n := 0
expandedFields := make([]string, len(places))
copy(expandedFields, defaults)
for i, place := range places {
if options&place > 0 {
expandedFields[i] = fields[n]
n++
}
}
return expandedFields, nil
}
var standardParser = NewParser(
Minute | Hour | Dom | Month | Dow | Descriptor,
)
// ParseStandard returns a new crontab schedule representing the given
// standardSpec (https://en.wikipedia.org/wiki/Cron). It requires 5 entries
// representing: minute, hour, day of month, month and day of week, in that
// order. It returns a descriptive error if the spec is not valid.
//
// It accepts
// - Standard crontab specs, e.g. "* * * * ?"
// - Descriptors, e.g. "@midnight", "@every 1h30m"
func ParseStandard(standardSpec string) (Schedule, error) {
return standardParser.Parse(standardSpec)
}
// getField returns an Int with the bits set representing all of the times that
// the field represents or error parsing field value. A "field" is a comma-separated
// list of "ranges".
func getField(field string, r bounds) (uint64, error) {
var bits uint64
ranges := strings.FieldsFunc(field, func(r rune) bool { return r == ',' })
for _, expr := range ranges {
bit, err := getRange(expr, r)
if err != nil {
return bits, err
}
bits |= bit
}
return bits, nil
}
// getRange returns the bits indicated by the given expression:
// number | number "-" number [ "/" number ]
// or error parsing range.
func getRange(expr string, r bounds) (uint64, error) {
var (
start, end, step uint
rangeAndStep = strings.Split(expr, "/")
lowAndHigh = strings.Split(rangeAndStep[0], "-")
singleDigit = len(lowAndHigh) == 1
err error
)
var extra uint64
if lowAndHigh[0] == "*" || lowAndHigh[0] == "?" {
start = r.min
end = r.max
extra = starBit
} else {
start, err = parseIntOrName(lowAndHigh[0], r.names)
if err != nil {
return 0, err
}
switch len(lowAndHigh) {
case 1:
end = start
case 2:
end, err = parseIntOrName(lowAndHigh[1], r.names)
if err != nil {
return 0, err
}
default:
return 0, fmt.Errorf("too many hyphens: %s", expr)
}
}
switch len(rangeAndStep) {
case 1:
step = 1
case 2:
step, err = mustParseInt(rangeAndStep[1])
if err != nil {
return 0, err
}
// Special handling: "N/step" means "N-max/step".
if singleDigit {
end = r.max
}
if step > 1 {
extra = 0
}
default:
return 0, fmt.Errorf("too many slashes: %s", expr)
}
if start < r.min {
return 0, fmt.Errorf("beginning of range (%d) below minimum (%d): %s", start, r.min, expr)
}
if end > r.max {
return 0, fmt.Errorf("end of range (%d) above maximum (%d): %s", end, r.max, expr)
}
if start > end {
return 0, fmt.Errorf("beginning of range (%d) beyond end of range (%d): %s", start, end, expr)
}
if step == 0 {
return 0, fmt.Errorf("step of range should be a positive number: %s", expr)
}
return getBits(start, end, step) | extra, nil
}
// parseIntOrName returns the (possibly-named) integer contained in expr.
func parseIntOrName(expr string, names map[string]uint) (uint, error) {
if names != nil {
if namedInt, ok := names[strings.ToLower(expr)]; ok {
return namedInt, nil
}
}
return mustParseInt(expr)
}
// mustParseInt parses the given expression as an int or returns an error.
func mustParseInt(expr string) (uint, error) {
num, err := strconv.Atoi(expr)
if err != nil {
return 0, fmt.Errorf("failed to parse int from %s: %s", expr, err)
}
if num < 0 {
return 0, fmt.Errorf("negative number (%d) not allowed: %s", num, expr)
}
return uint(num), nil
}
// getBits sets all bits in the range [min, max], modulo the given step size.
func getBits(min, max, step uint) uint64 {
var bits uint64
// If step is 1, use shifts.
if step == 1 {
return ^(math.MaxUint64 << (max + 1)) & (math.MaxUint64 << min)
}
// Else, use a simple loop.
for i := min; i <= max; i += step {
bits |= 1 << i
}
return bits
}
// all returns all bits within the given bounds. (plus the star bit)
func all(r bounds) uint64 {
return getBits(r.min, r.max, 1) | starBit
}
// parseDescriptor returns a predefined schedule for the expression, or error if none matches.
func parseDescriptor(descriptor string, loc *time.Location) (Schedule, error) {
switch descriptor {
case "@yearly", "@annually":
return &SpecSchedule{
Second: 1 << seconds.min,
Minute: 1 << minutes.min,
Hour: 1 << hours.min,
Dom: 1 << dom.min,
Month: 1 << months.min,
Dow: all(dow),
Location: loc,
}, nil
case "@monthly":
return &SpecSchedule{
Second: 1 << seconds.min,
Minute: 1 << minutes.min,
Hour: 1 << hours.min,
Dom: 1 << dom.min,
Month: all(months),
Dow: all(dow),
Location: loc,
}, nil
case "@weekly":
return &SpecSchedule{
Second: 1 << seconds.min,
Minute: 1 << minutes.min,
Hour: 1 << hours.min,
Dom: all(dom),
Month: all(months),
Dow: 1 << dow.min,
Location: loc,
}, nil
case "@daily", "@midnight":
return &SpecSchedule{
Second: 1 << seconds.min,
Minute: 1 << minutes.min,
Hour: 1 << hours.min,
Dom: all(dom),
Month: all(months),
Dow: all(dow),
Location: loc,
}, nil
case "@hourly":
return &SpecSchedule{
Second: 1 << seconds.min,
Minute: 1 << minutes.min,
Hour: all(hours),
Dom: all(dom),
Month: all(months),
Dow: all(dow),
Location: loc,
}, nil
}
const every = "@every "
if strings.HasPrefix(descriptor, every) {
duration, err := time.ParseDuration(descriptor[len(every):])
if err != nil {
return nil, fmt.Errorf("failed to parse duration %s: %s", descriptor, err)
}
return Every(duration), nil
}
return nil, fmt.Errorf("unrecognized descriptor: %s", descriptor)
}

188
vendor/github.com/robfig/cron/v3/spec.go generated vendored Normal file
View file

@ -0,0 +1,188 @@
package cron
import "time"
// SpecSchedule specifies a duty cycle (to the second granularity), based on a
// traditional crontab specification. It is computed initially and stored as bit sets.
type SpecSchedule struct {
Second, Minute, Hour, Dom, Month, Dow uint64
// Override location for this schedule.
Location *time.Location
}
// bounds provides a range of acceptable values (plus a map of name to value).
type bounds struct {
min, max uint
names map[string]uint
}
// The bounds for each field.
var (
seconds = bounds{0, 59, nil}
minutes = bounds{0, 59, nil}
hours = bounds{0, 23, nil}
dom = bounds{1, 31, nil}
months = bounds{1, 12, map[string]uint{
"jan": 1,
"feb": 2,
"mar": 3,
"apr": 4,
"may": 5,
"jun": 6,
"jul": 7,
"aug": 8,
"sep": 9,
"oct": 10,
"nov": 11,
"dec": 12,
}}
dow = bounds{0, 6, map[string]uint{
"sun": 0,
"mon": 1,
"tue": 2,
"wed": 3,
"thu": 4,
"fri": 5,
"sat": 6,
}}
)
const (
// Set the top bit if a star was included in the expression.
starBit = 1 << 63
)
// Next returns the next time this schedule is activated, greater than the given
// time. If no time can be found to satisfy the schedule, return the zero time.
func (s *SpecSchedule) Next(t time.Time) time.Time {
// General approach
//
// For Month, Day, Hour, Minute, Second:
// Check if the time value matches. If yes, continue to the next field.
// If the field doesn't match the schedule, then increment the field until it matches.
// While incrementing the field, a wrap-around brings it back to the beginning
// of the field list (since it is necessary to re-verify previous field
// values)
// Convert the given time into the schedule's timezone, if one is specified.
// Save the original timezone so we can convert back after we find a time.
// Note that schedules without a time zone specified (time.Local) are treated
// as local to the time provided.
origLocation := t.Location()
loc := s.Location
if loc == time.Local {
loc = t.Location()
}
if s.Location != time.Local {
t = t.In(s.Location)
}
// Start at the earliest possible time (the upcoming second).
t = t.Add(1*time.Second - time.Duration(t.Nanosecond())*time.Nanosecond)
// This flag indicates whether a field has been incremented.
added := false
// If no time is found within five years, return zero.
yearLimit := t.Year() + 5
WRAP:
if t.Year() > yearLimit {
return time.Time{}
}
// Find the first applicable month.
// If it's this month, then do nothing.
for 1<<uint(t.Month())&s.Month == 0 {
// If we have to add a month, reset the other parts to 0.
if !added {
added = true
// Otherwise, set the date at the beginning (since the current time is irrelevant).
t = time.Date(t.Year(), t.Month(), 1, 0, 0, 0, 0, loc)
}
t = t.AddDate(0, 1, 0)
// Wrapped around.
if t.Month() == time.January {
goto WRAP
}
}
// Now get a day in that month.
//
// NOTE: This causes issues for daylight savings regimes where midnight does
// not exist. For example: Sao Paulo has DST that transforms midnight on
// 11/3 into 1am. Handle that by noticing when the Hour ends up != 0.
for !dayMatches(s, t) {
if !added {
added = true
t = time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, loc)
}
t = t.AddDate(0, 0, 1)
// Notice if the hour is no longer midnight due to DST.
// Add an hour if it's 23, subtract an hour if it's 1.
if t.Hour() != 0 {
if t.Hour() > 12 {
t = t.Add(time.Duration(24-t.Hour()) * time.Hour)
} else {
t = t.Add(time.Duration(-t.Hour()) * time.Hour)
}
}
if t.Day() == 1 {
goto WRAP
}
}
for 1<<uint(t.Hour())&s.Hour == 0 {
if !added {
added = true
t = time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), 0, 0, 0, loc)
}
t = t.Add(1 * time.Hour)
if t.Hour() == 0 {
goto WRAP
}
}
for 1<<uint(t.Minute())&s.Minute == 0 {
if !added {
added = true
t = t.Truncate(time.Minute)
}
t = t.Add(1 * time.Minute)
if t.Minute() == 0 {
goto WRAP
}
}
for 1<<uint(t.Second())&s.Second == 0 {
if !added {
added = true
t = t.Truncate(time.Second)
}
t = t.Add(1 * time.Second)
if t.Second() == 0 {
goto WRAP
}
}
return t.In(origLocation)
}
// dayMatches returns true if the schedule's day-of-week and day-of-month
// restrictions are satisfied by the given time.
func dayMatches(s *SpecSchedule, t time.Time) bool {
var (
domMatch bool = 1<<uint(t.Day())&s.Dom > 0
dowMatch bool = 1<<uint(t.Weekday())&s.Dow > 0
)
if s.Dom&starBit > 0 || s.Dow&starBit > 0 {
return domMatch && dowMatch
}
return domMatch || dowMatch
}

23
vendor/github.com/uptrace/bun/CHANGELOG.md generated vendored
View file

@ -1,3 +1,26 @@
## [1.0.20](https://github.com/uptrace/bun/compare/v1.0.19...v1.0.20) (2021-12-19)
### Bug Fixes
* add Event.QueryTemplate and change Event.Query to be always formatted ([52b1ccd](https://github.com/uptrace/bun/commit/52b1ccdf3578418aa427adef9dcf942d90ae4fdd))
* change GetTableName to return formatted table name in case ModelTableExpr ([95144dd](https://github.com/uptrace/bun/commit/95144dde937b4ac88b36b0bd8b01372421069b44))
* change ScanAndCount to work with transactions ([5b3f2c0](https://github.com/uptrace/bun/commit/5b3f2c021c424da366caffd33589e8adde821403))
* **dbfixture:** directly call funcs bypassing template eval ([a61974b](https://github.com/uptrace/bun/commit/a61974ba2d24361c5357fb9bda1f3eceec5a45cd))
* don't append CASCADE by default in drop table/column queries ([26457ea](https://github.com/uptrace/bun/commit/26457ea5cb20862d232e6e5fa4dbdeac5d444bf1))
* **migrate:** mark migrations as applied on error so the migration can be rolled back ([8ce33fb](https://github.com/uptrace/bun/commit/8ce33fbbac8e33077c20daf19a14c5ff2291bcae))
* respect nullzero when appending struct fields. Fixes [#339](https://github.com/uptrace/bun/issues/339) ([ffd02f3](https://github.com/uptrace/bun/commit/ffd02f3170b3cccdd670a48d563cfb41094c05d6))
* reuse tx for relation join ([#366](https://github.com/uptrace/bun/issues/366)) ([60bdb1a](https://github.com/uptrace/bun/commit/60bdb1ac84c0a699429eead3b7fdfbf14fe69ac6))
### Features
* add `Dialect()` to Transaction and IDB interface ([693f1e1](https://github.com/uptrace/bun/commit/693f1e135999fc31cf83b99a2530a695b20f4e1b))
* add model embedding via embed:prefix_ ([9a2cedc](https://github.com/uptrace/bun/commit/9a2cedc8b08fa8585d4bfced338bd0a40d736b1d))
* change the default logoutput to stderr ([4bf5773](https://github.com/uptrace/bun/commit/4bf577382f19c64457cbf0d64490401450954654)), closes [#349](https://github.com/uptrace/bun/issues/349)
## [1.0.19](https://github.com/uptrace/bun/compare/v1.0.18...v1.0.19) (2021-11-30)

141
vendor/github.com/uptrace/bun/README.md generated vendored
View file

@ -43,8 +43,147 @@ Projects using Bun:
demand.
- [RealWorld app](https://github.com/go-bun/bun-realworld-app)
## Benchmark
[https://github.com/davars/dbeval](https://github.com/davars/dbeval)
<details>
<summary>github.com/frederikhors/orm-benchmark results</summary>
<summary>results</summary>
```
BenchmarkInsert
BenchmarkInsert/*dbeval.Memory/Authors
BenchmarkInsert/*dbeval.Memory/Authors-4 84450 12104 ns/op 2623 B/op 70 allocs/op
BenchmarkInsert/*dbeval.Xorm/Authors
BenchmarkInsert/*dbeval.Xorm/Authors-4 7291 153505 ns/op 9024 B/op 311 allocs/op
BenchmarkInsert/*dbeval.UpperDB/Authors
BenchmarkInsert/*dbeval.UpperDB/Authors-4 4608 223672 ns/op 24160 B/op 1100 allocs/op
BenchmarkInsert/*dbeval.Bun/Authors
BenchmarkInsert/*dbeval.Bun/Authors-4 6034 186439 ns/op 6818 B/op 80 allocs/op
BenchmarkInsert/*dbeval.PQ/Authors
BenchmarkInsert/*dbeval.PQ/Authors-4 1141 907494 ns/op 6487 B/op 193 allocs/op
BenchmarkInsert/*dbeval.SQLX/Authors
BenchmarkInsert/*dbeval.SQLX/Authors-4 1165 916987 ns/op 10089 B/op 271 allocs/op
BenchmarkInsert/*dbeval.Ozzo/Authors
BenchmarkInsert/*dbeval.Ozzo/Authors-4 1105 1058082 ns/op 27826 B/op 588 allocs/op
BenchmarkInsert/*dbeval.PGXStdlib/Authors
BenchmarkInsert/*dbeval.PGXStdlib/Authors-4 1228 900207 ns/op 6032 B/op 180 allocs/op
BenchmarkInsert/*dbeval.Gorm/Authors
BenchmarkInsert/*dbeval.Gorm/Authors-4 946 1184285 ns/op 35634 B/op 918 allocs/op
BenchmarkInsert/*dbeval.PGX/Authors
BenchmarkInsert/*dbeval.PGX/Authors-4 1116 923728 ns/op 3839 B/op 130 allocs/op
BenchmarkInsert/*dbeval.DBR/Authors
BenchmarkInsert/*dbeval.DBR/Authors-4 5800 183982 ns/op 8646 B/op 230 allocs/op
BenchmarkInsert/*dbeval.GoPG/Authors
BenchmarkInsert/*dbeval.GoPG/Authors-4 6110 173923 ns/op 2906 B/op 87 allocs/op
BenchmarkInsert/*dbeval.DBR/Articles
BenchmarkInsert/*dbeval.DBR/Articles-4 1706 684466 ns/op 133346 B/op 1614 allocs/op
BenchmarkInsert/*dbeval.PQ/Articles
BenchmarkInsert/*dbeval.PQ/Articles-4 884 1249791 ns/op 100403 B/op 1491 allocs/op
BenchmarkInsert/*dbeval.PGX/Articles
BenchmarkInsert/*dbeval.PGX/Articles-4 916 1288143 ns/op 83539 B/op 1392 allocs/op
BenchmarkInsert/*dbeval.GoPG/Articles
BenchmarkInsert/*dbeval.GoPG/Articles-4 1726 622639 ns/op 78638 B/op 1359 allocs/op
BenchmarkInsert/*dbeval.SQLX/Articles
BenchmarkInsert/*dbeval.SQLX/Articles-4 860 1262599 ns/op 92030 B/op 1574 allocs/op
BenchmarkInsert/*dbeval.Gorm/Articles
BenchmarkInsert/*dbeval.Gorm/Articles-4 782 1421550 ns/op 136534 B/op 2411 allocs/op
BenchmarkInsert/*dbeval.PGXStdlib/Articles
BenchmarkInsert/*dbeval.PGXStdlib/Articles-4 938 1230576 ns/op 86743 B/op 1441 allocs/op
BenchmarkInsert/*dbeval.Bun/Articles
BenchmarkInsert/*dbeval.Bun/Articles-4 1843 626681 ns/op 101610 B/op 1323 allocs/op
BenchmarkInsert/*dbeval.Xorm/Articles
BenchmarkInsert/*dbeval.Xorm/Articles-4 1677 650244 ns/op 126677 B/op 1752 allocs/op
BenchmarkInsert/*dbeval.Memory/Articles
BenchmarkInsert/*dbeval.Memory/Articles-4 1988 1223308 ns/op 77576 B/op 1310 allocs/op
BenchmarkInsert/*dbeval.UpperDB/Articles
BenchmarkInsert/*dbeval.UpperDB/Articles-4 1696 687130 ns/op 139680 B/op 2862 allocs/op
BenchmarkInsert/*dbeval.Ozzo/Articles
BenchmarkInsert/*dbeval.Ozzo/Articles-4 697 1496859 ns/op 114780 B/op 1950 allocs/op
BenchmarkFindAuthorByID
BenchmarkFindAuthorByID/*dbeval.UpperDB
BenchmarkFindAuthorByID/*dbeval.UpperDB-4 10184 117527 ns/op 9953 B/op 441 allocs/op
BenchmarkFindAuthorByID/*dbeval.Bun
BenchmarkFindAuthorByID/*dbeval.Bun-4 20716 54261 ns/op 5096 B/op 15 allocs/op
BenchmarkFindAuthorByID/*dbeval.Ozzo
BenchmarkFindAuthorByID/*dbeval.Ozzo-4 11166 91043 ns/op 3088 B/op 64 allocs/op
BenchmarkFindAuthorByID/*dbeval.PQ
BenchmarkFindAuthorByID/*dbeval.PQ-4 13875 86171 ns/op 844 B/op 24 allocs/op
BenchmarkFindAuthorByID/*dbeval.PGX
BenchmarkFindAuthorByID/*dbeval.PGX-4 13846 79983 ns/op 719 B/op 15 allocs/op
BenchmarkFindAuthorByID/*dbeval.Memory
BenchmarkFindAuthorByID/*dbeval.Memory-4 14113720 82.33 ns/op 0 B/op 0 allocs/op
BenchmarkFindAuthorByID/*dbeval.Xorm
BenchmarkFindAuthorByID/*dbeval.Xorm-4 12027 98519 ns/op 3633 B/op 106 allocs/op
BenchmarkFindAuthorByID/*dbeval.Gorm
BenchmarkFindAuthorByID/*dbeval.Gorm-4 11521 102241 ns/op 6592 B/op 143 allocs/op
BenchmarkFindAuthorByID/*dbeval.PGXStdlib
BenchmarkFindAuthorByID/*dbeval.PGXStdlib-4 13933 82626 ns/op 1174 B/op 28 allocs/op
BenchmarkFindAuthorByID/*dbeval.DBR
BenchmarkFindAuthorByID/*dbeval.DBR-4 21920 51175 ns/op 1756 B/op 39 allocs/op
BenchmarkFindAuthorByID/*dbeval.SQLX
BenchmarkFindAuthorByID/*dbeval.SQLX-4 13603 80788 ns/op 1327 B/op 32 allocs/op
BenchmarkFindAuthorByID/*dbeval.GoPG
BenchmarkFindAuthorByID/*dbeval.GoPG-4 23174 50042 ns/op 869 B/op 17 allocs/op
BenchmarkFindAuthorByName
BenchmarkFindAuthorByName/*dbeval.SQLX
BenchmarkFindAuthorByName/*dbeval.SQLX-4 1070 1065272 ns/op 126348 B/op 4018 allocs/op
BenchmarkFindAuthorByName/*dbeval.Bun
BenchmarkFindAuthorByName/*dbeval.Bun-4 877 1231377 ns/op 115803 B/op 5005 allocs/op
BenchmarkFindAuthorByName/*dbeval.Xorm
BenchmarkFindAuthorByName/*dbeval.Xorm-4 471 2345445 ns/op 455711 B/op 19080 allocs/op
BenchmarkFindAuthorByName/*dbeval.DBR
BenchmarkFindAuthorByName/*dbeval.DBR-4 954 1089977 ns/op 120070 B/op 6023 allocs/op
BenchmarkFindAuthorByName/*dbeval.PQ
BenchmarkFindAuthorByName/*dbeval.PQ-4 1333 784400 ns/op 87159 B/op 4006 allocs/op
BenchmarkFindAuthorByName/*dbeval.GoPG
BenchmarkFindAuthorByName/*dbeval.GoPG-4 1580 770966 ns/op 87525 B/op 3028 allocs/op
BenchmarkFindAuthorByName/*dbeval.UpperDB
BenchmarkFindAuthorByName/*dbeval.UpperDB-4 789 1314164 ns/op 190689 B/op 6428 allocs/op
BenchmarkFindAuthorByName/*dbeval.Ozzo
BenchmarkFindAuthorByName/*dbeval.Ozzo-4 948 1255282 ns/op 238764 B/op 6053 allocs/op
BenchmarkFindAuthorByName/*dbeval.PGXStdlib
BenchmarkFindAuthorByName/*dbeval.PGXStdlib-4 1279 920391 ns/op 126163 B/op 4014 allocs/op
BenchmarkFindAuthorByName/*dbeval.PGX
BenchmarkFindAuthorByName/*dbeval.PGX-4 1364 780970 ns/op 101967 B/op 2028 allocs/op
BenchmarkFindAuthorByName/*dbeval.Gorm
BenchmarkFindAuthorByName/*dbeval.Gorm-4 340 3445818 ns/op 1573637 B/op 27102 allocs/op
BenchmarkFindAuthorByName/*dbeval.Memory
BenchmarkFindAuthorByName/*dbeval.Memory-4 38081223 31.24 ns/op 0 B/op 0 allocs/op
BenchmarkRecentArticles
BenchmarkRecentArticles/*dbeval.PGXStdlib
BenchmarkRecentArticles/*dbeval.PGXStdlib-4 358 3344119 ns/op 3425578 B/op 14177 allocs/op
BenchmarkRecentArticles/*dbeval.GoPG
BenchmarkRecentArticles/*dbeval.GoPG-4 364 3156372 ns/op 1794091 B/op 10032 allocs/op
BenchmarkRecentArticles/*dbeval.Xorm
BenchmarkRecentArticles/*dbeval.Xorm-4 157 7567835 ns/op 5018011 B/op 81425 allocs/op
BenchmarkRecentArticles/*dbeval.Gorm
BenchmarkRecentArticles/*dbeval.Gorm-4 139 7980084 ns/op 6776277 B/op 85418 allocs/op
BenchmarkRecentArticles/*dbeval.SQLX
BenchmarkRecentArticles/*dbeval.SQLX-4 338 3289802 ns/op 3425890 B/op 14181 allocs/op
BenchmarkRecentArticles/*dbeval.Ozzo
BenchmarkRecentArticles/*dbeval.Ozzo-4 320 3508322 ns/op 4025966 B/op 18207 allocs/op
BenchmarkRecentArticles/*dbeval.DBR
BenchmarkRecentArticles/*dbeval.DBR-4 237 5248644 ns/op 3331003 B/op 21370 allocs/op
BenchmarkRecentArticles/*dbeval.Bun
BenchmarkRecentArticles/*dbeval.Bun-4 280 4528582 ns/op 1864362 B/op 15965 allocs/op
BenchmarkRecentArticles/*dbeval.UpperDB
BenchmarkRecentArticles/*dbeval.UpperDB-4 297 3704663 ns/op 3607287 B/op 18542 allocs/op
BenchmarkRecentArticles/*dbeval.PQ
BenchmarkRecentArticles/*dbeval.PQ-4 308 3489229 ns/op 3277050 B/op 17359 allocs/op
BenchmarkRecentArticles/*dbeval.Memory
BenchmarkRecentArticles/*dbeval.Memory-4 29590380 42.27 ns/op 0 B/op 0 allocs/op
BenchmarkRecentArticles/*dbeval.PGX
BenchmarkRecentArticles/*dbeval.PGX-4 356 3345500 ns/op 3297316 B/op 6226 allocs/op
```
</details>
[https://github.com/frederikhors/orm-benchmark](https://github.com/frederikhors/orm-benchmark)
<details>
<summary>results</summary>
```
4000 times - Insert

62
vendor/github.com/uptrace/bun/db.go generated vendored
View file

@ -32,6 +32,7 @@ func WithDiscardUnknownColumns() DBOption {
type DB struct {
*sql.DB
dialect schema.Dialect
features feature.Feature
@ -125,7 +126,7 @@ func (db *DB) NewDropColumn() *DropColumnQuery {
func (db *DB) ResetModel(ctx context.Context, models ...interface{}) error {
for _, model := range models {
if _, err := db.NewDropTable().Model(model).IfExists().Exec(ctx); err != nil {
if _, err := db.NewDropTable().Model(model).IfExists().Cascade().Exec(ctx); err != nil {
return err
}
if _, err := db.NewCreateTable().Model(model).Exec(ctx); err != nil {
@ -226,8 +227,9 @@ func (db *DB) Exec(query string, args ...interface{}) (sql.Result, error) {
func (db *DB) ExecContext(
ctx context.Context, query string, args ...interface{},
) (sql.Result, error) {
ctx, event := db.beforeQuery(ctx, nil, query, args, nil)
res, err := db.DB.ExecContext(ctx, db.format(query, args))
formattedQuery := db.format(query, args)
ctx, event := db.beforeQuery(ctx, nil, query, args, formattedQuery, nil)
res, err := db.DB.ExecContext(ctx, formattedQuery)
db.afterQuery(ctx, event, res, err)
return res, err
}
@ -239,8 +241,9 @@ func (db *DB) Query(query string, args ...interface{}) (*sql.Rows, error) {
func (db *DB) QueryContext(
ctx context.Context, query string, args ...interface{},
) (*sql.Rows, error) {
ctx, event := db.beforeQuery(ctx, nil, query, args, nil)
rows, err := db.DB.QueryContext(ctx, db.format(query, args))
formattedQuery := db.format(query, args)
ctx, event := db.beforeQuery(ctx, nil, query, args, formattedQuery, nil)
rows, err := db.DB.QueryContext(ctx, formattedQuery)
db.afterQuery(ctx, event, nil, err)
return rows, err
}
@ -250,8 +253,9 @@ func (db *DB) QueryRow(query string, args ...interface{}) *sql.Row {
}
func (db *DB) QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row {
ctx, event := db.beforeQuery(ctx, nil, query, args, nil)
row := db.DB.QueryRowContext(ctx, db.format(query, args))
formattedQuery := db.format(query, args)
ctx, event := db.beforeQuery(ctx, nil, query, args, formattedQuery, nil)
row := db.DB.QueryRowContext(ctx, formattedQuery)
db.afterQuery(ctx, event, nil, row.Err())
return row
}
@ -281,8 +285,9 @@ func (db *DB) Conn(ctx context.Context) (Conn, error) {
func (c Conn) ExecContext(
ctx context.Context, query string, args ...interface{},
) (sql.Result, error) {
ctx, event := c.db.beforeQuery(ctx, nil, query, args, nil)
res, err := c.Conn.ExecContext(ctx, c.db.format(query, args))
formattedQuery := c.db.format(query, args)
ctx, event := c.db.beforeQuery(ctx, nil, query, args, formattedQuery, nil)
res, err := c.Conn.ExecContext(ctx, formattedQuery)
c.db.afterQuery(ctx, event, res, err)
return res, err
}
@ -290,19 +295,25 @@ func (c Conn) ExecContext(
func (c Conn) QueryContext(
ctx context.Context, query string, args ...interface{},
) (*sql.Rows, error) {
ctx, event := c.db.beforeQuery(ctx, nil, query, args, nil)
rows, err := c.Conn.QueryContext(ctx, c.db.format(query, args))
formattedQuery := c.db.format(query, args)
ctx, event := c.db.beforeQuery(ctx, nil, query, args, formattedQuery, nil)
rows, err := c.Conn.QueryContext(ctx, formattedQuery)
c.db.afterQuery(ctx, event, nil, err)
return rows, err
}
func (c Conn) QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row {
ctx, event := c.db.beforeQuery(ctx, nil, query, args, nil)
row := c.Conn.QueryRowContext(ctx, c.db.format(query, args))
formattedQuery := c.db.format(query, args)
ctx, event := c.db.beforeQuery(ctx, nil, query, args, formattedQuery, nil)
row := c.Conn.QueryRowContext(ctx, formattedQuery)
c.db.afterQuery(ctx, event, nil, row.Err())
return row
}
func (c Conn) Dialect() schema.Dialect {
return c.db.Dialect()
}
func (c Conn) NewValues(model interface{}) *ValuesQuery {
return NewValuesQuery(c.db, model).Conn(c)
}
@ -408,7 +419,7 @@ func (db *DB) Begin() (Tx, error) {
}
func (db *DB) BeginTx(ctx context.Context, opts *sql.TxOptions) (Tx, error) {
ctx, event := db.beforeQuery(ctx, nil, "BEGIN", nil, nil)
ctx, event := db.beforeQuery(ctx, nil, "BEGIN", nil, "BEGIN", nil)
tx, err := db.DB.BeginTx(ctx, opts)
db.afterQuery(ctx, event, nil, err)
if err != nil {
@ -422,14 +433,14 @@ func (db *DB) BeginTx(ctx context.Context, opts *sql.TxOptions) (Tx, error) {
}
func (tx Tx) Commit() error {
ctx, event := tx.db.beforeQuery(tx.ctx, nil, "COMMIT", nil, nil)
ctx, event := tx.db.beforeQuery(tx.ctx, nil, "COMMIT", nil, "COMMIT", nil)
err := tx.Tx.Commit()
tx.db.afterQuery(ctx, event, nil, err)
return err
}
func (tx Tx) Rollback() error {
ctx, event := tx.db.beforeQuery(tx.ctx, nil, "ROLLBACK", nil, nil)
ctx, event := tx.db.beforeQuery(tx.ctx, nil, "ROLLBACK", nil, "ROLLBACK", nil)
err := tx.Tx.Rollback()
tx.db.afterQuery(ctx, event, nil, err)
return err
@ -442,8 +453,9 @@ func (tx Tx) Exec(query string, args ...interface{}) (sql.Result, error) {
func (tx Tx) ExecContext(
ctx context.Context, query string, args ...interface{},
) (sql.Result, error) {
ctx, event := tx.db.beforeQuery(ctx, nil, query, args, nil)
res, err := tx.Tx.ExecContext(ctx, tx.db.format(query, args))
formattedQuery := tx.db.format(query, args)
ctx, event := tx.db.beforeQuery(ctx, nil, query, args, formattedQuery, nil)
res, err := tx.Tx.ExecContext(ctx, formattedQuery)
tx.db.afterQuery(ctx, event, res, err)
return res, err
}
@ -455,8 +467,9 @@ func (tx Tx) Query(query string, args ...interface{}) (*sql.Rows, error) {
func (tx Tx) QueryContext(
ctx context.Context, query string, args ...interface{},
) (*sql.Rows, error) {
ctx, event := tx.db.beforeQuery(ctx, nil, query, args, nil)
rows, err := tx.Tx.QueryContext(ctx, tx.db.format(query, args))
formattedQuery := tx.db.format(query, args)
ctx, event := tx.db.beforeQuery(ctx, nil, query, args, formattedQuery, nil)
rows, err := tx.Tx.QueryContext(ctx, formattedQuery)
tx.db.afterQuery(ctx, event, nil, err)
return rows, err
}
@ -466,14 +479,19 @@ func (tx Tx) QueryRow(query string, args ...interface{}) *sql.Row {
}
func (tx Tx) QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row {
ctx, event := tx.db.beforeQuery(ctx, nil, query, args, nil)
row := tx.Tx.QueryRowContext(ctx, tx.db.format(query, args))
formattedQuery := tx.db.format(query, args)
ctx, event := tx.db.beforeQuery(ctx, nil, query, args, formattedQuery, nil)
row := tx.Tx.QueryRowContext(ctx, formattedQuery)
tx.db.afterQuery(ctx, event, nil, row.Err())
return row
}
//------------------------------------------------------------------------------
func (tx Tx) Dialect() schema.Dialect {
return tx.db.Dialect()
}
func (tx Tx) NewValues(model interface{}) *ValuesQuery {
return NewValuesQuery(tx.db, model).Conn(tx)
}

7
vendor/github.com/uptrace/bun/hook.go generated vendored
View file

@ -13,9 +13,10 @@ import (
type QueryEvent struct {
DB *DB
QueryAppender schema.QueryAppender // Deprecated: use IQuery instead
QueryAppender schema.QueryAppender // DEPRECATED: use IQuery instead
IQuery Query
Query string
QueryTemplate string
QueryArgs []interface{}
Model Model
@ -51,8 +52,9 @@ type QueryHook interface {
func (db *DB) beforeQuery(
ctx context.Context,
iquery Query,
query string,
queryTemplate string,
queryArgs []interface{},
query string,
model Model,
) (context.Context, *QueryEvent) {
atomic.AddUint32(&db.stats.Queries, 1)
@ -68,6 +70,7 @@ func (db *DB) beforeQuery(
QueryAppender: iquery,
IQuery: iquery,
Query: query,
QueryTemplate: queryTemplate,
QueryArgs: queryArgs,
StartTime: time.Now(),

55
vendor/github.com/uptrace/bun/migrate/migrator.go generated vendored
View file

@ -5,7 +5,6 @@ import (
"errors"
"fmt"
"io/ioutil"
"log"
"path/filepath"
"regexp"
"time"
@ -59,11 +58,16 @@ func (m *Migrator) DB() *bun.DB {
// MigrationsWithStatus returns migrations with status in ascending order.
func (m *Migrator) MigrationsWithStatus(ctx context.Context) (MigrationSlice, error) {
sorted, _, err := m.migrationsWithStatus(ctx)
return sorted, err
}
func (m *Migrator) migrationsWithStatus(ctx context.Context) (MigrationSlice, int64, error) {
sorted := m.migrations.Sorted()
applied, err := m.selectAppliedMigrations(ctx)
if err != nil {
return nil, err
return nil, 0, err
}
appliedMap := migrationMap(applied)
@ -76,7 +80,7 @@ func (m *Migrator) MigrationsWithStatus(ctx context.Context) (MigrationSlice, er
}
}
return sorted, nil
return sorted, applied.LastGroupID(), nil
}
func (m *Migrator) Init(ctx context.Context) error {
@ -128,7 +132,7 @@ func (m *Migrator) Migrate(ctx context.Context, opts ...MigrationOption) (*Migra
}
defer m.Unlock(ctx) //nolint:errcheck
migrations, err := m.MigrationsWithStatus(ctx)
migrations, lastGroupID, err := m.migrationsWithStatus(ctx)
if err != nil {
return nil, err
}
@ -139,21 +143,22 @@ func (m *Migrator) Migrate(ctx context.Context, opts ...MigrationOption) (*Migra
if len(group.Migrations) == 0 {
return group, nil
}
group.ID = migrations.LastGroupID() + 1
group.ID = lastGroupID + 1
for i := range group.Migrations {
migration := &group.Migrations[i]
migration.GroupID = group.ID
// Always mark migration as applied so the rollback has a chance to fix the database.
if err := m.MarkApplied(ctx, migration); err != nil {
return nil, err
}
if !cfg.nop && migration.Up != nil {
if err := migration.Up(ctx, m.db); err != nil {
return group, err
}
}
if err := m.MarkApplied(ctx, migration); err != nil {
return nil, err
}
}
return group, nil
@ -195,36 +200,6 @@ func (m *Migrator) Rollback(ctx context.Context, opts ...MigrationOption) (*Migr
return lastGroup, nil
}
type MigrationStatus struct {
Migrations MigrationSlice
NewMigrations MigrationSlice
LastGroup *MigrationGroup
}
func (m *Migrator) Status(ctx context.Context) (*MigrationStatus, error) {
log.Printf(
"DEPRECATED: bun: replace Status(ctx) with " +
"MigrationsWithStatus(ctx)")
migrations, err := m.MigrationsWithStatus(ctx)
if err != nil {
return nil, err
}
return &MigrationStatus{
Migrations: migrations,
NewMigrations: migrations.Unapplied(),
LastGroup: migrations.LastGroup(),
}, nil
}
func (m *Migrator) MarkCompleted(ctx context.Context) (*MigrationGroup, error) {
log.Printf(
"DEPRECATED: bun: replace MarkCompleted(ctx) with " +
"Migrate(ctx, migrate.WithNopMigration())")
return m.Migrate(ctx, WithNopMigration())
}
type goMigrationConfig struct {
packageName string
}
@ -320,7 +295,7 @@ func (m *Migrator) genMigrationName(name string) (string, error) {
return fmt.Sprintf("%s_%s", version, name), nil
}
// MarkApplied marks the migration as applied (applied).
// MarkApplied marks the migration as applied (completed).
func (m *Migrator) MarkApplied(ctx context.Context, migration *Migration) error {
_, err := m.db.NewInsert().Model(migration).
ModelTableExpr(m.table).

2
vendor/github.com/uptrace/bun/package.json generated vendored
View file

@ -1,6 +1,6 @@
{
"name": "bun",
"version": "1.0.19",
"version": "1.0.20",
"main": "index.js",
"repository": "git@github.com:uptrace/bun.git",
"author": "Vladimir Mihailenco <vladimir.webdev@gmail.com>",

101
vendor/github.com/uptrace/bun/query_base.go generated vendored
View file

@ -43,6 +43,7 @@ var (
// IDB is a common interface for *bun.DB, bun.Conn, and bun.Tx.
type IDB interface {
IConn
Dialect() schema.Dialect
NewValues(model interface{}) *ValuesQuery
NewSelect() *SelectQuery
@ -59,9 +60,9 @@ type IDB interface {
}
var (
_ IConn = (*DB)(nil)
_ IConn = (*Conn)(nil)
_ IConn = (*Tx)(nil)
_ IDB = (*DB)(nil)
_ IDB = (*Conn)(nil)
_ IDB = (*Tx)(nil)
)
type baseQuery struct {
@ -74,10 +75,10 @@ type baseQuery struct {
tableModel TableModel
table *schema.Table
with []withQuery
modelTable schema.QueryWithArgs
tables []schema.QueryWithArgs
columns []schema.QueryWithArgs
with []withQuery
modelTableName schema.QueryWithArgs
tables []schema.QueryWithArgs
columns []schema.QueryWithArgs
flags internal.Flag
}
@ -86,13 +87,6 @@ func (q *baseQuery) DB() *DB {
return q.db
}
type query interface {
GetModel() Model
GetTableName() string
}
var _ query = (*baseQuery)(nil)
func (q *baseQuery) GetModel() Model {
return q.model
}
@ -103,15 +97,16 @@ func (q *baseQuery) GetTableName() string {
}
for _, wq := range q.with {
if v, ok := wq.query.(query); ok {
if v, ok := wq.query.(Query); ok {
if model := v.GetModel(); model != nil {
return v.GetTableName()
}
}
}
if q.modelTable.Query != "" {
return q.modelTable.Query
if q.modelTableName.Query != "" {
b, _ := q.modelTableName.AppendQuery(q.db.fmter, nil)
return string(b)
}
if len(q.tables) > 0 {
return q.tables[0].Query
@ -304,8 +299,8 @@ func (q *baseQuery) _excludeColumn(column string) bool {
//------------------------------------------------------------------------------
func (q *baseQuery) modelHasTableName() bool {
if !q.modelTable.IsZero() {
return q.modelTable.Query != ""
if !q.modelTableName.IsZero() {
return q.modelTableName.Query != ""
}
return q.table != nil
}
@ -332,8 +327,8 @@ func (q *baseQuery) _appendTables(
startLen := len(b)
if q.modelHasTableName() {
if !q.modelTable.IsZero() {
b, err = q.modelTable.AppendQuery(fmter, b)
if !q.modelTableName.IsZero() {
b, err = q.modelTableName.AppendQuery(fmter, b)
if err != nil {
return nil, err
}
@ -372,8 +367,8 @@ func (q *baseQuery) appendFirstTableWithAlias(
func (q *baseQuery) _appendFirstTable(
fmter schema.Formatter, b []byte, withAlias bool,
) ([]byte, error) {
if !q.modelTable.IsZero() {
return q.modelTable.AppendQuery(fmter, b)
if !q.modelTableName.IsZero() {
return q.modelTableName.AppendQuery(fmter, b)
}
if q.table != nil {
@ -473,7 +468,7 @@ func (q *baseQuery) scan(
model Model,
hasDest bool,
) (sql.Result, error) {
ctx, event := q.db.beforeQuery(ctx, iquery, query, nil, q.model)
ctx, event := q.db.beforeQuery(ctx, iquery, query, nil, query, q.model)
rows, err := q.conn.QueryContext(ctx, query)
if err != nil {
@ -503,16 +498,10 @@ func (q *baseQuery) exec(
iquery Query,
query string,
) (sql.Result, error) {
ctx, event := q.db.beforeQuery(ctx, iquery, query, nil, q.model)
ctx, event := q.db.beforeQuery(ctx, iquery, query, nil, query, q.model)
res, err := q.conn.ExecContext(ctx, query)
if err != nil {
q.db.afterQuery(ctx, event, nil, err)
return res, err
}
q.db.afterQuery(ctx, event, res, err)
return res, nil
q.db.afterQuery(ctx, event, nil, err)
return res, err
}
//------------------------------------------------------------------------------
@ -607,34 +596,30 @@ func (q *whereBaseQuery) addWhereCols(cols []string) {
q.setErr(err)
return
}
var fields []*schema.Field
if cols == nil {
if err := q.table.CheckPKs(); err != nil {
q.setErr(err)
return
}
fields = q.table.PKs
} else {
fields = make([]*schema.Field, len(cols))
for i, col := range cols {
field, err := q.table.Field(col)
if err != nil {
q.setErr(err)
return
}
fields[i] = field
}
}
if q.whereFields != nil {
err := errors.New("bun: WherePK can only be called once")
q.setErr(err)
return
}
q.whereFields = fields
if cols == nil {
if err := q.table.CheckPKs(); err != nil {
q.setErr(err)
return
}
q.whereFields = q.table.PKs
return
}
q.whereFields = make([]*schema.Field, len(cols))
for i, col := range cols {
field, err := q.table.Field(col)
if err != nil {
q.setErr(err)
return
}
q.whereFields[i] = field
}
}
func (q *whereBaseQuery) mustAppendWhere(
@ -951,6 +936,7 @@ func (q setQuery) appendSet(fmter schema.Formatter, b []byte) (_ []byte, err err
//------------------------------------------------------------------------------
type cascadeQuery struct {
cascade bool
restrict bool
}
@ -958,10 +944,11 @@ func (q cascadeQuery) appendCascade(fmter schema.Formatter, b []byte) []byte {
if !fmter.HasFeature(feature.TableCascade) {
return b
}
if q.cascade {
b = append(b, " CASCADE"...)
}
if q.restrict {
b = append(b, " RESTRICT"...)
} else {
b = append(b, " CASCADE"...)
}
return b
}

4
vendor/github.com/uptrace/bun/query_column_add.go generated vendored
View file

@ -15,6 +15,8 @@ type AddColumnQuery struct {
ifNotExists bool
}
var _ Query = (*AddColumnQuery)(nil)
func NewAddColumnQuery(db *DB) *AddColumnQuery {
q := &AddColumnQuery{
baseQuery: baseQuery{
@ -50,7 +52,7 @@ func (q *AddColumnQuery) TableExpr(query string, args ...interface{}) *AddColumn
}
func (q *AddColumnQuery) ModelTableExpr(query string, args ...interface{}) *AddColumnQuery {
q.modelTable = schema.SafeQuery(query, args)
q.modelTableName = schema.SafeQuery(query, args)
return q
}

4
vendor/github.com/uptrace/bun/query_column_drop.go generated vendored
View file

@ -13,6 +13,8 @@ type DropColumnQuery struct {
baseQuery
}
var _ Query = (*DropColumnQuery)(nil)
func NewDropColumnQuery(db *DB) *DropColumnQuery {
q := &DropColumnQuery{
baseQuery: baseQuery{
@ -48,7 +50,7 @@ func (q *DropColumnQuery) TableExpr(query string, args ...interface{}) *DropColu
}
func (q *DropColumnQuery) ModelTableExpr(query string, args ...interface{}) *DropColumnQuery {
q.modelTable = schema.SafeQuery(query, args)
q.modelTableName = schema.SafeQuery(query, args)
return q
}

8
vendor/github.com/uptrace/bun/query_delete.go generated vendored
View file

@ -15,6 +15,8 @@ type DeleteQuery struct {
returningQuery
}
var _ Query = (*DeleteQuery)(nil)
func NewDeleteQuery(db *DB) *DeleteQuery {
q := &DeleteQuery{
whereBaseQuery: whereBaseQuery{
@ -60,7 +62,7 @@ func (q *DeleteQuery) TableExpr(query string, args ...interface{}) *DeleteQuery
}
func (q *DeleteQuery) ModelTableExpr(query string, args ...interface{}) *DeleteQuery {
q.modelTable = schema.SafeQuery(query, args)
q.modelTableName = schema.SafeQuery(query, args)
return q
}
@ -134,6 +136,10 @@ func (q *DeleteQuery) Operation() string {
}
func (q *DeleteQuery) AppendQuery(fmter schema.Formatter, b []byte) (_ []byte, err error) {
if q.err != nil {
return nil, q.err
}
fmter = formatterWithModel(fmter, q)
if q.isSoftDelete() {

4
vendor/github.com/uptrace/bun/query_index_create.go generated vendored
View file

@ -22,6 +22,8 @@ type CreateIndexQuery struct {
include []schema.QueryWithArgs
}
var _ Query = (*CreateIndexQuery)(nil)
func NewCreateIndexQuery(db *DB) *CreateIndexQuery {
q := &CreateIndexQuery{
whereBaseQuery: whereBaseQuery{
@ -86,7 +88,7 @@ func (q *CreateIndexQuery) TableExpr(query string, args ...interface{}) *CreateI
}
func (q *CreateIndexQuery) ModelTableExpr(query string, args ...interface{}) *CreateIndexQuery {
q.modelTable = schema.SafeQuery(query, args)
q.modelTableName = schema.SafeQuery(query, args)
return q
}

7
vendor/github.com/uptrace/bun/query_index_drop.go generated vendored
View file

@ -18,6 +18,8 @@ type DropIndexQuery struct {
index schema.QueryWithArgs
}
var _ Query = (*DropIndexQuery)(nil)
func NewDropIndexQuery(db *DB) *DropIndexQuery {
q := &DropIndexQuery{
baseQuery: baseQuery{
@ -50,6 +52,11 @@ func (q *DropIndexQuery) IfExists() *DropIndexQuery {
return q
}
func (q *DropIndexQuery) Cascade() *DropIndexQuery {
q.cascade = true
return q
}
func (q *DropIndexQuery) Restrict() *DropIndexQuery {
q.restrict = true
return q

8
vendor/github.com/uptrace/bun/query_insert.go generated vendored
View file

@ -24,6 +24,8 @@ type InsertQuery struct {
replace bool
}
var _ Query = (*InsertQuery)(nil)
func NewInsertQuery(db *DB) *InsertQuery {
q := &InsertQuery{
whereBaseQuery: whereBaseQuery{
@ -71,7 +73,7 @@ func (q *InsertQuery) TableExpr(query string, args ...interface{}) *InsertQuery
}
func (q *InsertQuery) ModelTableExpr(query string, args ...interface{}) *InsertQuery {
q.modelTable = schema.SafeQuery(query, args)
q.modelTableName = schema.SafeQuery(query, args)
return q
}
@ -159,6 +161,10 @@ func (q *InsertQuery) Operation() string {
}
func (q *InsertQuery) AppendQuery(fmter schema.Formatter, b []byte) (_ []byte, err error) {
if q.err != nil {
return nil, q.err
}
fmter = formatterWithModel(fmter, q)
b, err = q.appendWith(fmter, b)

38
vendor/github.com/uptrace/bun/query_select.go generated vendored
View file

@ -34,6 +34,8 @@ type SelectQuery struct {
union []union
}
var _ Query = (*SelectQuery)(nil)
func NewSelectQuery(db *DB) *SelectQuery {
return &SelectQuery{
whereBaseQuery: whereBaseQuery{
@ -90,7 +92,7 @@ func (q *SelectQuery) TableExpr(query string, args ...interface{}) *SelectQuery
}
func (q *SelectQuery) ModelTableExpr(query string, args ...interface{}) *SelectQuery {
q.modelTable = schema.SafeQuery(query, args)
q.modelTableName = schema.SafeQuery(query, args)
return q
}
@ -342,9 +344,9 @@ func (q *SelectQuery) selectJoins(ctx context.Context, joins []relationJoin) err
case schema.HasOneRelation, schema.BelongsToRelation:
err = q.selectJoins(ctx, j.JoinModel.getJoins())
case schema.HasManyRelation:
err = j.selectMany(ctx, q.db.NewSelect())
err = j.selectMany(ctx, q.db.NewSelect().Conn(q.conn))
case schema.ManyToManyRelation:
err = j.selectM2M(ctx, q.db.NewSelect())
err = j.selectM2M(ctx, q.db.NewSelect().Conn(q.conn))
default:
panic("not reached")
}
@ -369,6 +371,10 @@ func (q *SelectQuery) AppendQuery(fmter schema.Formatter, b []byte) (_ []byte, e
func (q *SelectQuery) appendQuery(
fmter schema.Formatter, b []byte, count bool,
) (_ []byte, err error) {
if q.err != nil {
return nil, q.err
}
fmter = formatterWithModel(fmter, q)
cteCount := count && (len(q.group) > 0 || q.distinctOn != nil)
@ -767,7 +773,7 @@ func (q *SelectQuery) Count(ctx context.Context) (int, error) {
}
query := internal.String(queryBytes)
ctx, event := q.db.beforeQuery(ctx, qq, query, nil, q.model)
ctx, event := q.db.beforeQuery(ctx, qq, query, nil, query, q.model)
var num int
err = q.conn.QueryRowContext(ctx, query).Scan(&num)
@ -778,6 +784,13 @@ func (q *SelectQuery) Count(ctx context.Context) (int, error) {
}
func (q *SelectQuery) ScanAndCount(ctx context.Context, dest ...interface{}) (int, error) {
if _, ok := q.conn.(*DB); ok {
return q.scanAndCountConc(ctx, dest...)
}
return q.scanAndCountSeq(ctx, dest...)
}
func (q *SelectQuery) scanAndCountConc(ctx context.Context, dest ...interface{}) (int, error) {
var count int
var wg sync.WaitGroup
var mu sync.Mutex
@ -817,6 +830,21 @@ func (q *SelectQuery) ScanAndCount(ctx context.Context, dest ...interface{}) (in
return count, firstErr
}
func (q *SelectQuery) scanAndCountSeq(ctx context.Context, dest ...interface{}) (int, error) {
var firstErr error
if q.limit >= 0 {
firstErr = q.Scan(ctx, dest...)
}
count, err := q.Count(ctx)
if err != nil && firstErr == nil {
firstErr = err
}
return count, firstErr
}
func (q *SelectQuery) Exists(ctx context.Context) (bool, error) {
if q.err != nil {
return false, q.err
@ -830,7 +858,7 @@ func (q *SelectQuery) Exists(ctx context.Context) (bool, error) {
}
query := internal.String(queryBytes)
ctx, event := q.db.beforeQuery(ctx, qq, query, nil, q.model)
ctx, event := q.db.beforeQuery(ctx, qq, query, nil, query, q.model)
var exists bool
err = q.conn.QueryRowContext(ctx, query).Scan(&exists)

4
vendor/github.com/uptrace/bun/query_table_create.go generated vendored
View file

@ -24,6 +24,8 @@ type CreateTableQuery struct {
tablespace schema.QueryWithArgs
}
var _ Query = (*CreateTableQuery)(nil)
func NewCreateTableQuery(db *DB) *CreateTableQuery {
q := &CreateTableQuery{
baseQuery: baseQuery{
@ -59,7 +61,7 @@ func (q *CreateTableQuery) TableExpr(query string, args ...interface{}) *CreateT
}
func (q *CreateTableQuery) ModelTableExpr(query string, args ...interface{}) *CreateTableQuery {
q.modelTable = schema.SafeQuery(query, args)
q.modelTableName = schema.SafeQuery(query, args)
return q
}

9
vendor/github.com/uptrace/bun/query_table_drop.go generated vendored
View file

@ -15,6 +15,8 @@ type DropTableQuery struct {
ifExists bool
}
var _ Query = (*DropTableQuery)(nil)
func NewDropTableQuery(db *DB) *DropTableQuery {
q := &DropTableQuery{
baseQuery: baseQuery{
@ -50,7 +52,7 @@ func (q *DropTableQuery) TableExpr(query string, args ...interface{}) *DropTable
}
func (q *DropTableQuery) ModelTableExpr(query string, args ...interface{}) *DropTableQuery {
q.modelTable = schema.SafeQuery(query, args)
q.modelTableName = schema.SafeQuery(query, args)
return q
}
@ -61,6 +63,11 @@ func (q *DropTableQuery) IfExists() *DropTableQuery {
return q
}
func (q *DropTableQuery) Cascade() *DropTableQuery {
q.cascade = true
return q
}
func (q *DropTableQuery) Restrict() *DropTableQuery {
q.restrict = true
return q

7
vendor/github.com/uptrace/bun/query_table_truncate.go generated vendored
View file

@ -16,6 +16,8 @@ type TruncateTableQuery struct {
continueIdentity bool
}
var _ Query = (*TruncateTableQuery)(nil)
func NewTruncateTableQuery(db *DB) *TruncateTableQuery {
q := &TruncateTableQuery{
baseQuery: baseQuery{
@ -57,6 +59,11 @@ func (q *TruncateTableQuery) ContinueIdentity() *TruncateTableQuery {
return q
}
func (q *TruncateTableQuery) Cascade() *TruncateTableQuery {
q.cascade = true
return q
}
func (q *TruncateTableQuery) Restrict() *TruncateTableQuery {
q.restrict = true
return q

8
vendor/github.com/uptrace/bun/query_update.go generated vendored
View file

@ -20,6 +20,8 @@ type UpdateQuery struct {
omitZero bool
}
var _ Query = (*UpdateQuery)(nil)
func NewUpdateQuery(db *DB) *UpdateQuery {
q := &UpdateQuery{
whereBaseQuery: whereBaseQuery{
@ -67,7 +69,7 @@ func (q *UpdateQuery) TableExpr(query string, args ...interface{}) *UpdateQuery
}
func (q *UpdateQuery) ModelTableExpr(query string, args ...interface{}) *UpdateQuery {
q.modelTable = schema.SafeQuery(query, args)
q.modelTableName = schema.SafeQuery(query, args)
return q
}
@ -170,6 +172,10 @@ func (q *UpdateQuery) Operation() string {
}
func (q *UpdateQuery) AppendQuery(fmter schema.Formatter, b []byte) (_ []byte, err error) {
if q.err != nil {
return nil, q.err
}
fmter = formatterWithModel(fmter, q)
b, err = q.appendWith(fmter, b)

5
vendor/github.com/uptrace/bun/query_values.go generated vendored
View file

@ -16,7 +16,10 @@ type ValuesQuery struct {
withOrder bool
}
var _ schema.NamedArgAppender = (*ValuesQuery)(nil)
var (
_ Query = (*ValuesQuery)(nil)
_ schema.NamedArgAppender = (*ValuesQuery)(nil)
)
func NewValuesQuery(db *DB, model interface{}) *ValuesQuery {
q := &ValuesQuery{

6
vendor/github.com/uptrace/bun/schema/formatter.go generated vendored
View file

@ -177,8 +177,6 @@ type NamedArgAppender interface {
AppendNamedArg(fmter Formatter, b []byte, name string) ([]byte, bool)
}
//------------------------------------------------------------------------------
type namedArgList struct {
arg NamedArgAppender
next *namedArgList
@ -219,13 +217,13 @@ func (a *namedArg) AppendNamedArg(fmter Formatter, b []byte, name string) ([]byt
//------------------------------------------------------------------------------
var _ NamedArgAppender = (*structArgs)(nil)
type structArgs struct {
table *Table
strct reflect.Value
}
var _ NamedArgAppender = (*structArgs)(nil)
func newStructArgs(fmter Formatter, strct interface{}) (*structArgs, bool) {
v := reflect.ValueOf(strct)
if !v.IsValid() {

48
vendor/github.com/uptrace/bun/schema/table.go generated vendored
View file

@ -203,7 +203,7 @@ func (t *Table) fieldByGoName(name string) *Field {
func (t *Table) initFields() {
t.Fields = make([]*Field, 0, t.Type.NumField())
t.FieldMap = make(map[string]*Field, t.Type.NumField())
t.addFields(t.Type, nil)
t.addFields(t.Type, "", nil)
if len(t.PKs) == 0 {
for _, name := range []string{"id", "uuid", "pk_" + t.ModelName} {
@ -230,7 +230,7 @@ func (t *Table) initFields() {
}
}
func (t *Table) addFields(typ reflect.Type, baseIndex []int) {
func (t *Table) addFields(typ reflect.Type, prefix string, index []int) {
for i := 0; i < typ.NumField(); i++ {
f := typ.Field(i)
unexported := f.PkgPath != ""
@ -242,10 +242,6 @@ func (t *Table) addFields(typ reflect.Type, baseIndex []int) {
continue
}
// Make a copy so the slice is not shared between fields.
index := make([]int, len(baseIndex))
copy(index, baseIndex)
if f.Anonymous {
if f.Name == "BaseModel" && f.Type == baseModelType {
if len(index) == 0 {
@ -258,7 +254,7 @@ func (t *Table) addFields(typ reflect.Type, baseIndex []int) {
if fieldType.Kind() != reflect.Struct {
continue
}
t.addFields(fieldType, append(index, f.Index...))
t.addFields(fieldType, "", withIndex(index, f.Index))
tag := tagparser.Parse(f.Tag.Get("bun"))
if _, inherit := tag.Options["inherit"]; inherit {
@ -274,7 +270,7 @@ func (t *Table) addFields(typ reflect.Type, baseIndex []int) {
continue
}
if field := t.newField(f, index); field != nil {
if field := t.newField(f, prefix, index); field != nil {
t.addField(field)
}
}
@ -315,10 +311,20 @@ func (t *Table) processBaseModelField(f reflect.StructField) {
}
//nolint
func (t *Table) newField(f reflect.StructField, index []int) *Field {
sqlName := internal.Underscore(f.Name)
func (t *Table) newField(f reflect.StructField, prefix string, index []int) *Field {
tag := tagparser.Parse(f.Tag.Get("bun"))
if prefix, ok := tag.Option("embed"); ok {
fieldType := indirectType(f.Type)
if fieldType.Kind() != reflect.Struct {
panic(fmt.Errorf("bun: embed %s.%s: got %s, wanted reflect.Struct",
t.TypeName, f.Name, fieldType.Kind()))
}
t.addFields(fieldType, prefix, withIndex(index, f.Index))
return nil
}
sqlName := internal.Underscore(f.Name)
if tag.Name != "" && tag.Name != sqlName {
if isKnownFieldOption(tag.Name) {
internal.Warn.Printf(
@ -328,10 +334,10 @@ func (t *Table) newField(f reflect.StructField, index []int) *Field {
}
sqlName = tag.Name
}
if s, ok := tag.Option("column"); ok {
sqlName = s
}
sqlName = prefix + sqlName
for name := range tag.Options {
if !isKnownFieldOption(name) {
@ -339,7 +345,7 @@ func (t *Table) newField(f reflect.StructField, index []int) *Field {
}
}
index = append(index, f.Index...)
index = withIndex(index, f.Index)
if field := t.fieldWithLock(sqlName); field != nil {
if indexEqual(field.Index, index) {
return field
@ -795,7 +801,7 @@ func (t *Table) inlineFields(field *Field, seen map[reflect.Type]struct{}) {
f.GoName = field.GoName + "_" + f.GoName
f.Name = field.Name + "__" + f.Name
f.SQLName = t.quoteIdent(f.Name)
f.Index = appendNew(field.Index, f.Index...)
f.Index = withIndex(field.Index, f.Index)
t.fieldsMapMu.Lock()
if _, ok := t.FieldMap[f.Name]; !ok {
@ -834,7 +840,7 @@ func (t *Table) AppendNamedArg(
fmter Formatter, b []byte, name string, strct reflect.Value,
) ([]byte, bool) {
if field, ok := t.FieldMap[name]; ok {
return fmter.appendArg(b, field.Value(strct).Interface()), true
return field.AppendValue(fmter, b, strct), true
}
return b, false
}
@ -853,13 +859,6 @@ func (t *Table) quoteIdent(s string) Safe {
return Safe(NewFormatter(t.dialect).AppendIdent(nil, s))
}
func appendNew(dst []int, src ...int) []int {
cp := make([]int, len(dst)+len(src))
copy(cp, dst)
copy(cp[len(dst):], src)
return cp
}
func isKnownTableOption(name string) bool {
switch name {
case "table", "alias", "select":
@ -991,3 +990,10 @@ func softDeleteFieldUpdaterFallback(field *Field) func(fv reflect.Value, tm time
return field.ScanWithCheck(fv, tm)
}
}
func withIndex(a, b []int) []int {
dest := make([]int, 0, len(a)+len(b))
dest = append(dest, a...)
dest = append(dest, b...)
return dest
}

2
vendor/github.com/uptrace/bun/version.go generated vendored
View file

@ -2,5 +2,5 @@ package bun
// Version is the current release version.
func Version() string {
return "1.0.19"
return "1.0.20"
}

2
vendor/golang.org/x/sys/unix/mkerrors.sh generated vendored
View file

@ -261,6 +261,7 @@ struct ltchars {
#include <linux/vm_sockets.h>
#include <linux/wait.h>
#include <linux/watchdog.h>
#include <linux/wireguard.h>
#include <mtd/ubi-user.h>
#include <mtd/mtd-user.h>
@ -606,6 +607,7 @@ ccflags="$@"
$2 ~ /^MTD/ ||
$2 ~ /^OTP/ ||
$2 ~ /^MEM/ ||
$2 ~ /^WG/ ||
$2 ~ /^BLK[A-Z]*(GET$|SET$|BUF$|PART$|SIZE)/ {printf("\t%s = C.%s\n", $2, $2)}
$2 ~ /^__WCOREFLAG$/ {next}
$2 ~ /^__W[A-Z0-9]+$/ {printf("\t%s = C.%s\n", substr($2,3), $2)}

30
vendor/golang.org/x/sys/unix/zerrors_linux.go generated vendored
View file

@ -741,6 +741,7 @@ const (
ETH_P_QINQ2 = 0x9200
ETH_P_QINQ3 = 0x9300
ETH_P_RARP = 0x8035
ETH_P_REALTEK = 0x8899
ETH_P_SCA = 0x6007
ETH_P_SLOW = 0x8809
ETH_P_SNAP = 0x5
@ -810,10 +811,12 @@ const (
FAN_EPIDFD = -0x2
FAN_EVENT_INFO_TYPE_DFID = 0x3
FAN_EVENT_INFO_TYPE_DFID_NAME = 0x2
FAN_EVENT_INFO_TYPE_ERROR = 0x5
FAN_EVENT_INFO_TYPE_FID = 0x1
FAN_EVENT_INFO_TYPE_PIDFD = 0x4
FAN_EVENT_METADATA_LEN = 0x18
FAN_EVENT_ON_CHILD = 0x8000000
FAN_FS_ERROR = 0x8000
FAN_MARK_ADD = 0x1
FAN_MARK_DONT_FOLLOW = 0x4
FAN_MARK_FILESYSTEM = 0x100
@ -1827,6 +1830,8 @@ const (
PERF_MEM_BLK_DATA = 0x2
PERF_MEM_BLK_NA = 0x1
PERF_MEM_BLK_SHIFT = 0x28
PERF_MEM_HOPS_0 = 0x1
PERF_MEM_HOPS_SHIFT = 0x2b
PERF_MEM_LOCK_LOCKED = 0x2
PERF_MEM_LOCK_NA = 0x1
PERF_MEM_LOCK_SHIFT = 0x18
@ -1986,6 +1991,9 @@ const (
PR_SCHED_CORE_CREATE = 0x1
PR_SCHED_CORE_GET = 0x0
PR_SCHED_CORE_MAX = 0x4
PR_SCHED_CORE_SCOPE_PROCESS_GROUP = 0x2
PR_SCHED_CORE_SCOPE_THREAD = 0x0
PR_SCHED_CORE_SCOPE_THREAD_GROUP = 0x1
PR_SCHED_CORE_SHARE_FROM = 0x3
PR_SCHED_CORE_SHARE_TO = 0x2
PR_SET_CHILD_SUBREAPER = 0x24
@ -2167,12 +2175,23 @@ const (
RTCF_NAT = 0x800000
RTCF_VALVE = 0x200000
RTC_AF = 0x20
RTC_BSM_DIRECT = 0x1
RTC_BSM_DISABLED = 0x0
RTC_BSM_LEVEL = 0x2
RTC_BSM_STANDBY = 0x3
RTC_FEATURE_ALARM = 0x0
RTC_FEATURE_ALARM_RES_2S = 0x3
RTC_FEATURE_ALARM_RES_MINUTE = 0x1
RTC_FEATURE_CNT = 0x3
RTC_FEATURE_BACKUP_SWITCH_MODE = 0x6
RTC_FEATURE_CNT = 0x7
RTC_FEATURE_CORRECTION = 0x5
RTC_FEATURE_NEED_WEEK_DAY = 0x2
RTC_FEATURE_UPDATE_INTERRUPT = 0x4
RTC_IRQF = 0x80
RTC_MAX_FREQ = 0x2000
RTC_PARAM_BACKUP_SWITCH_MODE = 0x2
RTC_PARAM_CORRECTION = 0x1
RTC_PARAM_FEATURES = 0x0
RTC_PF = 0x40
RTC_UF = 0x10
RTF_ADDRCLASSMASK = 0xf8000000
@ -2532,6 +2551,8 @@ const (
SO_VM_SOCKETS_BUFFER_MIN_SIZE = 0x1
SO_VM_SOCKETS_BUFFER_SIZE = 0x0
SO_VM_SOCKETS_CONNECT_TIMEOUT = 0x6
SO_VM_SOCKETS_CONNECT_TIMEOUT_NEW = 0x8
SO_VM_SOCKETS_CONNECT_TIMEOUT_OLD = 0x6
SO_VM_SOCKETS_NONBLOCK_TXRX = 0x7
SO_VM_SOCKETS_PEER_HOST_VM_ID = 0x3
SO_VM_SOCKETS_TRUSTED = 0x5
@ -2826,6 +2847,13 @@ const (
WDIOS_TEMPPANIC = 0x4
WDIOS_UNKNOWN = -0x1
WEXITED = 0x4
WGALLOWEDIP_A_MAX = 0x3
WGDEVICE_A_MAX = 0x8
WGPEER_A_MAX = 0xa
WG_CMD_MAX = 0x1
WG_GENL_NAME = "wireguard"
WG_GENL_VERSION = 0x1
WG_KEY_LEN = 0x20
WIN_ACKMEDIACHANGE = 0xdb
WIN_CHECKPOWERMODE1 = 0xe5
WIN_CHECKPOWERMODE2 = 0x98

3
vendor/golang.org/x/sys/unix/zerrors_linux_386.go generated vendored
View file

@ -250,6 +250,8 @@ const (
RTC_EPOCH_SET = 0x4004700e
RTC_IRQP_READ = 0x8004700b
RTC_IRQP_SET = 0x4004700c
RTC_PARAM_GET = 0x40187013
RTC_PARAM_SET = 0x40187014
RTC_PIE_OFF = 0x7006
RTC_PIE_ON = 0x7005
RTC_PLL_GET = 0x801c7011
@ -327,6 +329,7 @@ const (
SO_RCVTIMEO = 0x14
SO_RCVTIMEO_NEW = 0x42
SO_RCVTIMEO_OLD = 0x14
SO_RESERVE_MEM = 0x49
SO_REUSEADDR = 0x2
SO_REUSEPORT = 0xf
SO_RXQ_OVFL = 0x28

3
vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go generated vendored
View file

@ -251,6 +251,8 @@ const (
RTC_EPOCH_SET = 0x4008700e
RTC_IRQP_READ = 0x8008700b
RTC_IRQP_SET = 0x4008700c
RTC_PARAM_GET = 0x40187013
RTC_PARAM_SET = 0x40187014
RTC_PIE_OFF = 0x7006
RTC_PIE_ON = 0x7005
RTC_PLL_GET = 0x80207011
@ -328,6 +330,7 @@ const (
SO_RCVTIMEO = 0x14
SO_RCVTIMEO_NEW = 0x42
SO_RCVTIMEO_OLD = 0x14
SO_RESERVE_MEM = 0x49
SO_REUSEADDR = 0x2
SO_REUSEPORT = 0xf
SO_RXQ_OVFL = 0x28

3
vendor/golang.org/x/sys/unix/zerrors_linux_arm.go generated vendored
View file

@ -257,6 +257,8 @@ const (
RTC_EPOCH_SET = 0x4004700e
RTC_IRQP_READ = 0x8004700b
RTC_IRQP_SET = 0x4004700c
RTC_PARAM_GET = 0x40187013
RTC_PARAM_SET = 0x40187014
RTC_PIE_OFF = 0x7006
RTC_PIE_ON = 0x7005
RTC_PLL_GET = 0x801c7011
@ -334,6 +336,7 @@ const (
SO_RCVTIMEO = 0x14
SO_RCVTIMEO_NEW = 0x42
SO_RCVTIMEO_OLD = 0x14
SO_RESERVE_MEM = 0x49
SO_REUSEADDR = 0x2
SO_REUSEPORT = 0xf
SO_RXQ_OVFL = 0x28

3
vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go generated vendored
View file

@ -247,6 +247,8 @@ const (
RTC_EPOCH_SET = 0x4008700e
RTC_IRQP_READ = 0x8008700b
RTC_IRQP_SET = 0x4008700c
RTC_PARAM_GET = 0x40187013
RTC_PARAM_SET = 0x40187014
RTC_PIE_OFF = 0x7006
RTC_PIE_ON = 0x7005
RTC_PLL_GET = 0x80207011
@ -324,6 +326,7 @@ const (
SO_RCVTIMEO = 0x14
SO_RCVTIMEO_NEW = 0x42
SO_RCVTIMEO_OLD = 0x14
SO_RESERVE_MEM = 0x49
SO_REUSEADDR = 0x2
SO_REUSEPORT = 0xf
SO_RXQ_OVFL = 0x28

3
vendor/golang.org/x/sys/unix/zerrors_linux_mips.go generated vendored
View file

@ -250,6 +250,8 @@ const (
RTC_EPOCH_SET = 0x8004700e
RTC_IRQP_READ = 0x4004700b
RTC_IRQP_SET = 0x8004700c
RTC_PARAM_GET = 0x80187013
RTC_PARAM_SET = 0x80187014
RTC_PIE_OFF = 0x20007006
RTC_PIE_ON = 0x20007005
RTC_PLL_GET = 0x401c7011
@ -327,6 +329,7 @@ const (
SO_RCVTIMEO = 0x1006
SO_RCVTIMEO_NEW = 0x42
SO_RCVTIMEO_OLD = 0x1006
SO_RESERVE_MEM = 0x49
SO_REUSEADDR = 0x4
SO_REUSEPORT = 0x200
SO_RXQ_OVFL = 0x28

3
vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go generated vendored
View file

@ -250,6 +250,8 @@ const (
RTC_EPOCH_SET = 0x8008700e
RTC_IRQP_READ = 0x4008700b
RTC_IRQP_SET = 0x8008700c
RTC_PARAM_GET = 0x80187013
RTC_PARAM_SET = 0x80187014
RTC_PIE_OFF = 0x20007006
RTC_PIE_ON = 0x20007005
RTC_PLL_GET = 0x40207011
@ -327,6 +329,7 @@ const (
SO_RCVTIMEO = 0x1006
SO_RCVTIMEO_NEW = 0x42
SO_RCVTIMEO_OLD = 0x1006
SO_RESERVE_MEM = 0x49
SO_REUSEADDR = 0x4
SO_REUSEPORT = 0x200
SO_RXQ_OVFL = 0x28

3
vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go generated vendored
View file

@ -250,6 +250,8 @@ const (
RTC_EPOCH_SET = 0x8008700e
RTC_IRQP_READ = 0x4008700b
RTC_IRQP_SET = 0x8008700c
RTC_PARAM_GET = 0x80187013
RTC_PARAM_SET = 0x80187014
RTC_PIE_OFF = 0x20007006
RTC_PIE_ON = 0x20007005
RTC_PLL_GET = 0x40207011
@ -327,6 +329,7 @@ const (
SO_RCVTIMEO = 0x1006
SO_RCVTIMEO_NEW = 0x42
SO_RCVTIMEO_OLD = 0x1006
SO_RESERVE_MEM = 0x49
SO_REUSEADDR = 0x4
SO_REUSEPORT = 0x200
SO_RXQ_OVFL = 0x28

3
vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go generated vendored
View file

@ -250,6 +250,8 @@ const (
RTC_EPOCH_SET = 0x8004700e
RTC_IRQP_READ = 0x4004700b
RTC_IRQP_SET = 0x8004700c
RTC_PARAM_GET = 0x80187013
RTC_PARAM_SET = 0x80187014
RTC_PIE_OFF = 0x20007006
RTC_PIE_ON = 0x20007005
RTC_PLL_GET = 0x401c7011
@ -327,6 +329,7 @@ const (
SO_RCVTIMEO = 0x1006
SO_RCVTIMEO_NEW = 0x42
SO_RCVTIMEO_OLD = 0x1006
SO_RESERVE_MEM = 0x49
SO_REUSEADDR = 0x4
SO_REUSEPORT = 0x200
SO_RXQ_OVFL = 0x28

3
vendor/golang.org/x/sys/unix/zerrors_linux_ppc.go generated vendored
View file

@ -305,6 +305,8 @@ const (
RTC_EPOCH_SET = 0x8004700e
RTC_IRQP_READ = 0x4004700b
RTC_IRQP_SET = 0x8004700c
RTC_PARAM_GET = 0x80187013
RTC_PARAM_SET = 0x80187014
RTC_PIE_OFF = 0x20007006
RTC_PIE_ON = 0x20007005
RTC_PLL_GET = 0x401c7011
@ -382,6 +384,7 @@ const (
SO_RCVTIMEO = 0x12
SO_RCVTIMEO_NEW = 0x42
SO_RCVTIMEO_OLD = 0x12
SO_RESERVE_MEM = 0x49
SO_REUSEADDR = 0x2
SO_REUSEPORT = 0xf
SO_RXQ_OVFL = 0x28

3
vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go generated vendored
View file

@ -309,6 +309,8 @@ const (
RTC_EPOCH_SET = 0x8008700e
RTC_IRQP_READ = 0x4008700b
RTC_IRQP_SET = 0x8008700c
RTC_PARAM_GET = 0x80187013
RTC_PARAM_SET = 0x80187014
RTC_PIE_OFF = 0x20007006
RTC_PIE_ON = 0x20007005
RTC_PLL_GET = 0x40207011
@ -386,6 +388,7 @@ const (
SO_RCVTIMEO = 0x12
SO_RCVTIMEO_NEW = 0x42
SO_RCVTIMEO_OLD = 0x12
SO_RESERVE_MEM = 0x49
SO_REUSEADDR = 0x2
SO_REUSEPORT = 0xf
SO_RXQ_OVFL = 0x28

3
vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go generated vendored
View file

@ -309,6 +309,8 @@ const (
RTC_EPOCH_SET = 0x8008700e
RTC_IRQP_READ = 0x4008700b
RTC_IRQP_SET = 0x8008700c
RTC_PARAM_GET = 0x80187013
RTC_PARAM_SET = 0x80187014
RTC_PIE_OFF = 0x20007006
RTC_PIE_ON = 0x20007005
RTC_PLL_GET = 0x40207011
@ -386,6 +388,7 @@ const (
SO_RCVTIMEO = 0x12
SO_RCVTIMEO_NEW = 0x42
SO_RCVTIMEO_OLD = 0x12
SO_RESERVE_MEM = 0x49
SO_REUSEADDR = 0x2
SO_REUSEPORT = 0xf
SO_RXQ_OVFL = 0x28

Some files were not shown because too many files have changed in this diff Show more