diff --git a/README.md b/README.md index dfb43fb89..37ff31f24 100644 --- a/README.md +++ b/README.md @@ -219,6 +219,8 @@ The following libraries and frameworks are used by GoToSocial, with gratitude - [go-playground/validator](https://github.com/go-playground/validator); struct validation. [MIT License](https://spdx.org/licenses/MIT.html). - [gorilla/websocket](https://github.com/gorilla/websocket); Websocket connectivity. [BSD-2-Clause License](https://spdx.org/licenses/BSD-2-Clause.html). - [gruf/go-debug](https://codeberg.org/gruf/go-debug); profiling support in debug builds. [MIT License](https://spdx.org/licenses/MIT.html). +- [gruf/go-bytesize](https://codeberg.org/gruf/go-bytesize); byte size parsing / formatting. [MIT License](https://spdx.org/licenses/MIT.html). +- [gruf/go-cache](https://codeberg.org/gruf/go-cache); object caching. [MIT License](https://spdx.org/licenses/MIT.html). - [gruf/go-kv](https://codeberg.org/gruf/go-kv); key-value field formatting. [MIT License](https://spdx.org/licenses/MIT.html). - [gruf/go-mutexes](https://codeberg.org/gruf/go-mutexes); mutex map. [MIT License](https://spdx.org/licenses/MIT.html). - [gruf/go-runners](https://codeberg.org/gruf/go-runners); worker pool library. [MIT License](https://spdx.org/licenses/MIT.html). diff --git a/go.mod b/go.mod index a4baa9fe8..1b68e9d06 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.19 require ( codeberg.org/gruf/go-atomics v1.1.0 - codeberg.org/gruf/go-bytesize v0.2.1 + codeberg.org/gruf/go-bytesize v1.0.0 codeberg.org/gruf/go-byteutil v1.0.2 codeberg.org/gruf/go-cache/v2 v2.1.4 codeberg.org/gruf/go-debug v1.2.0 diff --git a/go.sum b/go.sum index 0e669f337..4ec6e919e 100644 --- a/go.sum +++ b/go.sum @@ -64,8 +64,8 @@ codeberg.org/gruf/go-bitutil v1.0.1/go.mod h1:3ezHnADoiRJs9jgn65AEZ3HY7dsabAYLmm codeberg.org/gruf/go-bytes v1.0.0/go.mod h1:1v/ibfaosfXSZtRdW2rWaVrDXMc9E3bsi/M9Ekx39cg= codeberg.org/gruf/go-bytes v1.0.2 h1:malqE42Ni+h1nnYWBUAJaDDtEzF4aeN4uPN8DfMNNvo= codeberg.org/gruf/go-bytes v1.0.2/go.mod h1:1v/ibfaosfXSZtRdW2rWaVrDXMc9E3bsi/M9Ekx39cg= -codeberg.org/gruf/go-bytesize v0.2.1 h1:nbAta3FCYe3Q18osqg8Ylk/naOopdqEKiKMpo6KTpAA= -codeberg.org/gruf/go-bytesize v0.2.1/go.mod h1:n/GU8HzL9f3UNp/mUKyr1qVmTlj7+xacpp0OHfkvLPs= +codeberg.org/gruf/go-bytesize v1.0.0 h1:/Mcv4prniJLkPEqZ+LZ5/D/e27rNrZZEMmty9jpIvlc= +codeberg.org/gruf/go-bytesize v1.0.0/go.mod h1:n/GU8HzL9f3UNp/mUKyr1qVmTlj7+xacpp0OHfkvLPs= codeberg.org/gruf/go-byteutil v1.0.0/go.mod h1:cWM3tgMCroSzqoBXUXMhvxTxYJp+TbCr6ioISRY5vSU= codeberg.org/gruf/go-byteutil v1.0.2 h1:OesVyK5VKWeWdeDR00zRJ+Oy8hjXx1pBhn7WVvcZWVE= codeberg.org/gruf/go-byteutil v1.0.2/go.mod h1:cWM3tgMCroSzqoBXUXMhvxTxYJp+TbCr6ioISRY5vSU= diff --git a/internal/config/config.go b/internal/config/config.go index ff77bd367..79babf8bc 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -21,6 +21,7 @@ package config import ( "reflect" + "codeberg.org/gruf/go-bytesize" "github.com/mitchellh/mapstructure" ) @@ -76,13 +77,13 @@ type Configuration struct { AccountsReasonRequired bool `name:"accounts-reason-required" usage:"Do new account signups require a reason to be submitted on registration?"` AccountsAllowCustomCSS bool `name:"accounts-allow-custom-css" usage:"Allow accounts to enable custom CSS for their profile pages and statuses."` - MediaImageMaxSize int `name:"media-image-max-size" usage:"Max size of accepted images in bytes"` - MediaVideoMaxSize int `name:"media-video-max-size" usage:"Max size of accepted videos in bytes"` - MediaDescriptionMinChars int `name:"media-description-min-chars" usage:"Min required chars for an image description"` - MediaDescriptionMaxChars int `name:"media-description-max-chars" usage:"Max permitted chars for an image description"` - MediaRemoteCacheDays int `name:"media-remote-cache-days" usage:"Number of days to locally cache media from remote instances. If set to 0, remote media will be kept indefinitely."` - MediaEmojiLocalMaxSize int `name:"media-emoji-local-max-size" usage:"Max size in bytes of emojis uploaded to this instance via the admin API."` - MediaEmojiRemoteMaxSize int `name:"media-emoji-remote-max-size" usage:"Max size in bytes of emojis to download from other instances."` + MediaImageMaxSize bytesize.Size `name:"media-image-max-size" usage:"Max size of accepted images in bytes"` + MediaVideoMaxSize bytesize.Size `name:"media-video-max-size" usage:"Max size of accepted videos in bytes"` + MediaDescriptionMinChars int `name:"media-description-min-chars" usage:"Min required chars for an image description"` + MediaDescriptionMaxChars int `name:"media-description-max-chars" usage:"Max permitted chars for an image description"` + MediaRemoteCacheDays int `name:"media-remote-cache-days" usage:"Number of days to locally cache media from remote instances. If set to 0, remote media will be kept indefinitely."` + MediaEmojiLocalMaxSize bytesize.Size `name:"media-emoji-local-max-size" usage:"Max size in bytes of emojis uploaded to this instance via the admin API."` + MediaEmojiRemoteMaxSize bytesize.Size `name:"media-emoji-remote-max-size" usage:"Max size in bytes of emojis to download from other instances."` StorageBackend string `name:"storage-backend" usage:"Storage backend to use for media attachments"` StorageLocalBasePath string `name:"storage-local-base-path" usage:"Full path to an already-created directory where gts should store/retrieve media files. Subfolders will be created within this dir."` diff --git a/internal/config/flags.go b/internal/config/flags.go index f37a50c9e..7b416815e 100644 --- a/internal/config/flags.go +++ b/internal/config/flags.go @@ -72,13 +72,13 @@ func AddServerFlags(cmd *cobra.Command) { cmd.Flags().Bool(AccountsAllowCustomCSSFlag(), cfg.AccountsAllowCustomCSS, fieldtag("AccountsAllowCustomCSS", "usage")) // Media - cmd.Flags().Int(MediaImageMaxSizeFlag(), cfg.MediaImageMaxSize, fieldtag("MediaImageMaxSize", "usage")) - cmd.Flags().Int(MediaVideoMaxSizeFlag(), cfg.MediaVideoMaxSize, fieldtag("MediaVideoMaxSize", "usage")) + cmd.Flags().Uint64(MediaImageMaxSizeFlag(), uint64(cfg.MediaImageMaxSize), fieldtag("MediaImageMaxSize", "usage")) + cmd.Flags().Uint64(MediaVideoMaxSizeFlag(), uint64(cfg.MediaVideoMaxSize), fieldtag("MediaVideoMaxSize", "usage")) cmd.Flags().Int(MediaDescriptionMinCharsFlag(), cfg.MediaDescriptionMinChars, fieldtag("MediaDescriptionMinChars", "usage")) cmd.Flags().Int(MediaDescriptionMaxCharsFlag(), cfg.MediaDescriptionMaxChars, fieldtag("MediaDescriptionMaxChars", "usage")) cmd.Flags().Int(MediaRemoteCacheDaysFlag(), cfg.MediaRemoteCacheDays, fieldtag("MediaRemoteCacheDays", "usage")) - cmd.Flags().Int(MediaEmojiLocalMaxSizeFlag(), cfg.MediaEmojiLocalMaxSize, fieldtag("MediaEmojiLocalMaxSize", "usage")) - cmd.Flags().Int(MediaEmojiRemoteMaxSizeFlag(), cfg.MediaEmojiRemoteMaxSize, fieldtag("MediaEmojiRemoteMaxSize", "usage")) + cmd.Flags().Uint64(MediaEmojiLocalMaxSizeFlag(), uint64(cfg.MediaEmojiLocalMaxSize), fieldtag("MediaEmojiLocalMaxSize", "usage")) + cmd.Flags().Uint64(MediaEmojiRemoteMaxSizeFlag(), uint64(cfg.MediaEmojiRemoteMaxSize), fieldtag("MediaEmojiRemoteMaxSize", "usage")) // Storage cmd.Flags().String(StorageBackendFlag(), cfg.StorageBackend, fieldtag("StorageBackend", "usage")) diff --git a/internal/config/gen/gen.go b/internal/config/gen/gen.go index d36fee0e9..e92483939 100644 --- a/internal/config/gen/gen.go +++ b/internal/config/gen/gen.go @@ -100,7 +100,7 @@ func main() { fmt.Fprintf(output, "func Set%[1]s(v %[2]s) { global.Set%[1]s(v) }\n\n", field.Name, field.Type.String()) } _ = output.Close() - _ = exec.Command("gofmt", "-w", out).Run() + _ = exec.Command("gofumports", "-w", out).Run() // The plain here is that eventually we might be able // to generate an example configuration from struct tags diff --git a/internal/config/global.go b/internal/config/global.go index 8752fc925..707afb3cd 100644 --- a/internal/config/global.go +++ b/internal/config/global.go @@ -18,7 +18,9 @@ package config -import "github.com/spf13/cobra" +import ( + "github.com/spf13/cobra" +) var global *ConfigState diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index adc020aba..71f96920f 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -18,6 +18,8 @@ */ package config +import "codeberg.org/gruf/go-bytesize" + // GetLogLevel safely fetches the Configuration value for state's 'LogLevel' field func (st *ConfigState) GetLogLevel() (v string) { st.mutex.Lock() @@ -719,7 +721,7 @@ func GetAccountsAllowCustomCSS() bool { return global.GetAccountsAllowCustomCSS( func SetAccountsAllowCustomCSS(v bool) { global.SetAccountsAllowCustomCSS(v) } // GetMediaImageMaxSize safely fetches the Configuration value for state's 'MediaImageMaxSize' field -func (st *ConfigState) GetMediaImageMaxSize() (v int) { +func (st *ConfigState) GetMediaImageMaxSize() (v bytesize.Size) { st.mutex.Lock() v = st.config.MediaImageMaxSize st.mutex.Unlock() @@ -727,7 +729,7 @@ func (st *ConfigState) GetMediaImageMaxSize() (v int) { } // SetMediaImageMaxSize safely sets the Configuration value for state's 'MediaImageMaxSize' field -func (st *ConfigState) SetMediaImageMaxSize(v int) { +func (st *ConfigState) SetMediaImageMaxSize(v bytesize.Size) { st.mutex.Lock() defer st.mutex.Unlock() st.config.MediaImageMaxSize = v @@ -738,13 +740,13 @@ func (st *ConfigState) SetMediaImageMaxSize(v int) { func MediaImageMaxSizeFlag() string { return "media-image-max-size" } // GetMediaImageMaxSize safely fetches the value for global configuration 'MediaImageMaxSize' field -func GetMediaImageMaxSize() int { return global.GetMediaImageMaxSize() } +func GetMediaImageMaxSize() bytesize.Size { return global.GetMediaImageMaxSize() } // SetMediaImageMaxSize safely sets the value for global configuration 'MediaImageMaxSize' field -func SetMediaImageMaxSize(v int) { global.SetMediaImageMaxSize(v) } +func SetMediaImageMaxSize(v bytesize.Size) { global.SetMediaImageMaxSize(v) } // GetMediaVideoMaxSize safely fetches the Configuration value for state's 'MediaVideoMaxSize' field -func (st *ConfigState) GetMediaVideoMaxSize() (v int) { +func (st *ConfigState) GetMediaVideoMaxSize() (v bytesize.Size) { st.mutex.Lock() v = st.config.MediaVideoMaxSize st.mutex.Unlock() @@ -752,7 +754,7 @@ func (st *ConfigState) GetMediaVideoMaxSize() (v int) { } // SetMediaVideoMaxSize safely sets the Configuration value for state's 'MediaVideoMaxSize' field -func (st *ConfigState) SetMediaVideoMaxSize(v int) { +func (st *ConfigState) SetMediaVideoMaxSize(v bytesize.Size) { st.mutex.Lock() defer st.mutex.Unlock() st.config.MediaVideoMaxSize = v @@ -763,10 +765,10 @@ func (st *ConfigState) SetMediaVideoMaxSize(v int) { func MediaVideoMaxSizeFlag() string { return "media-video-max-size" } // GetMediaVideoMaxSize safely fetches the value for global configuration 'MediaVideoMaxSize' field -func GetMediaVideoMaxSize() int { return global.GetMediaVideoMaxSize() } +func GetMediaVideoMaxSize() bytesize.Size { return global.GetMediaVideoMaxSize() } // SetMediaVideoMaxSize safely sets the value for global configuration 'MediaVideoMaxSize' field -func SetMediaVideoMaxSize(v int) { global.SetMediaVideoMaxSize(v) } +func SetMediaVideoMaxSize(v bytesize.Size) { global.SetMediaVideoMaxSize(v) } // GetMediaDescriptionMinChars safely fetches the Configuration value for state's 'MediaDescriptionMinChars' field func (st *ConfigState) GetMediaDescriptionMinChars() (v int) { @@ -844,7 +846,7 @@ func GetMediaRemoteCacheDays() int { return global.GetMediaRemoteCacheDays() } func SetMediaRemoteCacheDays(v int) { global.SetMediaRemoteCacheDays(v) } // GetMediaEmojiLocalMaxSize safely fetches the Configuration value for state's 'MediaEmojiLocalMaxSize' field -func (st *ConfigState) GetMediaEmojiLocalMaxSize() (v int) { +func (st *ConfigState) GetMediaEmojiLocalMaxSize() (v bytesize.Size) { st.mutex.Lock() v = st.config.MediaEmojiLocalMaxSize st.mutex.Unlock() @@ -852,7 +854,7 @@ func (st *ConfigState) GetMediaEmojiLocalMaxSize() (v int) { } // SetMediaEmojiLocalMaxSize safely sets the Configuration value for state's 'MediaEmojiLocalMaxSize' field -func (st *ConfigState) SetMediaEmojiLocalMaxSize(v int) { +func (st *ConfigState) SetMediaEmojiLocalMaxSize(v bytesize.Size) { st.mutex.Lock() defer st.mutex.Unlock() st.config.MediaEmojiLocalMaxSize = v @@ -863,13 +865,13 @@ func (st *ConfigState) SetMediaEmojiLocalMaxSize(v int) { func MediaEmojiLocalMaxSizeFlag() string { return "media-emoji-local-max-size" } // GetMediaEmojiLocalMaxSize safely fetches the value for global configuration 'MediaEmojiLocalMaxSize' field -func GetMediaEmojiLocalMaxSize() int { return global.GetMediaEmojiLocalMaxSize() } +func GetMediaEmojiLocalMaxSize() bytesize.Size { return global.GetMediaEmojiLocalMaxSize() } // SetMediaEmojiLocalMaxSize safely sets the value for global configuration 'MediaEmojiLocalMaxSize' field -func SetMediaEmojiLocalMaxSize(v int) { global.SetMediaEmojiLocalMaxSize(v) } +func SetMediaEmojiLocalMaxSize(v bytesize.Size) { global.SetMediaEmojiLocalMaxSize(v) } // GetMediaEmojiRemoteMaxSize safely fetches the Configuration value for state's 'MediaEmojiRemoteMaxSize' field -func (st *ConfigState) GetMediaEmojiRemoteMaxSize() (v int) { +func (st *ConfigState) GetMediaEmojiRemoteMaxSize() (v bytesize.Size) { st.mutex.Lock() v = st.config.MediaEmojiRemoteMaxSize st.mutex.Unlock() @@ -877,7 +879,7 @@ func (st *ConfigState) GetMediaEmojiRemoteMaxSize() (v int) { } // SetMediaEmojiRemoteMaxSize safely sets the Configuration value for state's 'MediaEmojiRemoteMaxSize' field -func (st *ConfigState) SetMediaEmojiRemoteMaxSize(v int) { +func (st *ConfigState) SetMediaEmojiRemoteMaxSize(v bytesize.Size) { st.mutex.Lock() defer st.mutex.Unlock() st.config.MediaEmojiRemoteMaxSize = v @@ -888,10 +890,10 @@ func (st *ConfigState) SetMediaEmojiRemoteMaxSize(v int) { func MediaEmojiRemoteMaxSizeFlag() string { return "media-emoji-remote-max-size" } // GetMediaEmojiRemoteMaxSize safely fetches the value for global configuration 'MediaEmojiRemoteMaxSize' field -func GetMediaEmojiRemoteMaxSize() int { return global.GetMediaEmojiRemoteMaxSize() } +func GetMediaEmojiRemoteMaxSize() bytesize.Size { return global.GetMediaEmojiRemoteMaxSize() } // SetMediaEmojiRemoteMaxSize safely sets the value for global configuration 'MediaEmojiRemoteMaxSize' field -func SetMediaEmojiRemoteMaxSize(v int) { global.SetMediaEmojiRemoteMaxSize(v) } +func SetMediaEmojiRemoteMaxSize(v bytesize.Size) { global.SetMediaEmojiRemoteMaxSize(v) } // GetStorageBackend safely fetches the Configuration value for state's 'StorageBackend' field func (st *ConfigState) GetStorageBackend() (v string) { diff --git a/internal/config/state.go b/internal/config/state.go index 1364fb84e..17fd31e2a 100644 --- a/internal/config/state.go +++ b/internal/config/state.go @@ -133,6 +133,12 @@ func (st *ConfigState) reloadFromViper() { if err := st.viper.Unmarshal(&st.config, func(c *mapstructure.DecoderConfig) { c.TagName = "name" c.ZeroFields = true // empty the config struct before we marshal values into it + + oldhook := c.DecodeHook + c.DecodeHook = mapstructure.ComposeDecodeHookFunc( + mapstructure.TextUnmarshallerHookFunc(), + oldhook, + ) }); err != nil { panic(err) } diff --git a/internal/config/validate.go b/internal/config/validate.go index b9fdb013b..064eae07a 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -67,14 +67,6 @@ func Validate() error { errs = append(errs, fmt.Errorf("%s must be set", WebAssetBaseDirFlag())) } - if m := GetMediaEmojiLocalMaxSize(); m < 0 { - errs = append(errs, fmt.Errorf("%s must not be less than 0", MediaEmojiLocalMaxSizeFlag())) - } - - if m := GetMediaEmojiRemoteMaxSize(); m < 0 { - errs = append(errs, fmt.Errorf("%s must not be less than 0", MediaEmojiRemoteMaxSizeFlag())) - } - if len(errs) > 0 { errStrings := []string{} for _, err := range errs { diff --git a/internal/config/validate_test.go b/internal/config/validate_test.go index f7450cdaa..c3a998a4a 100644 --- a/internal/config/validate_test.go +++ b/internal/config/validate_test.go @@ -141,16 +141,6 @@ func (suite *ConfigValidateTestSuite) TestValidateConfigBadProtocolNoHost() { suite.EqualError(err, "host must be set; protocol must be set to either http or https, provided value was foo") } -func (suite *ConfigValidateTestSuite) TestValidateConfigBadEmojiSizes() { - testrig.InitTestConfig() - - config.SetMediaEmojiLocalMaxSize(-10) - config.SetMediaEmojiRemoteMaxSize(-50) - - err := config.Validate() - suite.EqualError(err, "media-emoji-local-max-size must not be less than 0; media-emoji-remote-max-size must not be less than 0") -} - func TestConfigValidateTestSuite(t *testing.T) { suite.Run(t, &ConfigValidateTestSuite{}) } diff --git a/internal/federation/dereferencing/account.go b/internal/federation/dereferencing/account.go index 41a8aa8a9..dfe7693fb 100644 --- a/internal/federation/dereferencing/account.go +++ b/internal/federation/dereferencing/account.go @@ -496,7 +496,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm } } - data := func(innerCtx context.Context) (io.Reader, int, error) { + data := func(innerCtx context.Context) (io.Reader, int64, error) { return t.DereferenceMedia(innerCtx, avatarIRI) } @@ -562,7 +562,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm } } - data := func(innerCtx context.Context) (io.Reader, int, error) { + data := func(innerCtx context.Context) (io.Reader, int64, error) { return t.DereferenceMedia(innerCtx, headerIRI) } diff --git a/internal/federation/dereferencing/emoji.go b/internal/federation/dereferencing/emoji.go index 87d0bd515..622b131c9 100644 --- a/internal/federation/dereferencing/emoji.go +++ b/internal/federation/dereferencing/emoji.go @@ -42,7 +42,7 @@ func (d *deref) GetRemoteEmoji(ctx context.Context, requestingUsername string, r return nil, fmt.Errorf("GetRemoteEmoji: error parsing url: %s", err) } - dataFunc := func(innerCtx context.Context) (io.Reader, int, error) { + dataFunc := func(innerCtx context.Context) (io.Reader, int64, error) { return t.DereferenceMedia(innerCtx, derefURI) } diff --git a/internal/federation/dereferencing/media.go b/internal/federation/dereferencing/media.go index 05fd70589..1b99eaa96 100644 --- a/internal/federation/dereferencing/media.go +++ b/internal/federation/dereferencing/media.go @@ -42,7 +42,7 @@ func (d *deref) GetRemoteMedia(ctx context.Context, requestingUsername string, a return nil, fmt.Errorf("GetRemoteMedia: error parsing url: %s", err) } - dataFunc := func(innerCtx context.Context) (io.Reader, int, error) { + dataFunc := func(innerCtx context.Context) (io.Reader, int64, error) { return t.DereferenceMedia(innerCtx, derefURI) } diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go index 7d1f59e70..f9c96259d 100644 --- a/internal/media/manager_test.go +++ b/internal/media/manager_test.go @@ -43,13 +43,13 @@ type ManagerTestSuite struct { func (suite *ManagerTestSuite) TestEmojiProcessBlocking() { ctx := context.Background() - data := func(_ context.Context) (io.Reader, int, error) { + data := func(_ context.Context) (io.Reader, int64, error) { // load bytes from a test image b, err := os.ReadFile("./test/rainbow-original.png") if err != nil { panic(err) } - return bytes.NewBuffer(b), len(b), nil + return bytes.NewBuffer(b), int64(len(b)), nil } emojiID := "01GDQ9G782X42BAMFASKP64343" @@ -104,13 +104,13 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlocking() { func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLarge() { ctx := context.Background() - data := func(_ context.Context) (io.Reader, int, error) { + data := func(_ context.Context) (io.Reader, int64, error) { // load bytes from a test image b, err := os.ReadFile("./test/big-panda.gif") if err != nil { panic(err) } - return bytes.NewBuffer(b), len(b), nil + return bytes.NewBuffer(b), int64(len(b)), nil } emojiID := "01GDQ9G782X42BAMFASKP64343" @@ -128,13 +128,13 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLarge() { func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLargeNoSizeGiven() { ctx := context.Background() - data := func(_ context.Context) (io.Reader, int, error) { + data := func(_ context.Context) (io.Reader, int64, error) { // load bytes from a test image b, err := os.ReadFile("./test/big-panda.gif") if err != nil { panic(err) } - return bytes.NewBuffer(b), len(b), nil + return bytes.NewBuffer(b), int64(len(b)), nil } emojiID := "01GDQ9G782X42BAMFASKP64343" @@ -152,7 +152,7 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlockingTooLargeNoSizeGiven() { func (suite *ManagerTestSuite) TestEmojiProcessBlockingNoFileSizeGiven() { ctx := context.Background() - data := func(_ context.Context) (io.Reader, int, error) { + data := func(_ context.Context) (io.Reader, int64, error) { // load bytes from a test image b, err := os.ReadFile("./test/rainbow-original.png") if err != nil { @@ -214,13 +214,13 @@ func (suite *ManagerTestSuite) TestEmojiProcessBlockingNoFileSizeGiven() { func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() { ctx := context.Background() - data := func(_ context.Context) (io.Reader, int, error) { + data := func(_ context.Context) (io.Reader, int64, error) { // load bytes from a test image b, err := os.ReadFile("./test/test-jpeg.jpg") if err != nil { panic(err) } - return bytes.NewBuffer(b), len(b), nil + return bytes.NewBuffer(b), int64(len(b)), nil } accountID := "01FS1X72SK9ZPW0J1QQ68BD264" @@ -286,7 +286,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() { func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingNoContentLengthGiven() { ctx := context.Background() - data := func(_ context.Context) (io.Reader, int, error) { + data := func(_ context.Context) (io.Reader, int64, error) { // load bytes from a test image b, err := os.ReadFile("./test/test-jpeg.jpg") if err != nil { @@ -359,7 +359,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingNoContentLengthGiven func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingReadCloser() { ctx := context.Background() - data := func(_ context.Context) (io.Reader, int, error) { + data := func(_ context.Context) (io.Reader, int64, error) { // open test image as a file f, err := os.Open("./test/test-jpeg.jpg") if err != nil { @@ -432,13 +432,13 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingReadCloser() { func (suite *ManagerTestSuite) TestPngNoAlphaChannelProcessBlocking() { ctx := context.Background() - data := func(_ context.Context) (io.Reader, int, error) { + data := func(_ context.Context) (io.Reader, int64, error) { // load bytes from a test image b, err := os.ReadFile("./test/test-png-noalphachannel.png") if err != nil { panic(err) } - return bytes.NewBuffer(b), len(b), nil + return bytes.NewBuffer(b), int64(len(b)), nil } accountID := "01FS1X72SK9ZPW0J1QQ68BD264" @@ -504,13 +504,13 @@ func (suite *ManagerTestSuite) TestPngNoAlphaChannelProcessBlocking() { func (suite *ManagerTestSuite) TestPngAlphaChannelProcessBlocking() { ctx := context.Background() - data := func(_ context.Context) (io.Reader, int, error) { + data := func(_ context.Context) (io.Reader, int64, error) { // load bytes from a test image b, err := os.ReadFile("./test/test-png-alphachannel.png") if err != nil { panic(err) } - return bytes.NewBuffer(b), len(b), nil + return bytes.NewBuffer(b), int64(len(b)), nil } accountID := "01FS1X72SK9ZPW0J1QQ68BD264" @@ -576,13 +576,13 @@ func (suite *ManagerTestSuite) TestPngAlphaChannelProcessBlocking() { func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithCallback() { ctx := context.Background() - data := func(_ context.Context) (io.Reader, int, error) { + data := func(_ context.Context) (io.Reader, int64, error) { // load bytes from a test image b, err := os.ReadFile("./test/test-jpeg.jpg") if err != nil { panic(err) } - return bytes.NewBuffer(b), len(b), nil + return bytes.NewBuffer(b), int64(len(b)), nil } // test the callback function by setting a simple boolean @@ -659,13 +659,13 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithCallback() { func (suite *ManagerTestSuite) TestSimpleJpegProcessAsync() { ctx := context.Background() - data := func(_ context.Context) (io.Reader, int, error) { + data := func(_ context.Context) (io.Reader, int64, error) { // load bytes from a test image b, err := os.ReadFile("./test/test-jpeg.jpg") if err != nil { panic(err) } - return bytes.NewBuffer(b), len(b), nil + return bytes.NewBuffer(b), int64(len(b)), nil } accountID := "01FS1X72SK9ZPW0J1QQ68BD264" @@ -744,9 +744,9 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() { panic(err) } - data := func(_ context.Context) (io.Reader, int, error) { + data := func(_ context.Context) (io.Reader, int64, error) { // load bytes from a test image - return bytes.NewReader(b), len(b), nil + return bytes.NewReader(b), int64(len(b)), nil } accountID := "01FS1X72SK9ZPW0J1QQ68BD264" @@ -820,13 +820,13 @@ func (suite *ManagerTestSuite) TestSimpleJpegQueueSpamming() { func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithDiskStorage() { ctx := context.Background() - data := func(_ context.Context) (io.Reader, int, error) { + data := func(_ context.Context) (io.Reader, int64, error) { // load bytes from a test image b, err := os.ReadFile("./test/test-jpeg.jpg") if err != nil { panic(err) } - return bytes.NewBuffer(b), len(b), nil + return bytes.NewBuffer(b), int64(len(b)), nil } accountID := "01FS1X72SK9ZPW0J1QQ68BD264" diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go index 7dfe51cb3..6495b991e 100644 --- a/internal/media/processingemoji.go +++ b/internal/media/processingemoji.go @@ -210,11 +210,11 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { // concatenate the first bytes with the existing bytes still in the reader (thanks Mara) readerToStore := io.MultiReader(bytes.NewBuffer(firstBytes), reader) - var maxEmojiSize int + var maxEmojiSize int64 if p.emoji.Domain == "" { - maxEmojiSize = config.GetMediaEmojiLocalMaxSize() + maxEmojiSize = int64(config.GetMediaEmojiLocalMaxSize()) } else { - maxEmojiSize = config.GetMediaEmojiRemoteMaxSize() + maxEmojiSize = int64(config.GetMediaEmojiRemoteMaxSize()) } // if we know the fileSize already, make sure it's not bigger than our limit @@ -241,7 +241,7 @@ func (p *ProcessingEmoji) store(ctx context.Context) error { return fmt.Errorf("store: discovered emoji fileSize (%db) is larger than allowed emojiRemoteMaxSize (%db)", fileSize, maxEmojiSize) } - p.emoji.ImageFileSize = fileSize + p.emoji.ImageFileSize = int(fileSize) p.read = true if p.postData != nil { diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go index 5a8e6f590..c22bddfeb 100644 --- a/internal/media/processingmedia.go +++ b/internal/media/processingmedia.go @@ -315,7 +315,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error { p.attachment.Type = gtsmodel.FileTypeImage if fileSize > 0 { var err error - readerToStore, err = terminator.Terminate(readerToStore, fileSize, extension) + readerToStore, err = terminator.Terminate(readerToStore, int(fileSize), extension) if err != nil { return fmt.Errorf("store: exif error: %s", err) } @@ -344,7 +344,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error { cached := true p.attachment.Cached = &cached - p.attachment.File.FileSize = fileSize + p.attachment.File.FileSize = int(fileSize) p.read = true if p.postData != nil { diff --git a/internal/media/pruneremote_test.go b/internal/media/pruneremote_test.go index ddf4cb568..e71d3310b 100644 --- a/internal/media/pruneremote_test.go +++ b/internal/media/pruneremote_test.go @@ -74,13 +74,13 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() { suite.ErrorIs(err, storage.ErrNotFound) // now recache the image.... - data := func(_ context.Context) (io.Reader, int, error) { + data := func(_ context.Context) (io.Reader, int64, error) { // load bytes from a test image b, err := os.ReadFile("../../testrig/media/thoughtsofdog-original.jpeg") if err != nil { panic(err) } - return bytes.NewBuffer(b), len(b), nil + return bytes.NewBuffer(b), int64(len(b)), nil } processingRecache, err := suite.manager.RecacheMedia(ctx, data, nil, testAttachment.ID) suite.NoError(err) diff --git a/internal/media/types.go b/internal/media/types.go index cceff216c..d71080658 100644 --- a/internal/media/types.go +++ b/internal/media/types.go @@ -118,7 +118,7 @@ type AdditionalEmojiInfo struct { } // DataFunc represents a function used to retrieve the raw bytes of a piece of media. -type DataFunc func(ctx context.Context) (reader io.Reader, fileSize int, err error) +type DataFunc func(ctx context.Context) (reader io.Reader, fileSize int64, err error) // PostDataCallbackFunc represents a function executed after the DataFunc has been executed, // and the returned reader has been read. It can be used to clean up any remaining resources. diff --git a/internal/media/util.go b/internal/media/util.go index b89196f87..2968ca2f6 100644 --- a/internal/media/util.go +++ b/internal/media/util.go @@ -151,19 +151,19 @@ func parseOlderThan(olderThanDays int) (time.Time, error) { // lengthReader wraps a reader and reads the length of total bytes written as it goes. type lengthReader struct { source io.Reader - length int + length int64 } func (r *lengthReader) Read(b []byte) (int, error) { n, err := r.source.Read(b) - r.length += n + r.length += int64(n) return n, err } // putStream either puts a file with a known fileSize into storage directly, and returns the // fileSize unchanged, or it wraps the reader with a lengthReader and returns the discovered // fileSize. -func putStream(ctx context.Context, storage storage.Driver, key string, r io.Reader, fileSize int) (int, error) { +func putStream(ctx context.Context, storage storage.Driver, key string, r io.Reader, fileSize int64) (int64, error) { if fileSize > 0 { return fileSize, storage.PutStream(ctx, key, r) } diff --git a/internal/processing/account/update.go b/internal/processing/account/update.go index eddaeab27..94e91ca4c 100644 --- a/internal/processing/account/update.go +++ b/internal/processing/account/update.go @@ -184,13 +184,13 @@ func (p *processor) Update(ctx context.Context, account *gtsmodel.Account, form // the account's new avatar image. func (p *processor) UpdateAvatar(ctx context.Context, avatar *multipart.FileHeader, accountID string) (*gtsmodel.MediaAttachment, error) { maxImageSize := config.GetMediaImageMaxSize() - if int(avatar.Size) > maxImageSize { + if avatar.Size > int64(maxImageSize) { return nil, fmt.Errorf("UpdateAvatar: avatar with size %d exceeded max image size of %d bytes", avatar.Size, maxImageSize) } - dataFunc := func(innerCtx context.Context) (io.Reader, int, error) { + dataFunc := func(innerCtx context.Context) (io.Reader, int64, error) { f, err := avatar.Open() - return f, int(avatar.Size), err + return f, avatar.Size, err } isAvatar := true @@ -211,13 +211,13 @@ func (p *processor) UpdateAvatar(ctx context.Context, avatar *multipart.FileHead // the account's new header image. func (p *processor) UpdateHeader(ctx context.Context, header *multipart.FileHeader, accountID string) (*gtsmodel.MediaAttachment, error) { maxImageSize := config.GetMediaImageMaxSize() - if int(header.Size) > maxImageSize { + if header.Size > int64(maxImageSize) { return nil, fmt.Errorf("UpdateHeader: header with size %d exceeded max image size of %d bytes", header.Size, maxImageSize) } - dataFunc := func(innerCtx context.Context) (io.Reader, int, error) { + dataFunc := func(innerCtx context.Context) (io.Reader, int64, error) { f, err := header.Open() - return f, int(header.Size), err + return f, header.Size, err } isHeader := true diff --git a/internal/processing/admin/emoji.go b/internal/processing/admin/emoji.go index ffb369493..50399279c 100644 --- a/internal/processing/admin/emoji.go +++ b/internal/processing/admin/emoji.go @@ -52,9 +52,9 @@ func (p *processor) EmojiCreate(ctx context.Context, account *gtsmodel.Account, emojiURI := uris.GenerateURIForEmoji(emojiID) - data := func(innerCtx context.Context) (io.Reader, int, error) { + data := func(innerCtx context.Context) (io.Reader, int64, error) { f, err := form.Image.Open() - return f, int(form.Image.Size), err + return f, form.Image.Size, err } processingEmoji, err := p.mediaManager.ProcessEmoji(ctx, data, nil, form.Shortcode, emojiID, emojiURI, nil) diff --git a/internal/processing/media/create.go b/internal/processing/media/create.go index 1f40ac48f..eb0c251e9 100644 --- a/internal/processing/media/create.go +++ b/internal/processing/media/create.go @@ -30,9 +30,9 @@ import ( ) func (p *processor) Create(ctx context.Context, account *gtsmodel.Account, form *apimodel.AttachmentRequest) (*apimodel.Attachment, gtserror.WithCode) { - data := func(innerCtx context.Context) (io.Reader, int, error) { + data := func(innerCtx context.Context) (io.Reader, int64, error) { f, err := form.File.Open() - return f, int(form.File.Size), err + return f, form.File.Size, err } focusX, focusY, err := parseFocus(form.Focus) diff --git a/internal/processing/media/getfile.go b/internal/processing/media/getfile.go index 7435a241d..104bca770 100644 --- a/internal/processing/media/getfile.go +++ b/internal/processing/media/getfile.go @@ -138,7 +138,7 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount // if it's the thumbnail that's requested then the user will have to wait a bit while we process the // large version and derive a thumbnail from it, so use the normal recaching procedure: fetch the media, // process it, then return the thumbnail data - data = func(innerCtx context.Context) (io.Reader, int, error) { + data = func(innerCtx context.Context) (io.Reader, int64, error) { transport, err := p.transportController.NewTransportForUsername(innerCtx, requestingUsername) if err != nil { return nil, 0, err @@ -169,7 +169,7 @@ func (p *processor) getAttachmentContent(ctx context.Context, requestingAccount // the caller will read from the buffered reader, so it doesn't matter if they drop out without reading everything attachmentContent.Content = bufferedReader - data = func(innerCtx context.Context) (io.Reader, int, error) { + data = func(innerCtx context.Context) (io.Reader, int64, error) { transport, err := p.transportController.NewTransportForUsername(innerCtx, requestingUsername) if err != nil { return nil, 0, err diff --git a/internal/transport/derefmedia.go b/internal/transport/derefmedia.go index 8feb7ed20..14c054cdf 100644 --- a/internal/transport/derefmedia.go +++ b/internal/transport/derefmedia.go @@ -26,7 +26,7 @@ import ( "net/url" ) -func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, int, error) { +func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, int64, error) { // Build IRI just once iriStr := iri.String() @@ -50,5 +50,5 @@ func (t *transport) DereferenceMedia(ctx context.Context, iri *url.URL) (io.Read return nil, 0, fmt.Errorf("GET request to %s failed (%d): %s", iriStr, rsp.StatusCode, rsp.Status) } - return rsp.Body, int(rsp.ContentLength), nil + return rsp.Body, rsp.ContentLength, nil } diff --git a/internal/transport/transport.go b/internal/transport/transport.go index 80710a519..5af8b738e 100644 --- a/internal/transport/transport.go +++ b/internal/transport/transport.go @@ -47,7 +47,7 @@ import ( type Transport interface { pub.Transport // DereferenceMedia fetches the given media attachment IRI, returning the reader and filesize. - DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, int, error) + DereferenceMedia(ctx context.Context, iri *url.URL) (io.ReadCloser, int64, error) // DereferenceInstance dereferences remote instance information, first by checking /api/v1/instance, and then by checking /.well-known/nodeinfo. DereferenceInstance(ctx context.Context, iri *url.URL) (*gtsmodel.Instance, error) // Finger performs a webfinger request with the given username and domain, and returns the bytes from the response body. diff --git a/internal/typeutils/internaltofrontend.go b/internal/typeutils/internaltofrontend.go index ca86a1284..aa98180d3 100644 --- a/internal/typeutils/internaltofrontend.go +++ b/internal/typeutils/internaltofrontend.go @@ -688,9 +688,9 @@ func (c *converter) InstanceToAPIInstance(ctx context.Context, i *gtsmodel.Insta }, MediaAttachments: &model.InstanceConfigurationMediaAttachments{ SupportedMimeTypes: media.AllSupportedMIMETypes(), - ImageSizeLimit: config.GetMediaImageMaxSize(), + ImageSizeLimit: int(config.GetMediaImageMaxSize()), ImageMatrixLimit: instanceMediaAttachmentsImageMatrixLimit, // height*width - VideoSizeLimit: config.GetMediaVideoMaxSize(), + VideoSizeLimit: int(config.GetMediaVideoMaxSize()), VideoFrameRateLimit: instanceMediaAttachmentsVideoFrameRateLimit, VideoMatrixLimit: instanceMediaAttachmentsVideoMatrixLimit, // height*width }, diff --git a/test/envparsing.sh b/test/envparsing.sh index 7a31f428c..421fac3c2 100755 --- a/test/envparsing.sh +++ b/test/envparsing.sh @@ -2,7 +2,7 @@ set -eu -EXPECTED='{"account-domain":"peepee","accounts-allow-custom-css":true,"accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","application-name":"gts","bind-address":"127.0.0.1","config-path":"./test/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-password":"hunter2","db-port":6969,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","email":"","host":"example.com","instance-deliver-to-shared-inboxes":false,"instance-expose-peers":true,"instance-expose-suspended":true,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","smtp-from":"queen@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","docker.host.local"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}' +EXPECT='{"account-domain":"peepee","accounts-allow-custom-css":true,"accounts-approval-required":false,"accounts-reason-required":false,"accounts-registration-open":true,"advanced-cookies-samesite":"strict","application-name":"gts","bind-address":"127.0.0.1","config-path":"./test/test.yaml","db-address":":memory:","db-database":"gotosocial_prod","db-password":"hunter2","db-port":6969,"db-tls-ca-cert":"","db-tls-mode":"disable","db-type":"sqlite","db-user":"sex-haver","email":"","host":"example.com","instance-deliver-to-shared-inboxes":false,"instance-expose-peers":true,"instance-expose-suspended":true,"letsencrypt-cert-dir":"/gotosocial/storage/certs","letsencrypt-email-address":"","letsencrypt-enabled":true,"letsencrypt-port":80,"log-db-queries":true,"log-level":"info","media-description-max-chars":5000,"media-description-min-chars":69,"media-emoji-local-max-size":420,"media-emoji-remote-max-size":420,"media-image-max-size":420,"media-remote-cache-days":30,"media-video-max-size":420,"oidc-client-id":"1234","oidc-client-secret":"shhhh its a secret","oidc-enabled":true,"oidc-idp-name":"sex-haver","oidc-issuer":"whoknows","oidc-scopes":["read","write"],"oidc-skip-verification":true,"password":"","path":"","port":6969,"protocol":"http","smtp-from":"queen.rip.in.piss@terfisland.org","smtp-host":"example.com","smtp-password":"hunter2","smtp-port":4269,"smtp-username":"sex-haver","software-version":"","statuses-cw-max-chars":420,"statuses-max-chars":69,"statuses-media-max-files":1,"statuses-poll-max-options":1,"statuses-poll-option-max-chars":50,"storage-backend":"local","storage-local-base-path":"/root/store","storage-s3-access-key":"minio","storage-s3-bucket":"gts","storage-s3-endpoint":"localhost:9000","storage-s3-secret-key":"miniostorage","storage-s3-use-ssl":false,"syslog-address":"127.0.0.1:6969","syslog-enabled":true,"syslog-protocol":"udp","trusted-proxies":["127.0.0.1/32","docker.host.local"],"username":"","web-asset-base-dir":"/root","web-template-base-dir":"/root"}' # Set all the environment variables to # ensure that these are parsed without panic @@ -66,16 +66,24 @@ GTS_SMTP_HOST='example.com' \ GTS_SMTP_PORT=4269 \ GTS_SMTP_USERNAME='sex-haver' \ GTS_SMTP_PASSWORD='hunter2' \ -GTS_SMTP_FROM='queen@terfisland.org' \ +GTS_SMTP_FROM='queen.rip.in.piss@terfisland.org' \ GTS_SYSLOG_ENABLED=true \ GTS_SYSLOG_PROTOCOL='udp' \ GTS_SYSLOG_ADDRESS='127.0.0.1:6969' \ GTS_ADVANCED_COOKIES_SAMESITE='strict' \ go run ./cmd/gotosocial/... --config-path $(dirname ${0})/test.yaml debug config) -if [ "${OUTPUT}" != "${EXPECTED}" ]; then +OUTPUT_OUT=$(mktemp) +echo "$OUTPUT" > "$OUTPUT_OUT" + +EXPECT_OUT=$(mktemp) +echo "$EXPECT" > "$EXPECT_OUT" + +if ! DIFF=$(diff "$OUTPUT_OUT" "$EXPECT_OUT"); then echo "OUTPUT not equal EXPECTED" + echo "$DIFF" exit 1 else echo "OK" + exit 0 fi diff --git a/vendor/codeberg.org/gruf/go-bytesize/bytesize.go b/vendor/codeberg.org/gruf/go-bytesize/bytesize.go index c2fedc8d7..4426107bd 100644 --- a/vendor/codeberg.org/gruf/go-bytesize/bytesize.go +++ b/vendor/codeberg.org/gruf/go-bytesize/bytesize.go @@ -6,6 +6,24 @@ import ( "unsafe" ) +const ( + // SI units + KB Size = 1e3 + MB Size = 1e6 + GB Size = 1e9 + TB Size = 1e12 + PB Size = 1e15 + EB Size = 1e18 + + // IEC units + KiB Size = 1024 + MiB Size = KiB * 1024 + GiB Size = MiB * 1024 + TiB Size = GiB * 1024 + PiB Size = TiB * 1024 + EiB Size = PiB * 1024 +) + var ( // ErrInvalidUnit is returned when an invalid IEC/SI is provided. ErrInvalidUnit = errors.New("bytesize: invalid unit") @@ -13,42 +31,39 @@ var ( // ErrInvalidFormat is returned when an invalid size value is provided. ErrInvalidFormat = errors.New("bytesize: invalid format") - // bunits are the binary unit chars. - units = `kMGTPE` - // iecpows is a precomputed table of 1024^n. iecpows = [...]float64{ - float64(1024), // KiB - float64(1024 * 1024), // MiB - float64(1024 * 1024 * 1024), // GiB - float64(1024 * 1024 * 1024 * 1024), // TiB - float64(1024 * 1024 * 1024 * 1024 * 1024), // PiB - float64(1024 * 1024 * 1024 * 1024 * 1024 * 1024), // EiB + float64(KiB), + float64(MiB), + float64(GiB), + float64(TiB), + float64(PiB), + float64(EiB), } // sipows is a precomputed table of 1000^n. sipows = [...]float64{ - float64(1e3), // KB - float64(1e6), // MB - float64(1e9), // GB - float64(1e12), // TB - float64(1e15), // PB - float64(1e18), // EB + float64(KB), + float64(MB), + float64(GB), + float64(TB), + float64(PB), + float64(EB), } // bvals is a precomputed table of IEC unit values. - iecvals = [...]uint64{ - 'k': 1024, - 'K': 1024, - 'M': 1024 * 1024, - 'G': 1024 * 1024 * 1024, - 'T': 1024 * 1024 * 1024 * 1024, - 'P': 1024 * 1024 * 1024 * 1024 * 1024, - 'E': 1024 * 1024 * 1024 * 1024 * 1024 * 1024, + iecvals = [...]Size{ + 'k': KiB, + 'K': KiB, + 'M': MiB, + 'G': GiB, + 'T': TiB, + 'P': PiB, + 'E': EiB, } // sivals is a precomputed table of SI unit values. - sivals = [...]uint64{ + sivals = [...]Size{ // ASCII numbers _aren't_ valid SI unit values, // BUT if the space containing a possible unit // char is checked with this table -- it is valid @@ -64,12 +79,13 @@ var ( '8': 1, '9': 1, - 'k': 1e3, - 'M': 1e6, - 'G': 1e9, - 'T': 1e12, - 'P': 1e15, - 'E': 1e18, + 'k': KB, + 'K': KB, + 'M': MB, + 'G': GB, + 'T': TB, + 'P': PB, + 'E': EB, } ) @@ -91,7 +107,28 @@ func ParseSize(s string) (Size, error) { return 0, ErrInvalidFormat } - return Size(uint64(f) * unit), nil + return Size(f) * unit, nil +} + +// Set implements flag.Value{}. +func (sz *Size) Set(in string) error { + s, err := ParseSize(in) + if err != nil { + return err + } + *sz = s + return nil +} + +// MarshalText implements encoding.TextMarshaler{}. +func (sz *Size) MarshalText() ([]byte, error) { + const maxLen = 7 // max IEC string length + return sz.AppendFormatIEC(make([]byte, 0, maxLen)), nil +} + +// UnmarshalText implements encoding.TextUnmarshaler{}. +func (sz *Size) UnmarshalText(text []byte) error { + return sz.Set(*(*string)(unsafe.Pointer(&text))) } // AppendFormat defaults to using Size.AppendFormatIEC(). @@ -121,7 +158,13 @@ func (sz Size) AppendFormatIEC(dst []byte) []byte { // appendFormat will append formatted Size to 'dst', depending on base, powers table and single unit suffix. func (sz Size) appendFormat(dst []byte, base uint64, pows *[6]float64, sunit string) []byte { - const min = 0.75 + const ( + // min "small" unit threshold + min = 0.75 + + // binary unit chars. + units = `kMGTPE` + ) // Larger number: get value of // i / unit size. We have a 'min' @@ -143,13 +186,15 @@ func (sz Size) appendFormat(dst []byte, base uint64, pows *[6]float64, sunit str // StringSI returns an SI unit string format of Size. func (sz Size) StringSI() string { - b := sz.AppendFormatSI(make([]byte, 0, 6)) + const maxLen = 6 // max SI string length + b := sz.AppendFormatSI(make([]byte, 0, maxLen)) return *(*string)(unsafe.Pointer(&b)) } // StringIEC returns an IEC unit string format of Size. func (sz Size) StringIEC() string { - b := sz.AppendFormatIEC(make([]byte, 0, 7)) + const maxLen = 7 // max IEC string length + b := sz.AppendFormatIEC(make([]byte, 0, maxLen)) return *(*string)(unsafe.Pointer(&b)) } @@ -159,9 +204,7 @@ func (sz Size) String() string { } // parseUnit will parse the byte size unit from string 's'. -func parseUnit(s string) (uint64, int, error) { - var isIEC bool - +func parseUnit(s string) (Size, int, error) { // Check for string if len(s) < 1 { return 0, 0, ErrInvalidFormat @@ -171,8 +214,8 @@ func parseUnit(s string) (uint64, int, error) { if l := len(s) - 1; s[l] == 'B' { s = s[:l] - // Check str remains if len(s) < 1 { + // No remaining str before unit suffix return 0, 0, ErrInvalidFormat } } @@ -180,38 +223,41 @@ func parseUnit(s string) (uint64, int, error) { // Strip IEC binary unit suffix if l := len(s) - 1; s[l] == 'i' { s = s[:l] - isIEC = true - // Check str remains if len(s) < 1 { + // No remaining str before unit suffix return 0, 0, ErrInvalidFormat } + + // Location of unit char. + l := len(s) - 1 + c := int(s[l]) + + // Check valid unit char was provided + if len(iecvals) < c || iecvals[c] == 0 { + return 0, 0, ErrInvalidUnit + } + + // Return parsed IEC unit size + return iecvals[c], l, nil } // Location of unit char. l := len(s) - 1 + c := int(s[l]) - var unit uint64 - switch c := int(s[l]); { - // Determine IEC unit in use - case isIEC && c < len(iecvals): - unit = iecvals[c] - if unit == 0 { - return 0, 0, ErrInvalidUnit - } + switch { + // Check valid unit char provided + case len(sivals) < c || sivals[c] == 0: + return 0, 0, ErrInvalidUnit - // Determine SI unit in use - case c < len(sivals): - unit = sivals[c] - switch unit { - case 0: - return 0, 0, ErrInvalidUnit - case 1: - l++ - } + // No unit char (only ascii number) + case sivals[c] == 1: + l++ } - return unit, l, nil + // Return parsed SI unit size + return sivals[c], l, nil } // ftoa appends string formatted 'f' to 'dst', assumed < ~800. @@ -286,7 +332,7 @@ func ftoa(dst []byte, f float64) []byte { // itoa appends string formatted 'i' to 'dst'. func itoa(dst []byte, i uint64) []byte { - // Assemble int in reverse order. + // Assemble uint in reverse order. var b [4]byte bp := len(b) - 1 @@ -302,5 +348,10 @@ func itoa(dst []byte, i uint64) []byte { return append(dst, b[bp:]...) } +// We use the following internal strconv function usually +// used internally to parse float values, as we know that +// are value passed will always be of 64bit type, and knowing +// the returned float string length is very helpful! +// //go:linkname atof64 strconv.atof64 func atof64(string) (float64, int, error) diff --git a/vendor/modules.txt b/vendor/modules.txt index cdc38b6e4..704d97e3a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -7,7 +7,7 @@ codeberg.org/gruf/go-bitutil # codeberg.org/gruf/go-bytes v1.0.2 ## explicit; go 1.14 codeberg.org/gruf/go-bytes -# codeberg.org/gruf/go-bytesize v0.2.1 +# codeberg.org/gruf/go-bytesize v1.0.0 ## explicit; go 1.17 codeberg.org/gruf/go-bytesize # codeberg.org/gruf/go-byteutil v1.0.2