From 2a99df0588e168660d3b528209d8f51689ca92b7 Mon Sep 17 00:00:00 2001 From: tobi <31960611+tsmethurst@users.noreply.github.com> Date: Fri, 7 Jul 2023 16:17:39 +0200 Subject: [PATCH] [feature] enable + document explicit IP dialer allowing/denying (#1950) * [feature] enable + document explicit IP dialer allowing/denying * lord have mercy * allee jonge * shortcut check ipv6 prefixes * comment * separate httpclient_test, export Sanitizer --- cmd/gotosocial/action/server/server.go | 8 +- docs/configuration/httpclient.md | 56 +++++++++ docs/faq.md | 59 +++++++--- example/config.yaml | 51 ++++++++ internal/config/config.go | 9 ++ internal/config/defaults.go | 6 + internal/config/flags.go | 5 + internal/config/gen/gen.go | 14 ++- internal/config/helpers.gen.go | 75 ++++++++++++ internal/config/util.go | 39 +++++++ internal/httpclient/client.go | 6 +- internal/httpclient/sanitizer.go | 110 +++++++++++++++--- internal/httpclient/sanitizer_test.go | 154 +++++++++++++++++++++++++ internal/netutil/validate.go | 102 ---------------- internal/netutil/validate_test.go | 54 --------- mkdocs.yml | 1 + test/envparsing.sh | 5 + 17 files changed, 554 insertions(+), 200 deletions(-) create mode 100644 docs/configuration/httpclient.md create mode 100644 internal/config/util.go create mode 100644 internal/httpclient/sanitizer_test.go delete mode 100644 internal/netutil/validate.go delete mode 100644 internal/netutil/validate_test.go diff --git a/cmd/gotosocial/action/server/server.go b/cmd/gotosocial/action/server/server.go index fa4ec9b82..cdcedd47f 100644 --- a/cmd/gotosocial/action/server/server.go +++ b/cmd/gotosocial/action/server/server.go @@ -105,8 +105,12 @@ var Start action.GTSAction = func(ctx context.Context) error { // Set the state storage driver state.Storage = storage - // Build HTTP client (TODO: add configurables here) - client := httpclient.New(httpclient.Config{}) + // Build HTTP client + client := httpclient.New(httpclient.Config{ + AllowRanges: config.MustParseIPPrefixes(config.GetHTTPClientAllowIPs()), + BlockRanges: config.MustParseIPPrefixes(config.GetHTTPClientBlockIPs()), + Timeout: config.GetHTTPClientTimeout(), + }) // Initialize workers. state.Workers.Start() diff --git a/docs/configuration/httpclient.md b/docs/configuration/httpclient.md new file mode 100644 index 000000000..1fcf2d061 --- /dev/null +++ b/docs/configuration/httpclient.md @@ -0,0 +1,56 @@ +# HTTP Client + +## Settings + +```yaml +################################ +##### HTTP CLIENT SETTINGS ##### +################################ + +# Settings for OUTGOING http client connections used by GoToSocial to make +# requests to remote resources (status GETs, media GETs, inbox POSTs, etc). + +http-client: + + # Duration. Timeout to use for outgoing HTTP requests. If the timeout + # is exceeded, the connection to the remote server will be dropped. + # A value of 0s indicates no timeout: this is not advised! + # Examples: ["5s", "10s", "0s"] + # Default: "10s" + timeout: "10s" + + ######################################## + #### RESERVED IP RANGE EXCEPTIONS ###### + ######################################## + # + # Explicitly allow or block outgoing dialing within the provided IPv4/v6 CIDR ranges. + # + # By default, as a basic security precaution, GoToSocial blocks outgoing dialing within most "special-purpose" + # IP ranges. However, it may be desirable for admins with more exotic setups (proxies, funky NAT, etc) to + # explicitly override one or more of these otherwise blocked ranges. + # + # Each of the below allow/block config options accepts an array of IPv4 and/or IPv6 CIDR strings. + # For example, to override the hardcoded block of IPv4 and IPv6 dialing to localhost, set: + # + # allow-ips: ["127.0.0.1/32", "::1/128"]. + # + # You can also use YAML multi-line arrays to define these, but be diligent with indentation. + # + # When dialing, GoToSocial will first check if the destination falls within explicitly allowed IP ranges, + # then explicitly blocked IP ranges, then the default (hardcoded) blocked IP ranges, returning OK on the + # first allowed match, not OK on the first blocked match, or just defaulting to OK if nothing is matched. + # + # As with all security settings, it is better to start too restrictive and then ease off depending on + # your use case, than to start too permissive and try to close the stable door after the horse has + # already bolted. With this in mind: + # - Don't touch these settings unless you have a good reason to, and only if you know what you're doing. + # - When adding explicitly allowed exceptions, use the narrowest possible CIDR for your use case. + # + # For reserved / special ranges, see: + # - https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml + # - https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml + # + # Both allow-ips and block-ips default to an empty array. + allow-ips: [] + block-ips: [] +``` diff --git a/docs/faq.md b/docs/faq.md index 1622e13a1..c5a1b01df 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -1,29 +1,50 @@ # Frequently Asked Questions -- **Where's the user interface?** GoToSocial is just a bare server for the most part and is designed to be used thru external applications. [Semaphore](https://semaphore.social/) and [Tusky](https://tusky.app/) are the best-supported, but anything that supports the Mastodon API should work, other than the features GoToSocial doesn't yet have. Permalinks and profile pages are served directly thru GoToSocial as well as the settings panel, but most interaction goes thru the apps. +## Where's the user interface? -- **Why aren't my posts showing up on my profile page?** Unlike Mastodon, the default post visibility is Unlisted. If you want something to be visible on your profile page, the post must have Public visibility. +GoToSocial is just a bare server for the most part and is designed to be used thru external applications. [Semaphore](https://semaphore.social/) and [Tusky](https://tusky.app/) are the best-supported, but anything that supports the Mastodon API should work, other than the features GoToSocial doesn't yet have. Permalinks and profile pages are served directly through GoToSocial as well as the settings panel, but most interaction goes through the apps. -- **Why aren't my posts showing up on other servers?** First check the visibility as noted above. TODO: explain how to debug common federation issues +## Why aren't my posts showing up on my profile page? -- **Why am I getting frequent http 429 error responses?** GoToSocial is configured to use per-IP [rate limiting](./api/ratelimiting.md) by default, but in certain situations it can't accurately identify the remote IP and will treat all connections as coming from the same place. In those cases, the rate limiting needs to be disabled or reconfigured. +Unlike Mastodon, the default post visibility is Unlisted. If you want something to be visible on your profile page, the post must have Public visibility. -- **Why am I getting frequent http 503 error responses?** Code 503 is returned to callers when your instance is under heavy load and requests are being throttled. This behavior can be tuned as desired, or turned off entirely, see [here](./api/throttling.md). +## Why aren't my posts showing up on other servers? -- **I keep getting a 400 Bad Request error, and I have done everything suggested by the error message. What should I do?** Verify that the `host` configuration matches the domain that GoToSocial is served from (the domain that users use to acces the server). +First check the visibility as noted above. TODO: explain how to debug common federation issues -- **My instance is deployed and I'm logged in to a client but my timelines are empty, what's up there?** To see posts, you have to start following people! Once you've followed a few people and they've posted or boosted things, you'll start seeing them in your timelines. Right now GoToSocial doesn't have a way of 'backfilling' posts -- that is, fetching previous posts from other instances -- so you'll only see new posts of people you follow. If you want to interact with an older post of theirs, you can copy the link to the post from their web profile, and paste it in to your client's search bar. +## Why am I getting frequent http 429 error responses? -- **How can I sign up for a server?** Right now the only way to create an account is by the server's admin to run a command directly on the server. A web-based signup flow is in the roadmap but not implemented yet. +GoToSocial is configured to use per-IP [rate limiting](./api/ratelimiting.md) by default, but in certain situations it can't accurately identify the remote IP and will treat all connections as coming from the same place. In those cases, the rate limiting needs to be disabled or reconfigured. -- **Why's it still in alpha?** Take a look at the [list of open bugs](https://github.com/superseriousbusiness/gotosocial/issues?q=is%3Aissue+is%3Aopen+label%3Abug) and the [roadmap](https://github.com/superseriousbusiness/gotosocial/blob/main/ROADMAP.md) for a more detailed rundown, but the main missing features at the time of this writing are: - * muting conversations - * backfill of posts - * web-based signup - * profile metadata fields - * lists of users - * polls - * scheduling posts - * account migration - * federated hashtag search - * shared block lists across servers +## Why am I getting frequent HTTP 503 error responses? + +Code 503 is returned to callers when your instance is under heavy load and requests are being throttled. This behavior can be tuned as desired, or turned off entirely, see [here](./api/throttling.md). + +## I keep getting a 400 Bad Request error, and I have done everything suggested by the error message. What should I do? + +Verify that the `host` configuration matches the domain that GoToSocial is served from (the domain that users use to acces the server). + +## I keep seeing 'dial within blocked / reserved IP range' in my server logs, and I can't connect to some instances from my instance, what do I do? + +The IP address of the remote instance may be in one of the blocked "special use" IP ranges hardcoded into GoToSocial for security reasons. If you need to, you can override this in your configuration file. Have a look at the [http client docs](./configuration/httpclient.md) for this, and please read the warnings there carefully! If you add an explicit allow, you will have to restart your GoToSocial instance to make the config take effect. + +## My instance is deployed and I'm logged in to a client but my timelines are empty, what's up there? + +To see posts, you have to start following people! Once you've followed a few people and they've posted or boosted things, you'll start seeing them in your timelines. Right now GoToSocial doesn't have a way of 'backfilling' posts -- that is, fetching previous posts from other instances -- so you'll only see new posts of people you follow. If you want to interact with an older post of theirs, you can copy the link to the post from their web profile, and paste it in to your client's search bar. + +## How can I sign up for a server? + +Right now the only way to create an account is by the server's admin to run a command directly on the server. A web-based signup flow is in the roadmap but not implemented yet. + +## Why's it still in alpha? + +Take a look at the [list of open bugs](https://github.com/superseriousbusiness/gotosocial/issues?q=is%3Aissue+is%3Aopen+label%3Abug) and the [roadmap](https://github.com/superseriousbusiness/gotosocial/blob/main/ROADMAP.md) for a more detailed rundown, but the main missing features at the time of this writing are: + +- muting conversations +- backfill of posts +- web-based signup +- polls +- scheduling posts +- account migration +- federated hashtag search +- shared block lists across servers diff --git a/example/config.yaml b/example/config.yaml index c0b850ad9..823a126b3 100644 --- a/example/config.yaml +++ b/example/config.yaml @@ -813,6 +813,57 @@ tracing-endpoint: "" # Default: false tracing-insecure-transport: false +################################ +##### HTTP CLIENT SETTINGS ##### +################################ + +# Settings for OUTGOING http client connections used by GoToSocial to make +# requests to remote resources (status GETs, media GETs, inbox POSTs, etc). + +http-client: + + # Duration. Timeout to use for outgoing HTTP requests. If the timeout + # is exceeded, the connection to the remote server will be dropped. + # A value of 0s indicates no timeout: this is not advised! + # Examples: ["5s", "10s", "0s"] + # Default: "10s" + timeout: "10s" + + ######################################## + #### RESERVED IP RANGE EXCEPTIONS ###### + ######################################## + # + # Explicitly allow or block outgoing dialing within the provided IPv4/v6 CIDR ranges. + # + # By default, as a basic security precaution, GoToSocial blocks outgoing dialing within most "special-purpose" + # IP ranges. However, it may be desirable for admins with more exotic setups (proxies, funky NAT, etc) to + # explicitly override one or more of these otherwise blocked ranges. + # + # Each of the below allow/block config options accepts an array of IPv4 and/or IPv6 CIDR strings. + # For example, to override the hardcoded block of IPv4 and IPv6 dialing to localhost, set: + # + # allow-ips: ["127.0.0.1/32", "::1/128"]. + # + # You can also use YAML multi-line arrays to define these, but be diligent with indentation. + # + # When dialing, GoToSocial will first check if the destination falls within explicitly allowed IP ranges, + # then explicitly blocked IP ranges, then the default (hardcoded) blocked IP ranges, returning OK on the + # first allowed match, not OK on the first blocked match, or just defaulting to OK if nothing is matched. + # + # As with all security settings, it is better to start too restrictive and then ease off depending on + # your use case, than to start too permissive and try to close the stable door after the horse has + # already bolted. With this in mind: + # - Don't touch these settings unless you have a good reason to, and only if you know what you're doing. + # - When adding explicitly allowed exceptions, use the narrowest possible CIDR for your use case. + # + # For reserved / special ranges, see: + # - https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml + # - https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml + # + # Both allow-ips and block-ips default to an empty array. + allow-ips: [] + block-ips: [] + ############################# ##### ADVANCED SETTINGS ##### ############################# diff --git a/internal/config/config.go b/internal/config/config.go index c809bbc1b..53514e20b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -150,6 +150,9 @@ type Configuration struct { AdvancedThrottlingRetryAfter time.Duration `name:"advanced-throttling-retry-after" usage:"Retry-After duration response to send for throttled requests."` AdvancedSenderMultiplier int `name:"advanced-sender-multiplier" usage:"Multiplier to use per cpu for batching outgoing fedi messages. 0 or less turns batching off (not recommended)."` + // HTTPClient configuration vars. + HTTPClient HTTPClientConfiguration `name:"http-client"` + // Cache configuration vars. Cache CacheConfiguration `name:"cache"` @@ -163,6 +166,12 @@ type Configuration struct { RequestIDHeader string `name:"request-id-header" usage:"Header to extract the Request ID from. Eg.,'X-Request-Id'."` } +type HTTPClientConfiguration struct { + AllowIPs []string `name:"allow-ips"` + BlockIPs []string `name:"block-ips"` + Timeout time.Duration `name:"timeout"` +} + type CacheConfiguration struct { GTS GTSCacheConfiguration `name:"gts"` diff --git a/internal/config/defaults.go b/internal/config/defaults.go index 1cb53c8e2..34e46b342 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -208,6 +208,12 @@ var Defaults = Configuration{ VisibilitySweepFreq: time.Minute, }, + HTTPClient: HTTPClientConfiguration{ + AllowIPs: make([]string, 0), + BlockIPs: make([]string, 0), + Timeout: 10 * time.Second, + }, + AdminMediaPruneDryRun: true, RequestIDHeader: "X-Request-Id", diff --git a/internal/config/flags.go b/internal/config/flags.go index c9899b67e..c42b5c7b2 100644 --- a/internal/config/flags.go +++ b/internal/config/flags.go @@ -55,6 +55,11 @@ func (s *ConfigState) AddGlobalFlags(cmd *cobra.Command) { cmd.PersistentFlags().String(DbSqliteSynchronousFlag(), cfg.DbSqliteSynchronous, fieldtag("DbSqliteSynchronous", "usage")) cmd.PersistentFlags().Uint64(DbSqliteCacheSizeFlag(), uint64(cfg.DbSqliteCacheSize), fieldtag("DbSqliteCacheSize", "usage")) cmd.PersistentFlags().Duration(DbSqliteBusyTimeoutFlag(), cfg.DbSqliteBusyTimeout, fieldtag("DbSqliteBusyTimeout", "usage")) + + // HTTPClient + cmd.PersistentFlags().StringSlice(HTTPClientAllowIPsFlag(), cfg.HTTPClient.AllowIPs, "no usage string") + cmd.PersistentFlags().StringSlice(HTTPClientBlockIPsFlag(), cfg.HTTPClient.BlockIPs, "no usage string") + cmd.PersistentFlags().Duration(HTTPClientTimeoutFlag(), cfg.HTTPClient.Timeout, "no usage string") }) } diff --git a/internal/config/gen/gen.go b/internal/config/gen/gen.go index 30994c652..1c2c11747 100644 --- a/internal/config/gen/gen.go +++ b/internal/config/gen/gen.go @@ -96,16 +96,22 @@ func generateFields(output io.Writer, prefixes []string, t reflect.Type) { flagPath := strings.Join(append(prefixes, field.Tag.Get("name")), "-") flagPath = strings.ToLower(flagPath) + // Get type without "config." prefix. + fieldType := strings.ReplaceAll( + field.Type.String(), + "config.", "", + ) + // ConfigState structure helper methods fmt.Fprintf(output, "// Get%s safely fetches the Configuration value for state's '%s' field\n", name, fieldPath) - fmt.Fprintf(output, "func (st *ConfigState) Get%s() (v %s) {\n", name, field.Type.String()) + fmt.Fprintf(output, "func (st *ConfigState) Get%s() (v %s) {\n", name, fieldType) fmt.Fprintf(output, "\tst.mutex.Lock()\n") fmt.Fprintf(output, "\tv = st.config.%s\n", fieldPath) fmt.Fprintf(output, "\tst.mutex.Unlock()\n") fmt.Fprintf(output, "\treturn\n") fmt.Fprintf(output, "}\n\n") fmt.Fprintf(output, "// Set%s safely sets the Configuration value for state's '%s' field\n", name, fieldPath) - fmt.Fprintf(output, "func (st *ConfigState) Set%s(v %s) {\n", name, field.Type.String()) + fmt.Fprintf(output, "func (st *ConfigState) Set%s(v %s) {\n", name, fieldType) fmt.Fprintf(output, "\tst.mutex.Lock()\n") fmt.Fprintf(output, "\tdefer st.mutex.Unlock()\n") fmt.Fprintf(output, "\tst.config.%s = v\n", fieldPath) @@ -117,8 +123,8 @@ func generateFields(output io.Writer, prefixes []string, t reflect.Type) { fmt.Fprintf(output, "// %sFlag returns the flag name for the '%s' field\n", name, fieldPath) fmt.Fprintf(output, "func %sFlag() string { return \"%s\" }\n\n", name, flagPath) fmt.Fprintf(output, "// Get%s safely fetches the value for global configuration '%s' field\n", name, fieldPath) - fmt.Fprintf(output, "func Get%[1]s() %[2]s { return global.Get%[1]s() }\n\n", name, field.Type.String()) + fmt.Fprintf(output, "func Get%[1]s() %[2]s { return global.Get%[1]s() }\n\n", name, fieldType) fmt.Fprintf(output, "// Set%s safely sets the value for global configuration '%s' field\n", name, fieldPath) - fmt.Fprintf(output, "func Set%[1]s(v %[2]s) { global.Set%[1]s(v) }\n\n", name, field.Type.String()) + fmt.Fprintf(output, "func Set%[1]s(v %[2]s) { global.Set%[1]s(v) }\n\n", name, fieldType) } } diff --git a/internal/config/helpers.gen.go b/internal/config/helpers.gen.go index c82eba3b3..56360a6c2 100644 --- a/internal/config/helpers.gen.go +++ b/internal/config/helpers.gen.go @@ -2299,6 +2299,81 @@ func GetAdvancedSenderMultiplier() int { return global.GetAdvancedSenderMultipli // SetAdvancedSenderMultiplier safely sets the value for global configuration 'AdvancedSenderMultiplier' field func SetAdvancedSenderMultiplier(v int) { global.SetAdvancedSenderMultiplier(v) } +// GetHTTPClientAllowIPs safely fetches the Configuration value for state's 'HTTPClient.AllowIPs' field +func (st *ConfigState) GetHTTPClientAllowIPs() (v []string) { + st.mutex.Lock() + v = st.config.HTTPClient.AllowIPs + st.mutex.Unlock() + return +} + +// SetHTTPClientAllowIPs safely sets the Configuration value for state's 'HTTPClient.AllowIPs' field +func (st *ConfigState) SetHTTPClientAllowIPs(v []string) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.HTTPClient.AllowIPs = v + st.reloadToViper() +} + +// HTTPClientAllowIPsFlag returns the flag name for the 'HTTPClient.AllowIPs' field +func HTTPClientAllowIPsFlag() string { return "httpclient-allow-ips" } + +// GetHTTPClientAllowIPs safely fetches the value for global configuration 'HTTPClient.AllowIPs' field +func GetHTTPClientAllowIPs() []string { return global.GetHTTPClientAllowIPs() } + +// SetHTTPClientAllowIPs safely sets the value for global configuration 'HTTPClient.AllowIPs' field +func SetHTTPClientAllowIPs(v []string) { global.SetHTTPClientAllowIPs(v) } + +// GetHTTPClientBlockIPs safely fetches the Configuration value for state's 'HTTPClient.BlockIPs' field +func (st *ConfigState) GetHTTPClientBlockIPs() (v []string) { + st.mutex.Lock() + v = st.config.HTTPClient.BlockIPs + st.mutex.Unlock() + return +} + +// SetHTTPClientBlockIPs safely sets the Configuration value for state's 'HTTPClient.BlockIPs' field +func (st *ConfigState) SetHTTPClientBlockIPs(v []string) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.HTTPClient.BlockIPs = v + st.reloadToViper() +} + +// HTTPClientBlockIPsFlag returns the flag name for the 'HTTPClient.BlockIPs' field +func HTTPClientBlockIPsFlag() string { return "httpclient-block-ips" } + +// GetHTTPClientBlockIPs safely fetches the value for global configuration 'HTTPClient.BlockIPs' field +func GetHTTPClientBlockIPs() []string { return global.GetHTTPClientBlockIPs() } + +// SetHTTPClientBlockIPs safely sets the value for global configuration 'HTTPClient.BlockIPs' field +func SetHTTPClientBlockIPs(v []string) { global.SetHTTPClientBlockIPs(v) } + +// GetHTTPClientTimeout safely fetches the Configuration value for state's 'HTTPClient.Timeout' field +func (st *ConfigState) GetHTTPClientTimeout() (v time.Duration) { + st.mutex.Lock() + v = st.config.HTTPClient.Timeout + st.mutex.Unlock() + return +} + +// SetHTTPClientTimeout safely sets the Configuration value for state's 'HTTPClient.Timeout' field +func (st *ConfigState) SetHTTPClientTimeout(v time.Duration) { + st.mutex.Lock() + defer st.mutex.Unlock() + st.config.HTTPClient.Timeout = v + st.reloadToViper() +} + +// HTTPClientTimeoutFlag returns the flag name for the 'HTTPClient.Timeout' field +func HTTPClientTimeoutFlag() string { return "httpclient-timeout" } + +// GetHTTPClientTimeout safely fetches the value for global configuration 'HTTPClient.Timeout' field +func GetHTTPClientTimeout() time.Duration { return global.GetHTTPClientTimeout() } + +// SetHTTPClientTimeout safely sets the value for global configuration 'HTTPClient.Timeout' field +func SetHTTPClientTimeout(v time.Duration) { global.SetHTTPClientTimeout(v) } + // GetCacheGTSAccountMaxSize safely fetches the Configuration value for state's 'Cache.GTS.AccountMaxSize' field func (st *ConfigState) GetCacheGTSAccountMaxSize() (v int) { st.mutex.Lock() diff --git a/internal/config/util.go b/internal/config/util.go new file mode 100644 index 000000000..a9df08b3c --- /dev/null +++ b/internal/config/util.go @@ -0,0 +1,39 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package config + +import ( + "net/netip" + + "github.com/superseriousbusiness/gotosocial/internal/log" +) + +func MustParseIPPrefixes(in []string) []netip.Prefix { + prefs := make([]netip.Prefix, 0, len(in)) + + for _, i := range in { + pref, err := netip.ParsePrefix(i) + if err != nil { + log.Panicf(nil, "error parsing ip prefix from %q: %v", i, err) + } + + prefs = append(prefs, pref) + } + + return prefs +} diff --git a/internal/httpclient/client.go b/internal/httpclient/client.go index 2a2485561..65c521113 100644 --- a/internal/httpclient/client.go +++ b/internal/httpclient/client.go @@ -130,9 +130,9 @@ func New(cfg Config) *Client { } // Protect dialer with IP range sanitizer. - d.Control = (&sanitizer{ - allow: cfg.AllowRanges, - block: cfg.BlockRanges, + d.Control = (&Sanitizer{ + Allow: cfg.AllowRanges, + Block: cfg.BlockRanges, }).Sanitize // Prepare client fields. diff --git a/internal/httpclient/sanitizer.go b/internal/httpclient/sanitizer.go index 46540fd86..542698ba7 100644 --- a/internal/httpclient/sanitizer.go +++ b/internal/httpclient/sanitizer.go @@ -20,48 +20,126 @@ package httpclient import ( "net/netip" "syscall" - - "github.com/superseriousbusiness/gotosocial/internal/netutil" ) -type sanitizer struct { - allow []netip.Prefix - block []netip.Prefix +var ( + // ipv6GlobalUnicast is the prefix set aside by IANA for global unicast assignments, i.e "the internet". + // https://www.iana.org/assignments/ipv6-unicast-address-assignments/ipv6-unicast-address-assignments.xhtml + ipv6GlobalUnicast = netip.MustParsePrefix("2000::/3") + + // ipv6Reserved contains IPv6 reserved IP prefixes that fall within ipv6GlobalUnicast. + // https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml + ipv6Reserved = [...]netip.Prefix{ + netip.MustParsePrefix("2001::/23"), // IETF Protocol Assignments (RFC 2928) + netip.MustParsePrefix("2001:db8::/32"), // Documentation (RFC 3849) + netip.MustParsePrefix("2002::/16"), // 6to4 (RFC 3056) + netip.MustParsePrefix("2620:4f:8000::/48"), // Direct Delegation AS112 Service (RFC 7534) + } + + // ipv4Reserved contains IPv4 reserved IP prefixes. + // https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml + ipv4Reserved = [...]netip.Prefix{ + netip.MustParsePrefix("0.0.0.0/8"), // Current network + netip.MustParsePrefix("10.0.0.0/8"), // Private + netip.MustParsePrefix("100.64.0.0/10"), // RFC6598 + netip.MustParsePrefix("127.0.0.0/8"), // Loopback + netip.MustParsePrefix("169.254.0.0/16"), // Link-local + netip.MustParsePrefix("172.16.0.0/12"), // Private + netip.MustParsePrefix("192.0.0.0/24"), // RFC6890 + netip.MustParsePrefix("192.0.2.0/24"), // Test, doc, examples + netip.MustParsePrefix("192.31.196.0/24"), // AS112-v4, RFC 7535 + netip.MustParsePrefix("192.52.193.0/24"), // AMT, RFC 7450 + netip.MustParsePrefix("192.88.99.0/24"), // IPv6 to IPv4 relay + netip.MustParsePrefix("192.168.0.0/16"), // Private + netip.MustParsePrefix("192.175.48.0/24"), // Direct Delegation AS112 Service, RFC 7534 + netip.MustParsePrefix("198.18.0.0/15"), // Benchmarking tests + netip.MustParsePrefix("198.51.100.0/24"), // Test, doc, examples + netip.MustParsePrefix("203.0.113.0/24"), // Test, doc, examples + netip.MustParsePrefix("224.0.0.0/4"), // Multicast + netip.MustParsePrefix("240.0.0.0/4"), // Reserved (includes broadcast / 255.255.255.255) + } +) + +type Sanitizer struct { + Allow []netip.Prefix + Block []netip.Prefix } // Sanitize implements the required net.Dialer.Control function signature. -func (s *sanitizer) Sanitize(ntwrk, addr string, _ syscall.RawConn) error { +func (s *Sanitizer) Sanitize(ntwrk, addr string, _ syscall.RawConn) error { // Parse IP+port from addr ipport, err := netip.ParseAddrPort(addr) if err != nil { return err } - if !(ntwrk == "tcp4" || ntwrk == "tcp6") { + // Ensure valid network. + const ( + tcp4 = "tcp4" + tcp6 = "tcp6" + ) + + if !(ntwrk == tcp4 || ntwrk == tcp6) { return ErrInvalidNetwork } - // Seperate the IP + // Separate the IP. ip := ipport.Addr() - // Check if this is explicitly allowed - for i := 0; i < len(s.allow); i++ { - if s.allow[i].Contains(ip) { + // Check if this IP is explicitly allowed. + for i := 0; i < len(s.Allow); i++ { + if s.Allow[i].Contains(ip) { return nil } } - // Now check if explicity blocked - for i := 0; i < len(s.block); i++ { - if s.block[i].Contains(ip) { + // Check if this IP is explicitly blocked. + for i := 0; i < len(s.Block); i++ { + if s.Block[i].Contains(ip) { return ErrReservedAddr } } - // Validate this is a safe IP - if !netutil.ValidateIP(ip) { + // Validate this is a safe IP. + if !SafeIP(ip) { return ErrReservedAddr } return nil } + +// SafeIP returns whether ip is an IPv4/6 +// address in a non-reserved, public range. +func SafeIP(ip netip.Addr) bool { + switch { + // IPv4: check if IPv4 in reserved nets + case ip.Is4(): + for _, reserved := range ipv4Reserved { + if reserved.Contains(ip) { + return false + } + } + return true + + // IPv6: check if IP in IPv6 reserved nets + case ip.Is6(): + if !ipv6GlobalUnicast.Contains(ip) { + // Address is not globally routeable, + // ie., not "on the internet". + return false + } + + for _, reserved := range ipv6Reserved { + if reserved.Contains(ip) { + // Address is globally routeable + // but falls in a reserved range. + return false + } + } + return true + + // Assume malicious by default + default: + return false + } +} diff --git a/internal/httpclient/sanitizer_test.go b/internal/httpclient/sanitizer_test.go new file mode 100644 index 000000000..1cb8a7d2e --- /dev/null +++ b/internal/httpclient/sanitizer_test.go @@ -0,0 +1,154 @@ +// GoToSocial +// Copyright (C) GoToSocial Authors admin@gotosocial.org +// SPDX-License-Identifier: AGPL-3.0-or-later +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package httpclient_test + +import ( + "errors" + "net/netip" + "testing" + + "github.com/superseriousbusiness/gotosocial/internal/httpclient" +) + +func TestSafeIP(t *testing.T) { + tests := []struct { + name string + ip netip.Addr + }{ + // IPv4 tests + { + name: "IPv4 this host on this network", + ip: netip.MustParseAddr("0.0.0.0"), + }, + { + name: "IPv4 dummy address", + ip: netip.MustParseAddr("192.0.0.8"), + }, + { + name: "IPv4 Port Control Protocol Anycast", + ip: netip.MustParseAddr("192.0.0.9"), + }, + { + name: "IPv4 Traversal Using Relays around NAT Anycast", + ip: netip.MustParseAddr("192.0.0.10"), + }, + { + name: "IPv4 NAT64/DNS64 Discovery 1", + ip: netip.MustParseAddr("192.0.0.17"), + }, + { + name: "IPv4 NAT64/DNS64 Discovery 2", + ip: netip.MustParseAddr("192.0.0.171"), + }, + // IPv6 tests + { + name: "IPv4-mapped address", + ip: netip.MustParseAddr("::ffff:169.254.169.254"), + }, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if safe := httpclient.SafeIP(tc.ip); safe { + t.Fatalf("Expected IP %s to not safe (%t), got: %t", tc.ip, false, safe) + } + }) + } +} + +func TestSanitizer(t *testing.T) { + s := httpclient.Sanitizer{ + Allow: []netip.Prefix{ + netip.MustParsePrefix("192.0.0.8/32"), + netip.MustParsePrefix("::ffff:169.254.169.254/128"), + }, + Block: []netip.Prefix{ + netip.MustParsePrefix("93.184.216.34/32"), // example.org + }, + } + + tests := []struct { + name string + ntwrk string + addr string + expected error + }{ + // IPv4 tests + { + name: "IPv4 this host on this network", + ntwrk: "tcp4", + addr: "0.0.0.0:80", + expected: httpclient.ErrReservedAddr, + }, + { + name: "IPv4 dummy address", + ntwrk: "tcp4", + addr: "192.0.0.8:80", + expected: nil, // We allowed this explicitly. + }, + { + name: "IPv4 Port Control Protocol Anycast", + ntwrk: "tcp4", + addr: "192.0.0.9:80", + expected: httpclient.ErrReservedAddr, + }, + { + name: "IPv4 Traversal Using Relays around NAT Anycast", + ntwrk: "tcp4", + addr: "192.0.0.10:80", + expected: httpclient.ErrReservedAddr, + }, + { + name: "IPv4 NAT64/DNS64 Discovery 1", + ntwrk: "tcp4", + addr: "192.0.0.17:80", + expected: httpclient.ErrReservedAddr, + }, + { + name: "IPv4 NAT64/DNS64 Discovery 2", + ntwrk: "tcp4", + addr: "192.0.0.171:80", + expected: httpclient.ErrReservedAddr, + }, + { + name: "example.org", + ntwrk: "tcp4", + addr: "93.184.216.34:80", + expected: httpclient.ErrReservedAddr, // We blocked this explicitly. + }, + // IPv6 tests + { + name: "IPv4-mapped address", + ntwrk: "tcp6", + addr: "[::ffff:169.254.169.254]:80", + expected: nil, // We allowed this explicitly. + }, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if err := s.Sanitize(tc.ntwrk, tc.addr, nil); !errors.Is(err, tc.expected) { + t.Fatalf("Expected error %q for addr %s, got: %q", tc.expected, tc.addr, err) + } + }) + } +} diff --git a/internal/netutil/validate.go b/internal/netutil/validate.go deleted file mode 100644 index 798e1fc64..000000000 --- a/internal/netutil/validate.go +++ /dev/null @@ -1,102 +0,0 @@ -// GoToSocial -// Copyright (C) GoToSocial Authors admin@gotosocial.org -// SPDX-License-Identifier: AGPL-3.0-or-later -// -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU Affero General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Affero General Public License for more details. -// -// You should have received a copy of the GNU Affero General Public License -// along with this program. If not, see . - -package netutil - -import ( - "net/netip" -) - -var ( - // IPv6Reserved contains IPv6 reserved IP prefixes. - // https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml - IPv6Reserved = [...]netip.Prefix{ - netip.MustParsePrefix("::1/128"), // Loopback - netip.MustParsePrefix("::/128"), // Unspecified address - netip.MustParsePrefix("::ffff:0:0/96"), // IPv4-mapped address - netip.MustParsePrefix("64:ff9b::/96"), // IPv4/IPv6 translation, RFC 6052 - netip.MustParsePrefix("64:ff9b:1::/48"), // IPv4/IPv6 translation, RFC 8215 - netip.MustParsePrefix("100::/64"), // Discard prefix, RFC 6666 - netip.MustParsePrefix("2001::/23"), // IETF Protocol Assignments, RFC 2928 - netip.MustParsePrefix("2001:db8::/32"), // Test, doc, examples - netip.MustParsePrefix("2002::/16"), // 6to4 - netip.MustParsePrefix("2620:4f:8000::/48"), // Direct Delegation AS112 Service, RFC 7534 - netip.MustParsePrefix("fc00::/7"), // Unique Local - netip.MustParsePrefix("fe80::/10"), // Link-local - netip.MustParsePrefix("fec0::/10"), // Site-local, deprecated - netip.MustParsePrefix("ff00::/8"), // Multicast - } - - // IPv4Reserved contains IPv4 reserved IP prefixes. - // https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml - IPv4Reserved = [...]netip.Prefix{ - netip.MustParsePrefix("0.0.0.0/8"), // Current network - netip.MustParsePrefix("10.0.0.0/8"), // Private - netip.MustParsePrefix("100.64.0.0/10"), // RFC6598 - netip.MustParsePrefix("127.0.0.0/8"), // Loopback - netip.MustParsePrefix("169.254.0.0/16"), // Link-local - netip.MustParsePrefix("172.16.0.0/12"), // Private - netip.MustParsePrefix("192.0.0.0/24"), // RFC6890 - netip.MustParsePrefix("192.0.2.0/24"), // Test, doc, examples - netip.MustParsePrefix("192.31.196.0/24"), // AS112-v4, RFC 7535 - netip.MustParsePrefix("192.52.193.0/24"), // AMT, RFC 7450 - netip.MustParsePrefix("192.88.99.0/24"), // IPv6 to IPv4 relay - netip.MustParsePrefix("192.168.0.0/16"), // Private - netip.MustParsePrefix("192.175.48.0/24"), // Direct Delegation AS112 Service, RFC 7534 - netip.MustParsePrefix("198.18.0.0/15"), // Benchmarking tests - netip.MustParsePrefix("198.51.100.0/24"), // Test, doc, examples - netip.MustParsePrefix("203.0.113.0/24"), // Test, doc, examples - netip.MustParsePrefix("224.0.0.0/4"), // Multicast - netip.MustParsePrefix("240.0.0.0/4"), // Reserved (includes broadcast / 255.255.255.255) - } -) - -// ValidateAddr will parse a netip.AddrPort from string, and return the result of ValidateIP() on addr. -func ValidateAddr(s string) bool { - ipport, err := netip.ParseAddrPort(s) - if err != nil { - return false - } - return ValidateIP(ipport.Addr()) -} - -// ValidateIP returns whether IP is an IPv4/6 address in non-reserved, public ranges. -func ValidateIP(ip netip.Addr) bool { - switch { - // IPv4: check if IPv4 in reserved nets - case ip.Is4(): - for _, reserved := range IPv4Reserved { - if reserved.Contains(ip) { - return false - } - } - return true - - // IPv6: check if IP in IPv6 reserved nets - case ip.Is6(): - for _, reserved := range IPv6Reserved { - if reserved.Contains(ip) { - return false - } - } - return true - - // Assume malicious by default - default: - return false - } -} diff --git a/internal/netutil/validate_test.go b/internal/netutil/validate_test.go deleted file mode 100644 index 37def4ce6..000000000 --- a/internal/netutil/validate_test.go +++ /dev/null @@ -1,54 +0,0 @@ -package netutil - -import ( - "net/netip" - "testing" -) - -func TestValidateIP(t *testing.T) { - tests := []struct { - name string - ip netip.Addr - }{ - // IPv4 tests - { - name: "IPv4 this host on this network", - ip: netip.MustParseAddr("0.0.0.0"), - }, - { - name: "IPv4 dummy address", - ip: netip.MustParseAddr("192.0.0.8"), - }, - { - name: "IPv4 Port Control Protocol Anycast", - ip: netip.MustParseAddr("192.0.0.9"), - }, - { - name: "IPv4 Traversal Using Relays around NAT Anycast", - ip: netip.MustParseAddr("192.0.0.10"), - }, - { - name: "IPv4 NAT64/DNS64 Discovery 1", - ip: netip.MustParseAddr("192.0.0.17"), - }, - { - name: "IPv4 NAT64/DNS64 Discovery 2", - ip: netip.MustParseAddr("192.0.0.171"), - }, - // IPv6 tests - { - name: "IPv4-mapped address", - ip: netip.MustParseAddr("::ffff:169.254.169.254"), - }, - } - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - if valid := ValidateIP(tc.ip); valid != false { - t.Fatalf("Expected IP %s to be: %t, got: %t", tc.ip, false, valid) - } - }) - } -} diff --git a/mkdocs.yml b/mkdocs.yml index 593cb6526..d7e4fcda9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -81,6 +81,7 @@ nav: - "configuration/oidc.md" - "configuration/smtp.md" - "configuration/syslog.md" + - "configuration/httpclient.md" - "configuration/advanced.md" - "configuration/observability.md" - "Advanced": diff --git a/test/envparsing.sh b/test/envparsing.sh index eb6b8da1a..9b56dd9ca 100755 --- a/test/envparsing.sh +++ b/test/envparsing.sh @@ -98,6 +98,11 @@ EXPECT=$(cat <<"EOF" "dry-run": true, "email": "", "host": "example.com", + "http-client": { + "allow-ips": [], + "block-ips": [], + "timeout": 10000000000 + }, "instance-deliver-to-shared-inboxes": false, "instance-expose-peers": true, "instance-expose-public-timeline": true,