mirrors/gotosocial
1
0
Fork 0
mirror of https://github.com/superseriousbusiness/gotosocial.git synced 2024-11-21 16:55:38 +03:00
Code Issues Projects Releases Packages Wiki Activity

[chore] Webfinger rework (#627)

Browse source 
* move finger to dereferencer

* totally break GetRemoteAccount

* start reworking finger func a bit

* start reworking getRemoteAccount a bit

* move mention parts to namestring

* rework webfingerget

* use util function to extract webfinger parts

* use accountDomain

* rework finger again, final form

* just a real nasty commit, the worst

* remove refresh from account

* use new ASRepToAccount signature

* fix incorrect debug call

* fix for new getRemoteAccount

* rework GetRemoteAccount

* start updating tests to remove repetition

* break a lot of tests
Move shared test logic into the testrig,
rather than having it scattered all over
the place. This allows us to just mock
the transport controller once, and have
all tests use it (unless they need not to
for some other reason).

* fix up tests to use main mock httpclient

* webfinger only if necessary

* cheeky linting with the lads

* update mentionName regex
recognize instance accounts

* don't finger instance accounts

* test webfinger part extraction

* increase default worker count to 4 per cpu

* don't repeat regex parsing

* final search for discovered accountDomain

* be more permissive in namestring lookup

* add more extraction tests

* simplify GetParseMentionFunc

* skip long search if local account

* fix broken test
This commit is contained in:
tobi 2022-06-11 11:01:34 +02:00 committed by GitHub
parent 694a490589
commit dfdc473cef
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
62 changed files with 993 additions and 824 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cmd/gotosocial/action/testrig/testrig.go
View file

@ -85,7 +85,7 @@ var Start action.GTSAction = func(ctx context.Context) error {
StatusCode: 200,
Body: r,
}, nil
}), dbService, fedWorker)
}, ""), dbService, fedWorker)
mediaManager := testrig.NewTestMediaManager(dbService, storageBackend)
federator := testrig.NewTestFederator(dbService, transportController, storageBackend, mediaManager, fedWorker)

2
internal/ap/extract.go
View file

@ -574,7 +574,7 @@ func ExtractMention(i Mentionable) (*gtsmodel.Mention, error) {
}
// just make sure the mention string is valid so we can handle it properly later on...
_, _, err = util.ExtractMentionParts(mentionString)
_, _, err = util.ExtractNamestringParts(mentionString)
if err != nil {
return nil, err
}

2
internal/api/client/account/account_test.go
View file

@ -67,7 +67,7 @@ func (suite *AccountStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.sentEmails = make(map[string]string)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)

2
internal/api/client/admin/admin_test.go
View file

@ -85,7 +85,7 @@ func (suite *AdminStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.sentEmails = make(map[string]string)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)

2
internal/api/client/auth/auth_test.go
View file

@ -90,7 +90,7 @@ func (suite *AuthStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)

2
internal/api/client/fileserver/servefile_test.go
View file

@ -82,7 +82,7 @@ func (suite *ServeFileTestSuite) SetupSuite() {
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, testrig.NewTestMediaManager(suite.db, suite.storage), fedWorker)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, testrig.NewTestMediaManager(suite.db, suite.storage), clientWorker, fedWorker)

2
internal/api/client/followrequest/followrequest_test.go
View file

@ -82,7 +82,7 @@ func (suite *FollowRequestStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)
suite.followRequestModule = followrequest.New(suite.processor).(*followrequest.Module)

2
internal/api/client/instance/instance_test.go
View file

@ -85,7 +85,7 @@ func (suite *InstanceStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.sentEmails = make(map[string]string)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)

2
internal/api/client/media/mediacreate_test.go
View file

@ -91,7 +91,7 @@ func (suite *MediaCreateTestSuite) SetupSuite() {
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)

2
internal/api/client/media/mediaupdate_test.go
View file

@ -89,7 +89,7 @@ func (suite *MediaUpdateTestSuite) SetupSuite() {
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)

97
internal/api/client/status/status_test.go
View file

@ -19,18 +19,8 @@
package status_test
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/activity/pub"
"github.com/superseriousbusiness/activity/streams"
"github.com/superseriousbusiness/gotosocial/internal/api/client/status"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
"github.com/superseriousbusiness/gotosocial/internal/db"
@ -94,7 +84,7 @@ func (suite *StatusStandardTestSuite) SetupTest() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(suite.testHttpClient(), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)
suite.statusModule = status.New(suite.processor).(*status.Module)
@ -104,88 +94,3 @@ func (suite *StatusStandardTestSuite) TearDownTest() {
testrig.StandardDBTeardown(suite.db)
testrig.StandardStorageTeardown(suite.storage)
}
func (suite *StatusStandardTestSuite) testHttpClient() pub.HttpClient {
remoteAccount := suite.testAccounts["remote_account_1"]
remoteAccountNamestring := fmt.Sprintf("acct:%s@%s", remoteAccount.Username, remoteAccount.Domain)
remoteAccountWebfingerURI := fmt.Sprintf("https://%s/.well-known/webfinger?resource=%s", remoteAccount.Domain, remoteAccountNamestring)
fmt.Println(remoteAccountWebfingerURI)
httpClient := testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
// respond correctly to a webfinger lookup
if req.URL.String() == remoteAccountWebfingerURI {
responseJson := fmt.Sprintf(`
{
"subject": "%s",
"aliases": [
"%s",
"%s"
],
"links": [
{
"rel": "http://webfinger.net/rel/profile-page",
"type": "text/html",
"href": "%s"
},
{
"rel": "self",
"type": "application/activity+json",
"href": "%s"
}
]
}`, remoteAccountNamestring, remoteAccount.URI, remoteAccount.URL, remoteAccount.URL, remoteAccount.URI)
responseType := "application/json"
reader := bytes.NewReader([]byte(responseJson))
readCloser := io.NopCloser(reader)
response := &http.Response{
StatusCode: 200,
Body: readCloser,
ContentLength: int64(len(responseJson)),
Header: http.Header{
"content-type": {responseType},
},
}
return response, nil
}
// respond correctly to an account dereference
if req.URL.String() == remoteAccount.URI {
satanAS, err := suite.tc.AccountToAS(context.Background(), remoteAccount)
if err != nil {
panic(err)
}
satanI, err := streams.Serialize(satanAS)
if err != nil {
panic(err)
}
satanJson, err := json.Marshal(satanI)
if err != nil {
panic(err)
}
responseType := "application/activity+json"
reader := bytes.NewReader(satanJson)
readCloser := io.NopCloser(reader)
response := &http.Response{
StatusCode: 200,
Body: readCloser,
ContentLength: int64(len(satanJson)),
Header: http.Header{
"content-type": {responseType},
},
}
return response, nil
}
r := ioutil.NopCloser(bytes.NewReader([]byte{}))
return &http.Response{
StatusCode: 200,
Body: r,
}, nil
})
return httpClient
}

4
internal/api/client/status/statuscreate_test.go
View file

@ -128,7 +128,7 @@ func (suite *StatusCreateTestSuite) TestMentionUnknownAccount() {
ctx.Request = httptest.NewRequest(http.MethodPost, fmt.Sprintf("http://localhost:8080/%s", status.BasePath), nil) // the endpoint we're hitting
ctx.Request.Header.Set("accept", "application/json")
ctx.Request.Form = url.Values{
"status": {"hello @foss_satan@fossbros-anonymous.io"},
"status": {"hello @brand_new_person@unknown-instance.com"},
"visibility": {string(model.VisibilityPublic)},
}
suite.statusModule.StatusCreatePOSTHandler(ctx)
@ -145,7 +145,7 @@ func (suite *StatusCreateTestSuite) TestMentionUnknownAccount() {
suite.NoError(err)
// if the status is properly formatted, that means the account has been put in the db
suite.Equal("<p>hello <span class=\"h-card\"><a href=\"http://fossbros-anonymous.io/@foss_satan\" class=\"u-url mention\" rel=\"nofollow noreferrer noopener\" target=\"_blank\">@<span>foss_satan</span></a></span></p>", statusReply.Content)
suite.Equal(`<p>hello <span class="h-card"><a href="https://unknown-instance.com/@brand_new_person" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>brand_new_person</span></a></span></p>`, statusReply.Content)
suite.Equal(model.VisibilityPublic, statusReply.Visibility)
}

2
internal/api/client/user/user_test.go
View file

@ -69,7 +69,7 @@ func (suite *UserStandardTestSuite) SetupTest() {
suite.storage = testrig.NewTestStorage()
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.sentEmails = make(map[string]string)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", suite.sentEmails)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)

8
internal/api/s2s/user/inboxpost_test.go
View file

@ -88,7 +88,7 @@ func (suite *InboxPostTestSuite) TestPostBlock() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
@ -191,7 +191,7 @@ func (suite *InboxPostTestSuite) TestPostUnblock() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
@ -284,7 +284,7 @@ func (suite *InboxPostTestSuite) TestPostUpdate() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
@ -406,7 +406,7 @@ func (suite *InboxPostTestSuite) TestPostDelete() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)

6
internal/api/s2s/user/outboxget_test.go
View file

@ -49,7 +49,7 @@ func (suite *OutboxGetTestSuite) TestGetOutbox() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
@ -107,7 +107,7 @@ func (suite *OutboxGetTestSuite) TestGetOutboxFirstPage() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
@ -165,7 +165,7 @@ func (suite *OutboxGetTestSuite) TestGetOutboxNextPage() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)

6
internal/api/s2s/user/repliesget_test.go
View file

@ -52,7 +52,7 @@ func (suite *RepliesGetTestSuite) TestGetReplies() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
@ -116,7 +116,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesNext() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
@ -183,7 +183,7 @@ func (suite *RepliesGetTestSuite) TestGetRepliesLast() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)

4
internal/api/s2s/user/statusget_test.go
View file

@ -51,7 +51,7 @@ func (suite *StatusGetTestSuite) TestGetStatus() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
@ -119,7 +119,7 @@ func (suite *StatusGetTestSuite) TestGetStatusLowercase() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)

2
internal/api/s2s/user/user_test.go
View file

@ -85,7 +85,7 @@ func (suite *UserStandardTestSuite) SetupTest() {
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.storage = testrig.NewTestStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)
suite.userModule = user.New(suite.processor).(*user.Module)

8
internal/api/s2s/user/userget_test.go
View file

@ -52,7 +52,7 @@ func (suite *UserGetTestSuite) TestGetUser() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
@ -102,7 +102,7 @@ func (suite *UserGetTestSuite) TestGetUser() {
// convert person to account
// since this account is already known, we should get a pretty full model of it from the conversion
a, err := suite.tc.ASRepresentationToAccount(context.Background(), person, false)
a, err := suite.tc.ASRepresentationToAccount(context.Background(), person, "", false)
suite.NoError(err)
suite.EqualValues(targetAccount.Username, a.Username)
}
@ -133,7 +133,7 @@ func (suite *UserGetTestSuite) TestGetUserPublicKeyDeleted() {
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker)
federator := testrig.NewTestFederator(suite.db, tc, suite.storage, suite.mediaManager, fedWorker)
emailSender := testrig.NewEmailSender("../../../../web/template/", nil)
processor := testrig.NewTestProcessor(suite.db, suite.storage, federator, emailSender, suite.mediaManager, clientWorker, fedWorker)
@ -182,7 +182,7 @@ func (suite *UserGetTestSuite) TestGetUserPublicKeyDeleted() {
suite.True(ok)
// convert person to account
a, err := suite.tc.ASRepresentationToAccount(context.Background(), person, false)
a, err := suite.tc.ASRepresentationToAccount(context.Background(), person, "", false)
suite.NoError(err)
suite.EqualValues(targetAccount.Username, a.Username)
}

2
internal/api/s2s/webfinger/webfinger_test.go
View file

@ -88,7 +88,7 @@ func (suite *WebfingerStandardTestSuite) SetupTest() {
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.storage = testrig.NewTestStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.federator = testrig.NewTestFederator(suite.db, testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../../testrig/media"), suite.db, fedWorker), suite.storage, suite.mediaManager, fedWorker)
suite.emailSender = testrig.NewEmailSender("../../../../web/template/", nil)
suite.processor = testrig.NewTestProcessor(suite.db, suite.storage, suite.federator, suite.emailSender, suite.mediaManager, clientWorker, fedWorker)
suite.webfingerModule = webfinger.New(suite.processor).(*webfinger.Module)

34
internal/api/s2s/webfinger/webfingerget.go
View file

@ -22,13 +22,13 @@ import (
"context"
"fmt"
"net/http"
"strings"
"github.com/gin-gonic/gin"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/api"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
// WebfingerGETRequest swagger:operation GET /.well-known/webfinger webfingerGet
@ -67,35 +67,19 @@ func (m *Module) WebfingerGETRequest(c *gin.Context) {
return
}
// remove the acct: prefix if it's present
trimAcct := strings.TrimPrefix(resourceQuery, "acct:")
// remove the first @ in @whatever@example.org if it's present
namestring := strings.TrimPrefix(trimAcct, "@")
// at this point we should have a string like some_user@example.org
l.Debugf("got finger request for '%s'", namestring)
usernameAndAccountDomain := strings.Split(namestring, "@")
if len(usernameAndAccountDomain) != 2 {
l.Debugf("aborting request because username and domain could not be parsed from %s", namestring)
c.JSON(http.StatusBadRequest, gin.H{"error": "bad request"})
return
}
username := strings.ToLower(usernameAndAccountDomain[0])
requestedAccountDomain := strings.ToLower(usernameAndAccountDomain[1])
if username == "" || requestedAccountDomain == "" {
l.Debug("aborting request because username or domain was empty")
c.JSON(http.StatusBadRequest, gin.H{"error": "bad request"})
requestedUsername, requestedHost, err := util.ExtractWebfingerParts(resourceQuery)
if err != nil {
l.Debugf("bad webfinger request with resource query %s: %s", resourceQuery, err)
c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("bad webfinger request with resource query %s", resourceQuery)})
return
}
accountDomain := config.GetAccountDomain()
host := config.GetHost()
if requestedAccountDomain != accountDomain && requestedAccountDomain != host {
l.Debugf("aborting request because accountDomain %s does not belong to this instance", requestedAccountDomain)
c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("accountDomain %s does not belong to this instance", requestedAccountDomain)})
if requestedHost != host && requestedHost != accountDomain {
l.Debugf("aborting request because requestedHost %s does not belong to this instance", requestedHost)
c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("requested host %s does not belong to this instance", requestedHost)})
return
}
@ -106,7 +90,7 @@ func (m *Module) WebfingerGETRequest(c *gin.Context) {
ctx = context.WithValue(ctx, ap.ContextRequestingPublicKeyVerifier, verifier)
}
resp, errWithCode := m.processor.GetWebfingerAccount(ctx, username)
resp, errWithCode := m.processor.GetWebfingerAccount(ctx, requestedUsername)
if errWithCode != nil {
api.ErrorHandler(c, errWithCode, m.processor.InstanceGet)
return

2
internal/concurrency/workers.go
View file

@ -45,7 +45,7 @@ func NewWorkerPool[MsgType any](workers int, queueRatio int) *WorkerPool[MsgType
if workers < 1 {
// ensure sensible workers
workers = runtime.GOMAXPROCS(0) * 2
workers = runtime.GOMAXPROCS(0) * 4
}
if queueRatio < 1 {
// ensure sensible ratio

5
internal/federation/dereference.go
View file

@ -23,11 +23,12 @@ import (
"net/url"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
func (f *federator) GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error) {
return f.dereferencer.GetRemoteAccount(ctx, username, remoteAccountID, blocking, refresh)
func (f *federator) GetRemoteAccount(ctx context.Context, params dereferencing.GetRemoteAccountParams) (*gtsmodel.Account, error) {
return f.dereferencer.GetRemoteAccount(ctx, params)
}
func (f *federator) GetRemoteStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error) {

294
internal/federation/dereferencing/account.go
View file

@ -33,12 +33,15 @@ import (
"github.com/superseriousbusiness/activity/streams"
"github.com/superseriousbusiness/activity/streams/vocab"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/media"
"github.com/superseriousbusiness/gotosocial/internal/transport"
)
var webfingerInterval = -48 * time.Hour // 2 days in the past
func instanceAccount(account *gtsmodel.Account) bool {
return strings.EqualFold(account.Username, account.Domain) ||
account.FollowersURI == "" ||
@ -46,97 +49,238 @@ func instanceAccount(account *gtsmodel.Account) bool {
(account.Username == "internal.fetch" && strings.Contains(account.Note, "internal service actor"))
}
// GetRemoteAccountParams wraps parameters for a remote account lookup.
type GetRemoteAccountParams struct {
// The username of the user doing the lookup request (optional).
// If not set, then the GtS instance account will be used to do the lookup.
RequestingUsername string
// The ActivityPub URI of the remote account (optional).
// If not set (nil), the ActivityPub URI of the remote account will be discovered
// via webfinger, so you must set RemoteAccountUsername and RemoteAccountHost
// if this parameter is not set.
RemoteAccountID *url.URL
// The username of the remote account (optional).
// If RemoteAccountID is not set, then this value must be set.
RemoteAccountUsername string
// The host of the remote account (optional).
// If RemoteAccountID is not set, then this value must be set.
RemoteAccountHost string
// Whether to do a blocking call to the remote instance. If true,
// then the account's media and other fields will be fully dereferenced before it is returned.
// If false, then the account's media and other fields will be dereferenced in the background,
// so only a minimal account representation will be returned by GetRemoteAccount.
Blocking bool
// Whether to skip making calls to remote instances. This is useful when you want to
// quickly fetch a remote account from the database or fail, and don't want to cause
// http requests to go flying around.
SkipResolve bool
}
// GetRemoteAccount completely dereferences a remote account, converts it to a GtS model account,
// puts it in the database, and returns it to a caller.
//
// Refresh indicates whether--if the account exists in our db already--it should be refreshed by calling
// the remote instance again. Blocking indicates whether the function should block until processing of
// the fetched account is complete.
//
// SIDE EFFECTS: remote account will be stored in the database, or updated if it already exists (and refresh is true).
func (d *deref) GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error) {
new := true
// puts or updates it in the database (if necessary), and returns it to a caller.
func (d *deref) GetRemoteAccount(ctx context.Context, params GetRemoteAccountParams) (remoteAccount *gtsmodel.Account, err error) {
// check if we already have the account in our db, and just return it unless we'd doing a refresh
remoteAccount, err := d.db.GetAccountByURI(ctx, remoteAccountID.String())
if err == nil {
new = false
if !refresh {
// make sure the account fields are populated before returning:
// even if we're not doing a refresh, the caller might want to block
// until everything is loaded
changed, err := d.populateAccountFields(ctx, remoteAccount, username, refresh, blocking)
/*
In this function we want to retrieve a gtsmodel representation of a remote account, with its proper
accountDomain set, while making as few calls to remote instances as possible to save time and bandwidth.
There are a few different paths through this function, and the path taken depends on how much
initial information we are provided with via parameters, how much information we already have stored,
and what we're allowed to do according to the parameters we've been passed.
Scenario 1: We're not allowed to resolve remotely, but we've got either the account URI or the
account username + host, so we can check in our database and return if possible.
Scenario 2: We are allowed to resolve remotely, and we have an account URI but no username or host.
In this case, we can use the URI to resolve the remote account and find the username,
and then we can webfinger the account to discover the accountDomain if necessary.
Scenario 3: We are allowed to resolve remotely, and we have the username and host but no URI.
In this case, we can webfinger the account to discover the URI, and then dereference
from that.
*/
// first check if we can retrieve the account locally just with what we've been given
switch {
case params.RemoteAccountID != nil:
// try with uri
if a, dbErr := d.db.GetAccountByURI(ctx, params.RemoteAccountID.String()); dbErr == nil {
remoteAccount = a
} else if dbErr != db.ErrNoEntries {
err = fmt.Errorf("GetRemoteAccount: database error looking for account %s: %s", params.RemoteAccountID, err)
}
case params.RemoteAccountUsername != "" && params.RemoteAccountHost != "":
// try with username/host
a := &gtsmodel.Account{}
where := []db.Where{{Key: "username", Value: params.RemoteAccountUsername}, {Key: "domain", Value: params.RemoteAccountHost}}
if dbErr := d.db.GetWhere(ctx, where, a); dbErr == nil {
remoteAccount = a
} else if dbErr != db.ErrNoEntries {
err = fmt.Errorf("GetRemoteAccount: database error looking for account with username %s and host %s: %s", params.RemoteAccountUsername, params.RemoteAccountHost, err)
}
default:
err = errors.New("GetRemoteAccount: no identifying parameters were set so we cannot get account")
}
if err != nil {
return
}
if params.SkipResolve {
// if we can't resolve, return already since there's nothing more we can do
if remoteAccount == nil {
err = errors.New("GetRemoteAccount: error retrieving account with skipResolve set true")
}
return
}
var accountable ap.Accountable
if params.RemoteAccountUsername == "" || params.RemoteAccountHost == "" {
// try to populate the missing params
// the first one is easy ...
params.RemoteAccountHost = params.RemoteAccountID.Host
// ... but we still need the username so we can do a finger for the accountDomain
// check if we had the account stored already and got it earlier
if remoteAccount != nil {
params.RemoteAccountUsername = remoteAccount.Username
} else {
// if we didn't already have it, we have dereference it from remote and just...
accountable, err = d.dereferenceAccountable(ctx, params.RequestingUsername, params.RemoteAccountID)
if err != nil {
return nil, fmt.Errorf("GetRemoteAccount: error populating remoteAccount fields: %s", err)
err = fmt.Errorf("GetRemoteAccount: error dereferencing accountable: %s", err)
return
}
if changed {
updatedAccount, err := d.db.UpdateAccount(ctx, remoteAccount)
if err != nil {
return nil, fmt.Errorf("GetRemoteAccount: error updating remoteAccount: %s", err)
}
return updatedAccount, err
// ... take the username (for now)
params.RemoteAccountUsername, err = ap.ExtractPreferredUsername(accountable)
if err != nil {
err = fmt.Errorf("GetRemoteAccount: error extracting accountable username: %s", err)
return
}
return remoteAccount, nil
}
}
if new {
// we haven't seen this account before: dereference it from remote
accountable, err := d.dereferenceAccountable(ctx, username, remoteAccountID)
if err != nil {
return nil, fmt.Errorf("GetRemoteAccount: error dereferencing accountable: %s", err)
}
// if we reach this point, params.RemoteAccountHost and params.RemoteAccountUsername must be set
// params.RemoteAccountID may or may not be set, but we have enough information to fetch it if we need it
newAccount, err := d.typeConverter.ASRepresentationToAccount(ctx, accountable, refresh)
if err != nil {
return nil, fmt.Errorf("GetRemoteAccount: error converting accountable to account: %s", err)
}
ulid, err := id.NewRandomULID()
if err != nil {
return nil, fmt.Errorf("GetRemoteAccount: error generating new id for account: %s", err)
}
newAccount.ID = ulid
if _, err := d.populateAccountFields(ctx, newAccount, username, refresh, blocking); err != nil {
return nil, fmt.Errorf("GetRemoteAccount: error populating further account fields: %s", err)
}
if err := d.db.Put(ctx, newAccount); err != nil {
return nil, fmt.Errorf("GetRemoteAccount: error putting new account: %s", err)
}
return newAccount, nil
// we finger to fetch the account domain but just in case we're not fingering, make a best guess
// already about what the account domain might be; this var will be overwritten later if necessary
var accountDomain string
switch {
case remoteAccount != nil:
accountDomain = remoteAccount.Domain
case params.RemoteAccountID != nil:
accountDomain = params.RemoteAccountID.Host
default:
accountDomain = params.RemoteAccountHost
}
// we have seen this account before, but we have to refresh it
refreshedAccountable, err := d.dereferenceAccountable(ctx, username, remoteAccountID)
// to save on remote calls: only webfinger if we don't have a remoteAccount yet, or if we haven't
// fingered the remote account for at least 2 days; don't finger instance accounts
var fingered time.Time
if remoteAccount == nil || (remoteAccount.LastWebfingeredAt.Before(time.Now().Add(webfingerInterval)) && !instanceAccount(remoteAccount)) {
accountDomain, params.RemoteAccountID, err = d.fingerRemoteAccount(ctx, params.RequestingUsername, params.RemoteAccountUsername, params.RemoteAccountHost)
if err != nil {
err = fmt.Errorf("GetRemoteAccount: error while fingering: %s", err)
return
}
fingered = time.Now()
}
if !fingered.IsZero() && remoteAccount == nil {
// if we just fingered and now have a discovered account domain but still no account,
// we should do a final lookup in the database with the discovered username + accountDomain
// to make absolutely sure we don't already have this account
a := &gtsmodel.Account{}
where := []db.Where{{Key: "username", Value: params.RemoteAccountUsername}, {Key: "domain", Value: accountDomain}}
if dbErr := d.db.GetWhere(ctx, where, a); dbErr == nil {
remoteAccount = a
} else if dbErr != db.ErrNoEntries {
err = fmt.Errorf("GetRemoteAccount: database error looking for account with username %s and host %s: %s", params.RemoteAccountUsername, params.RemoteAccountHost, err)
return
}
}
// we may also have some extra information already, like the account we had in the db, or the
// accountable representation that we dereferenced from remote
if remoteAccount == nil {
// we still don't have the account, so deference it if we didn't earlier
if accountable == nil {
accountable, err = d.dereferenceAccountable(ctx, params.RequestingUsername, params.RemoteAccountID)
if err != nil {
err = fmt.Errorf("GetRemoteAccount: error dereferencing accountable: %s", err)
return
}
}
// then convert
remoteAccount, err = d.typeConverter.ASRepresentationToAccount(ctx, accountable, accountDomain, false)
if err != nil {
err = fmt.Errorf("GetRemoteAccount: error converting accountable to account: %s", err)
return
}
// this is a new account so we need to generate a new ID for it
var ulid string
ulid, err = id.NewRandomULID()
if err != nil {
err = fmt.Errorf("GetRemoteAccount: error generating new id for account: %s", err)
return
}
remoteAccount.ID = ulid
_, err = d.populateAccountFields(ctx, remoteAccount, params.RequestingUsername, params.Blocking)
if err != nil {
err = fmt.Errorf("GetRemoteAccount: error populating further account fields: %s", err)
return
}
remoteAccount.LastWebfingeredAt = fingered
remoteAccount.UpdatedAt = time.Now()
err = d.db.Put(ctx, remoteAccount)
if err != nil {
err = fmt.Errorf("GetRemoteAccount: error putting new account: %s", err)
return
}
return // the new account
}
// we had the account already, but now we know the account domain, so update it if it's different
if !strings.EqualFold(remoteAccount.Domain, accountDomain) {
remoteAccount.Domain = accountDomain
remoteAccount, err = d.db.UpdateAccount(ctx, remoteAccount)
if err != nil {
err = fmt.Errorf("GetRemoteAccount: error updating account: %s", err)
return
}
}
// make sure the account fields are populated before returning:
// the caller might want to block until everything is loaded
var fieldsChanged bool
fieldsChanged, err = d.populateAccountFields(ctx, remoteAccount, params.RequestingUsername, params.Blocking)
if err != nil {
return nil, fmt.Errorf("GetRemoteAccount: error dereferencing refreshedAccountable: %s", err)
return nil, fmt.Errorf("GetRemoteAccount: error populating remoteAccount fields: %s", err)
}
refreshedAccount, err := d.typeConverter.ASRepresentationToAccount(ctx, refreshedAccountable, refresh)
if err != nil {
return nil, fmt.Errorf("GetRemoteAccount: error converting refreshedAccountable to refreshedAccount: %s", err)
}
refreshedAccount.ID = remoteAccount.ID
changed, err := d.populateAccountFields(ctx, refreshedAccount, username, refresh, blocking)
if err != nil {
return nil, fmt.Errorf("GetRemoteAccount: error populating further refreshedAccount fields: %s", err)
var fingeredChanged bool
if !fingered.IsZero() {
fingeredChanged = true
remoteAccount.LastWebfingeredAt = fingered
}
if changed {
updatedAccount, err := d.db.UpdateAccount(ctx, refreshedAccount)
if fieldsChanged || fingeredChanged {
remoteAccount.UpdatedAt = time.Now()
remoteAccount, err = d.db.UpdateAccount(ctx, remoteAccount)
if err != nil {
return nil, fmt.Errorf("GetRemoteAccount: error updating refreshedAccount: %s", err)
return nil, fmt.Errorf("GetRemoteAccount: error updating remoteAccount: %s", err)
}
return updatedAccount, nil
}
return refreshedAccount, nil
return // the account we already had + possibly updated
}
// dereferenceAccountable calls remoteAccountID with a GET request, and tries to parse whatever
@ -209,7 +353,7 @@ func (d *deref) dereferenceAccountable(ctx context.Context, username string, rem
// populateAccountFields populates any fields on the given account that weren't populated by the initial
// dereferencing. This includes things like header and avatar etc.
func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Account, requestingUsername string, blocking bool, refresh bool) (bool, error) {
func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Account, requestingUsername string, blocking bool) (bool, error) {
// if we're dealing with an instance account, just bail, we don't need to do anything
if instanceAccount(account) {
return false, nil
@ -230,7 +374,7 @@ func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Acc
}
// fetch the header and avatar
changed, err := d.fetchRemoteAccountMedia(ctx, account, t, refresh, blocking)
changed, err := d.fetchRemoteAccountMedia(ctx, account, t, blocking)
if err != nil {
return false, fmt.Errorf("populateAccountFields: error fetching header/avi for account: %s", err)
}
@ -250,7 +394,7 @@ func (d *deref) populateAccountFields(ctx context.Context, account *gtsmodel.Acc
//
// If blocking is true, then the calls to the media manager made by this function will be blocking:
// in other words, the function won't return until the header and the avatar have been fully processed.
func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, t transport.Transport, blocking bool, refresh bool) (bool, error) {
func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsmodel.Account, t transport.Transport, blocking bool) (bool, error) {
changed := false
accountURI, err := url.Parse(targetAccount.URI)
@ -262,7 +406,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm
return changed, fmt.Errorf("fetchRemoteAccountMedia: domain %s is blocked", accountURI.Host)
}
if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "" || refresh) {
if targetAccount.AvatarRemoteURL != "" && (targetAccount.AvatarMediaAttachmentID == "") {
var processingMedia *media.ProcessingMedia
d.dereferencingAvatarsLock.Lock() // LOCK HERE
@ -320,7 +464,7 @@ func (d *deref) fetchRemoteAccountMedia(ctx context.Context, targetAccount *gtsm
changed = true
}
if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "" || refresh) {
if targetAccount.HeaderRemoteURL != "" && (targetAccount.HeaderMediaAttachmentID == "") {
var processingMedia *media.ProcessingMedia
d.dereferencingHeadersLock.Lock() // LOCK HERE

12
internal/federation/dereferencing/account_test.go
View file

@ -24,6 +24,7 @@ import (
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@ -35,7 +36,10 @@ func (suite *AccountTestSuite) TestDereferenceGroup() {
fetchingAccount := suite.testAccounts["local_account_1"]
groupURL := testrig.URLMustParse("https://unknown-instance.com/groups/some_group")
group, err := suite.dereferencer.GetRemoteAccount(context.Background(), fetchingAccount.Username, groupURL, false, false)
group, err := suite.dereferencer.GetRemoteAccount(context.Background(), dereferencing.GetRemoteAccountParams{
RequestingUsername: fetchingAccount.Username,
RemoteAccountID: groupURL,
})
suite.NoError(err)
suite.NotNil(group)
suite.NotNil(group)
@ -55,7 +59,10 @@ func (suite *AccountTestSuite) TestDereferenceService() {
fetchingAccount := suite.testAccounts["local_account_1"]
serviceURL := testrig.URLMustParse("https://owncast.example.org/federation/user/rgh")
service, err := suite.dereferencer.GetRemoteAccount(context.Background(), fetchingAccount.Username, serviceURL, false, false)
service, err := suite.dereferencer.GetRemoteAccount(context.Background(), dereferencing.GetRemoteAccountParams{
RequestingUsername: fetchingAccount.Username,
RemoteAccountID: serviceURL,
})
suite.NoError(err)
suite.NotNil(service)
suite.NotNil(service)
@ -69,6 +76,7 @@ func (suite *AccountTestSuite) TestDereferenceService() {
suite.NoError(err)
suite.Equal(service.ID, dbService.ID)
suite.Equal(ap.ActorService, dbService.ActorType)
suite.Equal("example.org", dbService.Domain)
}
func TestAccountTestSuite(t *testing.T) {

2
internal/federation/dereferencing/dereferencer.go
View file

@ -33,7 +33,7 @@ import (
// Dereferencer wraps logic and functionality for doing dereferencing of remote accounts, statuses, etc, from federated instances.
type Dereferencer interface {
GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error)
GetRemoteAccount(ctx context.Context, params GetRemoteAccountParams) (*gtsmodel.Account, error)
GetRemoteStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error)
EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error)

106
internal/federation/dereferencing/dereferencer_test.go
View file

@ -19,22 +19,14 @@
package dereferencing_test
import (
"bytes"
"encoding/json"
"io"
"net/http"
"codeberg.org/gruf/go-store/kv"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/activity/streams"
"github.com/superseriousbusiness/activity/streams/vocab"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/messages"
"github.com/superseriousbusiness/gotosocial/internal/transport"
"github.com/superseriousbusiness/gotosocial/testrig"
)
@ -66,106 +58,10 @@ func (suite *DereferencerStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.storage = testrig.NewTestStorage()
suite.dereferencer = dereferencing.NewDereferencer(suite.db, testrig.NewTestTypeConverter(suite.db), suite.mockTransportController(), testrig.NewTestMediaManager(suite.db, suite.storage))
suite.dereferencer = dereferencing.NewDereferencer(suite.db, testrig.NewTestTypeConverter(suite.db), testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, concurrency.NewWorkerPool[messages.FromFederator](-1, -1)), testrig.NewTestMediaManager(suite.db, suite.storage))
testrig.StandardDBSetup(suite.db, nil)
}
func (suite *DereferencerStandardTestSuite) TearDownTest() {
testrig.StandardDBTeardown(suite.db)
}
// mockTransportController returns basically a miniature muxer, which returns a different
// value based on the request URL. It can be used to return remote statuses, profiles, etc,
// as though they were actually being dereferenced. If the URL doesn't correspond to any person
// or note or attachment that we have stored, then just a 200 code will be returned, with an empty body.
func (suite *DereferencerStandardTestSuite) mockTransportController() transport.Controller {
do := func(req *http.Request) (*http.Response, error) {
logrus.Debugf("received request for %s", req.URL)
responseBytes := []byte{}
responseType := ""
responseLength := 0
if note, ok := suite.testRemoteStatuses[req.URL.String()]; ok {
// the request is for a note that we have stored
noteI, err := streams.Serialize(note)
if err != nil {
panic(err)
}
noteJson, err := json.Marshal(noteI)
if err != nil {
panic(err)
}
responseBytes = noteJson
responseType = "application/activity+json"
}
if person, ok := suite.testRemotePeople[req.URL.String()]; ok {
// the request is for a person that we have stored
personI, err := streams.Serialize(person)
if err != nil {
panic(err)
}
personJson, err := json.Marshal(personI)
if err != nil {
panic(err)
}
responseBytes = personJson
responseType = "application/activity+json"
}
if group, ok := suite.testRemoteGroups[req.URL.String()]; ok {
// the request is for a person that we have stored
groupI, err := streams.Serialize(group)
if err != nil {
panic(err)
}
groupJson, err := json.Marshal(groupI)
if err != nil {
panic(err)
}
responseBytes = groupJson
responseType = "application/activity+json"
}
if service, ok := suite.testRemoteServices[req.URL.String()]; ok {
serviceI, err := streams.Serialize(service)
if err != nil {
panic(err)
}
serviceJson, err := json.Marshal(serviceI)
if err != nil {
panic(err)
}
responseBytes = serviceJson
responseType = "application/activity+json"
}
if attachment, ok := suite.testRemoteAttachments[req.URL.String()]; ok {
responseBytes = attachment.Data
responseType = attachment.ContentType
}
if len(responseBytes) != 0 {
// we found something, so print what we're going to return
logrus.Debugf("returning response %s", string(responseBytes))
}
responseLength = len(responseBytes)
reader := bytes.NewReader(responseBytes)
readCloser := io.NopCloser(reader)
response := &http.Response{
StatusCode: 200,
Body: readCloser,
ContentLength: int64(responseLength),
Header: http.Header{
"content-type": {responseType},
},
}
return response, nil
}
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
mockClient := testrig.NewMockHTTPClient(do)
return testrig.NewTestTransportController(mockClient, suite.db, fedWorker)
}

80
internal/federation/dereferencing/finger.go Normal file
View file

@ -0,0 +1,80 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package dereferencing
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/url"
"strings"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
func (d *deref) fingerRemoteAccount(ctx context.Context, username string, targetUsername string, targetHost string) (accountDomain string, accountURI *url.URL, err error) {
t, err := d.transportController.NewTransportForUsername(ctx, username)
if err != nil {
err = fmt.Errorf("fingerRemoteAccount: error getting transport for %s: %s", username, err)
return
}
b, err := t.Finger(ctx, targetUsername, targetHost)
if err != nil {
err = fmt.Errorf("fingerRemoteAccount: error fingering @%s@%s: %s", targetUsername, targetHost, err)
return
}
resp := &apimodel.WellKnownResponse{}
if err = json.Unmarshal(b, resp); err != nil {
err = fmt.Errorf("fingerRemoteAccount: could not unmarshal server response as WebfingerAccountResponse while dereferencing @%s@%s: %s", targetUsername, targetHost, err)
return
}
if len(resp.Links) == 0 {
err = fmt.Errorf("fingerRemoteAccount: no links found in webfinger response %s", string(b))
return
}
if resp.Subject == "" {
err = fmt.Errorf("fingerRemoteAccount: no subject found in webfinger response %s", string(b))
return
}
_, accountDomain, err = util.ExtractWebfingerParts(resp.Subject)
if err != nil {
err = fmt.Errorf("fingerRemoteAccount: error extracting webfinger subject parts: %s", err)
}
// look through the links for the first one that matches what we need
for _, l := range resp.Links {
if l.Rel == "self" && (strings.EqualFold(l.Type, "application/activity+json") || strings.EqualFold(l.Type, "application/ld+json; profile=\"https://www.w3.org/ns/activitystreams\"")) {
if uri, thiserr := url.Parse(l.Href); thiserr == nil && (uri.Scheme == "http" || uri.Scheme == "https") {
// found it!
accountURI = uri
return
}
}
}
err = errors.New("fingerRemoteAccount: no match found in webfinger response")
return
}

10
internal/federation/dereferencing/status.go
View file

@ -85,7 +85,10 @@ func (d *deref) GetRemoteStatus(ctx context.Context, username string, remoteStat
return nil, nil, fmt.Errorf("GetRemoteStatus: error extracting attributedTo: %s", err)
}
_, err = d.GetRemoteAccount(ctx, username, accountURI, true, false)
_, err = d.GetRemoteAccount(ctx, GetRemoteAccountParams{
RequestingUsername: username,
RemoteAccountID: accountURI,
})
if err != nil {
return nil, nil, fmt.Errorf("GetRemoteStatus: couldn't get status author: %s", err)
}
@ -316,7 +319,10 @@ func (d *deref) populateStatusMentions(ctx context.Context, status *gtsmodel.Sta
if targetAccount == nil {
// we didn't find the account in our database already
// check if we can get the account remotely (dereference it)
if a, err := d.GetRemoteAccount(ctx, requestingUsername, targetAccountURI, false, false); err != nil {
if a, err := d.GetRemoteAccount(ctx, GetRemoteAccountParams{
RequestingUsername: requestingUsername,
RemoteAccountID: targetAccountURI,
}); err != nil {
errs = append(errs, err.Error())
} else {
logrus.Debugf("populateStatusMentions: got target account %s with id %s through GetRemoteAccount", targetAccountURI, a.ID)

43
internal/federation/federatingactor_test.go
View file

@ -19,10 +19,7 @@
package federation_test
import (
"bytes"
"context"
"io/ioutil"
"net/http"
"net/url"
"testing"
"time"
@ -60,15 +57,9 @@ func (suite *FederatingActorTestSuite) TestSendNoRemoteFollowers() {
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
// setup transport controller with a no-op client so we don't make external calls
sentMessages := []*url.URL{}
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
sentMessages = append(sentMessages, req.URL)
r := ioutil.NopCloser(bytes.NewReader([]byte{}))
return &http.Response{
StatusCode: 200,
Body: r,
}, nil
}), suite.db, fedWorker)
httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
// setup module being tested
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage))
@ -77,7 +68,7 @@ func (suite *FederatingActorTestSuite) TestSendNoRemoteFollowers() {
suite.NotNil(activity)
// because zork has no remote followers, sent messages should be empty (no messages sent to own instance)
suite.Empty(sentMessages)
suite.Empty(httpClient.SentMessages)
}
func (suite *FederatingActorTestSuite) TestSendRemoteFollower() {
@ -87,8 +78,8 @@ func (suite *FederatingActorTestSuite) TestSendRemoteFollower() {
err := suite.db.Put(ctx, &gtsmodel.Follow{
ID: "01G1TRWV4AYCDBX5HRWT2EVBCV",
CreatedAt: time.Now(),
UpdatedAt: time.Now(),
CreatedAt: testrig.TimeMustParse("2022-06-02T12:22:21+02:00"),
UpdatedAt: testrig.TimeMustParse("2022-06-02T12:22:21+02:00"),
AccountID: testRemoteAccount.ID,
TargetAccountID: testAccount.ID,
ShowReblogs: true,
@ -100,7 +91,7 @@ func (suite *FederatingActorTestSuite) TestSendRemoteFollower() {
testNote := testrig.NewAPNote(
testrig.URLMustParse("http://localhost:8080/users/the_mighty_zork/statuses/01G1TR6BADACCZWQMNF9X21TV5"),
testrig.URLMustParse("http://localhost:8080/@the_mighty_zork/statuses/01G1TR6BADACCZWQMNF9X21TV5"),
time.Now(),
testrig.TimeMustParse("2022-06-02T12:22:21+02:00"),
"boobies",
"",
testrig.URLMustParse(testAccount.URI),
@ -110,20 +101,12 @@ func (suite *FederatingActorTestSuite) TestSendRemoteFollower() {
nil,
nil,
)
testActivity := testrig.WrapAPNoteInCreate(testrig.URLMustParse("http://localhost:8080/whatever_some_create"), testrig.URLMustParse(testAccount.URI), time.Now(), testNote)
testActivity := testrig.WrapAPNoteInCreate(testrig.URLMustParse("http://localhost:8080/whatever_some_create"), testrig.URLMustParse(testAccount.URI), testrig.TimeMustParse("2022-06-02T12:22:21+02:00"), testNote)
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
// setup transport controller with a no-op client so we don't make external calls
sentMessages := []*url.URL{}
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
sentMessages = append(sentMessages, req.URL)
r := ioutil.NopCloser(bytes.NewReader([]byte{}))
return &http.Response{
StatusCode: 200,
Body: r,
}, nil
}), suite.db, fedWorker)
httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
// setup module being tested
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage))
@ -132,8 +115,10 @@ func (suite *FederatingActorTestSuite) TestSendRemoteFollower() {
suite.NotNil(activity)
// because we added 1 remote follower for zork, there should be a url in sentMessage
suite.Len(sentMessages, 1)
suite.Equal(testRemoteAccount.InboxURI, sentMessages[0].String())
suite.Len(httpClient.SentMessages, 1)
msg, ok := httpClient.SentMessages[testRemoteAccount.InboxURI]
suite.True(ok)
suite.Equal(`{"@context":"https://www.w3.org/ns/activitystreams","actor":"http://localhost:8080/users/the_mighty_zork","id":"http://localhost:8080/whatever_some_create","object":{"attributedTo":"http://localhost:8080/users/the_mighty_zork","content":"boobies","id":"http://localhost:8080/users/the_mighty_zork/statuses/01G1TR6BADACCZWQMNF9X21TV5","published":"2022-06-02T12:22:21+02:00","tag":[],"to":"http://localhost:8080/users/the_mighty_zork/followers","type":"Note","url":"http://localhost:8080/@the_mighty_zork/statuses/01G1TR6BADACCZWQMNF9X21TV5"},"published":"2022-06-02T12:22:21+02:00","to":"http://localhost:8080/users/the_mighty_zork/followers","type":"Create"}`, string(msg))
}
func TestFederatingActorTestSuite(t *testing.T) {

2
internal/federation/federatingdb/update.go
View file

@ -119,7 +119,7 @@ func (f *federatingDB) Update(ctx context.Context, asType vocab.Type) error {
accountable = i
}
updatedAcct, err := f.typeConverter.ASRepresentationToAccount(ctx, accountable, true)
updatedAcct, err := f.typeConverter.ASRepresentationToAccount(ctx, accountable, "", true)
if err != nil {
return fmt.Errorf("UPDATE: error converting to account: %s", err)
}

6
internal/federation/federatingprotocol.go
View file

@ -31,6 +31,7 @@ import (
"github.com/superseriousbusiness/activity/streams/vocab"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/uris"
"github.com/superseriousbusiness/gotosocial/internal/util"
@ -197,7 +198,10 @@ func (f *federator) AuthenticatePostInbox(ctx context.Context, w http.ResponseWr
}
}
requestingAccount, err := f.GetRemoteAccount(ctx, username, publicKeyOwnerURI, false, false)
requestingAccount, err := f.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
RequestingUsername: username,
RemoteAccountID: publicKeyOwnerURI,
})
if err != nil {
return nil, false, fmt.Errorf("couldn't get requesting account %s: %s", publicKeyOwnerURI, err)
}

36
internal/federation/federatingprotocol_test.go
View file

@ -45,10 +45,8 @@ func (suite *FederatingProtocolTestSuite) TestPostInboxRequestBodyHook1() {
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
// setup transport controller with a no-op client so we don't make external calls
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
return nil, nil
}), suite.db, fedWorker)
httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
// setup module being tested
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage))
@ -78,10 +76,9 @@ func (suite *FederatingProtocolTestSuite) TestPostInboxRequestBodyHook2() {
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
// setup transport controller with a no-op client so we don't make external calls
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
return nil, nil
}), suite.db, fedWorker)
httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
// setup module being tested
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage))
@ -112,10 +109,9 @@ func (suite *FederatingProtocolTestSuite) TestPostInboxRequestBodyHook3() {
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
// setup transport controller with a no-op client so we don't make external calls
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
return nil, nil
}), suite.db, fedWorker)
httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
// setup module being tested
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage))
@ -148,7 +144,9 @@ func (suite *FederatingProtocolTestSuite) TestAuthenticatePostInbox() {
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
// now setup module being tested, with the mock transport controller
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage))
@ -186,7 +184,8 @@ func (suite *FederatingProtocolTestSuite) TestAuthenticatePostInbox() {
func (suite *FederatingProtocolTestSuite) TestBlocked1() {
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage))
sendingAccount := suite.testAccounts["remote_account_1"]
@ -208,7 +207,8 @@ func (suite *FederatingProtocolTestSuite) TestBlocked1() {
func (suite *FederatingProtocolTestSuite) TestBlocked2() {
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage))
sendingAccount := suite.testAccounts["remote_account_1"]
@ -241,7 +241,8 @@ func (suite *FederatingProtocolTestSuite) TestBlocked2() {
func (suite *FederatingProtocolTestSuite) TestBlocked3() {
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage))
sendingAccount := suite.testAccounts["remote_account_1"]
@ -277,7 +278,8 @@ func (suite *FederatingProtocolTestSuite) TestBlocked3() {
func (suite *FederatingProtocolTestSuite) TestBlocked4() {
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
tc := testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
httpClient := testrig.NewMockHTTPClient(nil, "../../testrig/media")
tc := testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
federator := federation.NewFederator(suite.db, testrig.NewTestFederatingDB(suite.db, fedWorker), tc, suite.tc, testrig.NewTestMediaManager(suite.db, suite.storage))
sendingAccount := suite.testAccounts["remote_account_1"]

6
internal/federation/federator.go
View file

@ -53,14 +53,10 @@ type Federator interface {
// If something goes wrong during authentication, nil, false, and an error will be returned.
AuthenticateFederatedRequest(ctx context.Context, username string) (*url.URL, gtserror.WithCode)
// FingerRemoteAccount performs a webfinger lookup for a remote account, using the .well-known path. It will return the ActivityPub URI for that
// account, or an error if it doesn't exist or can't be retrieved.
FingerRemoteAccount(ctx context.Context, requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error)
DereferenceRemoteThread(ctx context.Context, username string, statusURI *url.URL) error
DereferenceAnnounce(ctx context.Context, announce *gtsmodel.Status, requestingUsername string) error
GetRemoteAccount(ctx context.Context, username string, remoteAccountID *url.URL, blocking bool, refresh bool) (*gtsmodel.Account, error)
GetRemoteAccount(ctx context.Context, params dereferencing.GetRemoteAccountParams) (*gtsmodel.Account, error)
GetRemoteStatus(ctx context.Context, username string, remoteStatusID *url.URL, refetch, includeParent bool) (*gtsmodel.Status, ap.Statusable, error)
EnrichRemoteStatus(ctx context.Context, username string, status *gtsmodel.Status, includeParent bool) (*gtsmodel.Status, error)

72
internal/federation/finger.go
View file

@ -1,72 +0,0 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package federation
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/url"
"strings"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
)
func (f *federator) FingerRemoteAccount(ctx context.Context, requestingUsername string, targetUsername string, targetDomain string) (*url.URL, error) {
if blocked, err := f.db.IsDomainBlocked(ctx, targetDomain); blocked || err != nil {
return nil, fmt.Errorf("FingerRemoteAccount: domain %s is blocked", targetDomain)
}
t, err := f.transportController.NewTransportForUsername(ctx, requestingUsername)
if err != nil {
return nil, fmt.Errorf("FingerRemoteAccount: error getting transport for username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err)
}
b, err := t.Finger(ctx, targetUsername, targetDomain)
if err != nil {
return nil, fmt.Errorf("FingerRemoteAccount: error doing request on behalf of username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err)
}
resp := &apimodel.WellKnownResponse{}
if err := json.Unmarshal(b, resp); err != nil {
return nil, fmt.Errorf("FingerRemoteAccount: could not unmarshal server response as WebfingerAccountResponse on behalf of username %s while dereferencing @%s@%s: %s", requestingUsername, targetUsername, targetDomain, err)
}
if len(resp.Links) == 0 {
return nil, fmt.Errorf("FingerRemoteAccount: no links found in webfinger response %s", string(b))
}
// look through the links for the first one that matches "application/activity+json", this is what we need
for _, l := range resp.Links {
if strings.EqualFold(l.Type, "application/activity+json") {
if l.Href == "" || l.Rel != "self" {
continue
}
accountURI, err := url.Parse(l.Href)
if err != nil {
return nil, fmt.Errorf("FingerRemoteAccount: couldn't parse url %s: %s", l.Href, err)
}
// found it!
return accountURI, nil
}
}
return nil, errors.New("FingerRemoteAccount: no match found in webfinger response")
}

5
internal/processing/account/account_test.go
View file

@ -23,7 +23,6 @@ import (
"codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/activity/pub"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
@ -48,7 +47,6 @@ type AccountStandardTestSuite struct {
mediaManager media.Manager
oauthServer oauth.Server
fromClientAPIChan chan messages.FromClientAPI
httpClient pub.HttpClient
transportController transport.Controller
federator federation.Federator
emailSender email.Sender
@ -97,8 +95,7 @@ func (suite *AccountStandardTestSuite) SetupTest() {
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)
suite.fromClientAPIChan = make(chan messages.FromClientAPI, 100)
suite.httpClient = testrig.NewMockHTTPClient(nil)
suite.transportController = testrig.NewTestTransportController(suite.httpClient, suite.db, fedWorker)
suite.transportController = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, fedWorker)
suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager, fedWorker)
suite.sentEmails = make(map[string]string)
suite.emailSender = testrig.NewEmailSender("../../../web/template/", suite.sentEmails)

9
internal/processing/account/get.go
View file

@ -26,6 +26,7 @@ import (
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
@ -80,7 +81,13 @@ func (p *processor) getAccountFor(ctx context.Context, requestingAccount *gtsmod
return nil, gtserror.NewErrorInternalError(fmt.Errorf("error parsing url %s: %s", targetAccount.URI, err))
}
a, err := p.federator.GetRemoteAccount(ctx, requestingAccount.Username, targetAccountURI, true, false)
a, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
RequestingUsername: requestingAccount.Username,
RemoteAccountID: targetAccountURI,
RemoteAccountHost: targetAccount.Domain,
RemoteAccountUsername: targetAccount.Username,
Blocking: true,
})
if err == nil {
targetAccount = a
}

2
internal/processing/account_test.go
View file

@ -60,7 +60,7 @@ func (suite *AccountTestSuite) TestAccountDeleteLocal() {
time.Sleep(1 * time.Second) // wait a sec for the delete to process
// the delete should be federated outwards to the following account's inbox
sent, ok := suite.sentHTTPRequests[followingAccount.InboxURI]
sent, ok := suite.httpClient.SentMessages[followingAccount.InboxURI]
suite.True(ok)
delete := &struct {
Actor string `json:"actor"`

6
internal/processing/federation/getfollowers.go
View file

@ -24,6 +24,7 @@ import (
"net/url"
"github.com/superseriousbusiness/activity/streams"
"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
)
@ -40,7 +41,10 @@ func (p *processor) GetFollowers(ctx context.Context, requestedUsername string,
return nil, errWithCode
}
requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
RequestingUsername: requestedUsername,
RemoteAccountID: requestingAccountURI,
})
if err != nil {
return nil, gtserror.NewErrorUnauthorized(err)
}

6
internal/processing/federation/getfollowing.go
View file

@ -24,6 +24,7 @@ import (
"net/url"
"github.com/superseriousbusiness/activity/streams"
"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
)
@ -40,7 +41,10 @@ func (p *processor) GetFollowing(ctx context.Context, requestedUsername string,
return nil, errWithCode
}
requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
RequestingUsername: requestedUsername,
RemoteAccountID: requestingAccountURI,
})
if err != nil {
return nil, gtserror.NewErrorUnauthorized(err)
}

6
internal/processing/federation/getoutbox.go
View file

@ -25,6 +25,7 @@ import (
"github.com/superseriousbusiness/activity/streams"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
)
@ -41,7 +42,10 @@ func (p *processor) GetOutbox(ctx context.Context, requestedUsername string, pag
return nil, errWithCode
}
requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
RequestingUsername: requestedUsername,
RemoteAccountID: requestingAccountURI,
})
if err != nil {
return nil, gtserror.NewErrorUnauthorized(err)
}

6
internal/processing/federation/getstatus.go
View file

@ -24,6 +24,7 @@ import (
"net/url"
"github.com/superseriousbusiness/activity/streams"
"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
)
@ -40,7 +41,10 @@ func (p *processor) GetStatus(ctx context.Context, requestedUsername string, req
return nil, errWithCode
}
requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
RequestingUsername: requestedUsername,
RemoteAccountID: requestingAccountURI,
})
if err != nil {
return nil, gtserror.NewErrorUnauthorized(err)
}

6
internal/processing/federation/getstatusreplies.go
View file

@ -25,6 +25,7 @@ import (
"github.com/superseriousbusiness/activity/streams"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
@ -42,7 +43,10 @@ func (p *processor) GetStatusReplies(ctx context.Context, requestedUsername stri
return nil, errWithCode
}
requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
RequestingUsername: requestedUsername,
RemoteAccountID: requestingAccountURI,
})
if err != nil {
return nil, gtserror.NewErrorUnauthorized(err)
}

6
internal/processing/federation/getuser.go
View file

@ -25,6 +25,7 @@ import (
"github.com/superseriousbusiness/activity/streams"
"github.com/superseriousbusiness/activity/streams/vocab"
"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/uris"
)
@ -52,7 +53,10 @@ func (p *processor) GetUser(ctx context.Context, requestedUsername string, reque
// if we're not already handshaking/dereferencing a remote account, dereference it now
if !p.federator.Handshaking(ctx, requestedUsername, requestingAccountURI) {
requestingAccount, err := p.federator.GetRemoteAccount(ctx, requestedUsername, requestingAccountURI, false, false)
requestingAccount, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
RequestingUsername: requestedUsername,
RemoteAccountID: requestingAccountURI,
})
if err != nil {
return nil, gtserror.NewErrorUnauthorized(err)
}

4
internal/processing/followrequest_test.go
View file

@ -57,7 +57,7 @@ func (suite *FollowRequestTestSuite) TestFollowRequestAccept() {
time.Sleep(1 * time.Second)
// accept should be sent to some_user
sent, ok := suite.sentHTTPRequests[requestingAccount.InboxURI]
sent, ok := suite.httpClient.SentMessages[requestingAccount.InboxURI]
suite.True(ok)
accept := &struct {
@ -109,7 +109,7 @@ func (suite *FollowRequestTestSuite) TestFollowRequestReject() {
time.Sleep(1 * time.Second)
// reject should be sent to some_user
sent, ok := suite.sentHTTPRequests[requestingAccount.InboxURI]
sent, ok := suite.httpClient.SentMessages[requestingAccount.InboxURI]
suite.True(ok)
reject := &struct {

31
internal/processing/fromfederator.go
View file

@ -26,6 +26,7 @@ import (
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/gotosocial/internal/ap"
"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/messages"
@ -130,7 +131,11 @@ func (p *processor) processCreateStatusFromFederator(ctx context.Context, federa
return err
}
a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false)
a, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
RequestingUsername: federatorMsg.ReceivingAccount.Username,
RemoteAccountID: remoteAccountID,
Blocking: true,
})
if err != nil {
return err
}
@ -172,7 +177,11 @@ func (p *processor) processCreateFaveFromFederator(ctx context.Context, federato
return err
}
a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false)
a, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
RequestingUsername: federatorMsg.ReceivingAccount.Username,
RemoteAccountID: remoteAccountID,
Blocking: true,
})
if err != nil {
return err
}
@ -210,7 +219,11 @@ func (p *processor) processCreateFollowRequestFromFederator(ctx context.Context,
return err
}
a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false)
a, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
RequestingUsername: federatorMsg.ReceivingAccount.Username,
RemoteAccountID: remoteAccountID,
Blocking: true,
})
if err != nil {
return err
}
@ -267,7 +280,11 @@ func (p *processor) processCreateAnnounceFromFederator(ctx context.Context, fede
return err
}
a, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, remoteAccountID, true, false)
a, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
RequestingUsername: federatorMsg.ReceivingAccount.Username,
RemoteAccountID: remoteAccountID,
Blocking: true,
})
if err != nil {
return err
}
@ -332,7 +349,11 @@ func (p *processor) processUpdateAccountFromFederator(ctx context.Context, feder
return err
}
if _, err := p.federator.GetRemoteAccount(ctx, federatorMsg.ReceivingAccount.Username, incomingAccountURL, false, true); err != nil {
if _, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
RequestingUsername: federatorMsg.ReceivingAccount.Username,
RemoteAccountID: incomingAccountURL,
Blocking: true,
}); err != nil {
return fmt.Errorf("error enriching updated account from federator: %s", err)
}

6
internal/processing/fromfederator_test.go
View file

@ -423,7 +423,7 @@ func (suite *FromFederatorTestSuite) TestProcessFollowRequestLocked() {
suite.Equal(originAccount.ID, notif.Account.ID)
// no messages should have been sent out, since we didn't need to federate an accept
suite.Empty(suite.sentHTTPRequests)
suite.Empty(suite.httpClient.SentMessages)
}
func (suite *FromFederatorTestSuite) TestProcessFollowRequestUnlocked() {
@ -474,8 +474,8 @@ func (suite *FromFederatorTestSuite) TestProcessFollowRequestUnlocked() {
suite.Equal(originAccount.ID, notif.Account.ID)
// an accept message should be sent to satan's inbox
suite.Len(suite.sentHTTPRequests, 1)
acceptBytes := suite.sentHTTPRequests[originAccount.InboxURI]
suite.Len(suite.httpClient.SentMessages, 1)
acceptBytes := suite.httpClient.SentMessages[originAccount.InboxURI]
accept := &struct {
Actor string `json:"actor"`
ID string `json:"id"`

44
internal/processing/media/media_test.go
View file

@ -19,12 +19,7 @@
package media_test
import (
"bytes"
"io"
"net/http"
"codeberg.org/gruf/go-store/kv"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
"github.com/superseriousbusiness/gotosocial/internal/db"
@ -79,7 +74,7 @@ func (suite *MediaStandardTestSuite) SetupTest() {
suite.tc = testrig.NewTestTypeConverter(suite.db)
suite.storage = testrig.NewTestStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.transportController = suite.mockTransportController()
suite.transportController = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, concurrency.NewWorkerPool[messages.FromFederator](-1, -1))
suite.mediaProcessor = mediaprocessing.New(suite.db, suite.tc, suite.mediaManager, suite.transportController, suite.storage)
testrig.StandardDBSetup(suite.db, nil)
testrig.StandardStorageSetup(suite.storage, "../../../testrig/media")
@ -89,40 +84,3 @@ func (suite *MediaStandardTestSuite) TearDownTest() {
testrig.StandardDBTeardown(suite.db)
testrig.StandardStorageTeardown(suite.storage)
}
func (suite *MediaStandardTestSuite) mockTransportController() transport.Controller {
do := func(req *http.Request) (*http.Response, error) {
logrus.Debugf("received request for %s", req.URL)
responseBytes := []byte{}
responseType := ""
responseLength := 0
if attachment, ok := suite.testRemoteAttachments[req.URL.String()]; ok {
responseBytes = attachment.Data
responseType = attachment.ContentType
}
if len(responseBytes) != 0 {
// we found something, so print what we're going to return
logrus.Debugf("returning response %s", string(responseBytes))
}
responseLength = len(responseBytes)
reader := bytes.NewReader(responseBytes)
readCloser := io.NopCloser(reader)
response := &http.Response{
StatusCode: 200,
Body: readCloser,
ContentLength: int64(responseLength),
Header: http.Header{
"content-type": {responseType},
},
}
return response, nil
}
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
mockClient := testrig.NewMockHTTPClient(do)
return testrig.NewTestTransportController(mockClient, suite.db, fedWorker)
}

126
internal/processing/processor_test.go
View file

@ -19,16 +19,8 @@
package processing_test
import (
"bytes"
"context"
"encoding/json"
"io"
"io/ioutil"
"net/http"
"codeberg.org/gruf/go-store/kv"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/activity/streams"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/email"
@ -50,6 +42,7 @@ type ProcessingStandardTestSuite struct {
storage *kv.KVStore
mediaManager media.Manager
typeconverter typeutils.TypeConverter
httpClient *testrig.MockHTTPClient
transportController transport.Controller
federator federation.Federator
oauthServer oauth.Server
@ -69,8 +62,6 @@ type ProcessingStandardTestSuite struct {
testBlocks map[string]*gtsmodel.Block
testActivities map[string]testrig.ActivityWithSignature
sentHTTPRequests map[string][]byte
processor processing.Processor
}
@ -102,123 +93,12 @@ func (suite *ProcessingStandardTestSuite) SetupTest() {
suite.testActivities = testrig.NewTestActivities(suite.testAccounts)
suite.storage = testrig.NewTestStorage()
suite.typeconverter = testrig.NewTestTypeConverter(suite.db)
// make an http client that stores POST requests it receives into a map,
// and also responds to correctly to dereference requests
suite.sentHTTPRequests = make(map[string][]byte)
httpClient := testrig.NewMockHTTPClient(func(req *http.Request) (*http.Response, error) {
if req.Method == http.MethodPost && req.Body != nil {
requestBytes, err := ioutil.ReadAll(req.Body)
if err != nil {
panic(err)
}
if err := req.Body.Close(); err != nil {
panic(err)
}
suite.sentHTTPRequests[req.URL.String()] = requestBytes
}
if req.URL.String() == suite.testAccounts["remote_account_1"].URI {
// the request is for remote account 1
satan := suite.testAccounts["remote_account_1"]
satanAS, err := suite.typeconverter.AccountToAS(context.Background(), satan)
if err != nil {
panic(err)
}
satanI, err := streams.Serialize(satanAS)
if err != nil {
panic(err)
}
satanJson, err := json.Marshal(satanI)
if err != nil {
panic(err)
}
responseType := "application/activity+json"
reader := bytes.NewReader(satanJson)
readCloser := io.NopCloser(reader)
response := &http.Response{
StatusCode: 200,
Body: readCloser,
ContentLength: int64(len(satanJson)),
Header: http.Header{
"content-type": {responseType},
},
}
return response, nil
}
if req.URL.String() == suite.testAccounts["remote_account_2"].URI {
// the request is for remote account 2
someAccount := suite.testAccounts["remote_account_2"]
someAccountAS, err := suite.typeconverter.AccountToAS(context.Background(), someAccount)
if err != nil {
panic(err)
}
someAccountI, err := streams.Serialize(someAccountAS)
if err != nil {
panic(err)
}
someAccountJson, err := json.Marshal(someAccountI)
if err != nil {
panic(err)
}
responseType := "application/activity+json"
reader := bytes.NewReader(someAccountJson)
readCloser := io.NopCloser(reader)
response := &http.Response{
StatusCode: 200,
Body: readCloser,
ContentLength: int64(len(someAccountJson)),
Header: http.Header{
"content-type": {responseType},
},
}
return response, nil
}
if req.URL.String() == "http://example.org/users/some_user/statuses/afaba698-5740-4e32-a702-af61aa543bc1" {
// the request is for the forwarded message
message := suite.testActivities["forwarded_message"].Activity.GetActivityStreamsObject().At(0).GetActivityStreamsNote()
messageI, err := streams.Serialize(message)
if err != nil {
panic(err)
}
messageJson, err := json.Marshal(messageI)
if err != nil {
panic(err)
}
responseType := "application/activity+json"
reader := bytes.NewReader(messageJson)
readCloser := io.NopCloser(reader)
response := &http.Response{
StatusCode: 200,
Body: readCloser,
ContentLength: int64(len(messageJson)),
Header: http.Header{
"content-type": {responseType},
},
}
return response, nil
}
r := ioutil.NopCloser(bytes.NewReader([]byte{}))
return &http.Response{
StatusCode: 200,
Body: r,
}, nil
})
suite.httpClient = testrig.NewMockHTTPClient(nil, "../../testrig/media")
clientWorker := concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
fedWorker := concurrency.NewWorkerPool[messages.FromFederator](-1, -1)
suite.transportController = testrig.NewTestTransportController(httpClient, suite.db, fedWorker)
suite.transportController = testrig.NewTestTransportController(suite.httpClient, suite.db, fedWorker)
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, suite.transportController, suite.storage, suite.mediaManager, fedWorker)
suite.oauthServer = testrig.NewTestOauthServer(suite.db)

94
internal/processing/search.go
View file

@ -20,6 +20,7 @@ package processing
import (
"context"
"errors"
"fmt"
"net/url"
"strings"
@ -28,53 +29,70 @@ import (
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
"github.com/superseriousbusiness/gotosocial/internal/gtserror"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/oauth"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
func (p *processor) SearchGet(ctx context.Context, authed *oauth.Auth, searchQuery *apimodel.SearchQuery) (*apimodel.SearchResult, gtserror.WithCode) {
func (p *processor) SearchGet(ctx context.Context, authed *oauth.Auth, search *apimodel.SearchQuery) (*apimodel.SearchResult, gtserror.WithCode) {
l := logrus.WithFields(logrus.Fields{
"func": "SearchGet",
"query": searchQuery.Query,
"query": search.Query,
})
results := &apimodel.SearchResult{
// tidy up the query and make sure it wasn't just spaces
query := strings.TrimSpace(search.Query)
if query == "" {
err := errors.New("search query was empty string after trimming space")
return nil, gtserror.NewErrorBadRequest(err, err.Error())
}
searchResult := &apimodel.SearchResult{
Accounts: []apimodel.Account{},
Statuses: []apimodel.Status{},
Hashtags: []apimodel.Tag{},
}
foundAccounts := []*gtsmodel.Account{}
foundStatuses := []*gtsmodel.Status{}
// foundHashtags := []*gtsmodel.Tag{}
// convert the query to lowercase and trim leading/trailing spaces
query := strings.ToLower(strings.TrimSpace(searchQuery.Query))
var foundOne bool
// check if the query is something like @whatever_username@example.org -- this means it's a remote account
if _, domain, err := util.ExtractMentionParts(searchQuery.Query); err == nil && domain != "" {
l.Debug("search term is a mention, looking it up...")
foundAccount, err := p.searchAccountByMention(ctx, authed, searchQuery.Query, searchQuery.Resolve)
if err == nil && foundAccount != nil {
/*
SEARCH BY MENTION
check if the query is something like @whatever_username@example.org -- this means it's a remote account
*/
maybeNamestring := query
if maybeNamestring[0] != '@' {
maybeNamestring = "@" + maybeNamestring
}
if username, domain, err := util.ExtractNamestringParts(maybeNamestring); err == nil {
l.Debugf("search term %s is a mention, looking it up...", maybeNamestring)
if foundAccount, err := p.searchAccountByMention(ctx, authed, username, domain, search.Resolve); err == nil && foundAccount != nil {
foundAccounts = append(foundAccounts, foundAccount)
foundOne = true
l.Debug("got an account by searching by mention")
} else if err != nil {
l.Debugf("error looking up account %s: %s", maybeNamestring, err)
}
}
// check if the query is a URI with a recognizable scheme and just do a lookup for that, straight up
/*
SEARCH BY URI
check if the query is a URI with a recognizable scheme and dereference it
*/
if !foundOne {
if uri, err := url.Parse(query); err == nil && (uri.Scheme == "https" || uri.Scheme == "http") {
// 1. check if it's a status
if foundStatus, err := p.searchStatusByURI(ctx, authed, uri, searchQuery.Resolve); err == nil && foundStatus != nil {
if foundStatus, err := p.searchStatusByURI(ctx, authed, uri, search.Resolve); err == nil && foundStatus != nil {
foundStatuses = append(foundStatuses, foundStatus)
l.Debug("got a status by searching by URI")
}
// 2. check if it's an account
if foundAccount, err := p.searchAccountByURI(ctx, authed, uri, searchQuery.Resolve); err == nil && foundAccount != nil {
if foundAccount, err := p.searchAccountByURI(ctx, authed, uri, search.Resolve); err == nil && foundAccount != nil {
foundAccounts = append(foundAccounts, foundAccount)
l.Debug("got an account by searching by URI")
}
@ -90,7 +108,7 @@ func (p *processor) SearchGet(ctx context.Context, authed *oauth.Auth, searchQue
if blocked, err := p.db.IsBlocked(ctx, authed.Account.ID, foundAccount.ID, true); err == nil && !blocked {
// all good, convert it and add it to the results
if apiAcct, err := p.tc.AccountToAPIAccountPublic(ctx, foundAccount); err == nil && apiAcct != nil {
results.Accounts = append(results.Accounts, *apiAcct)
searchResult.Accounts = append(searchResult.Accounts, *apiAcct)
}
}
}
@ -105,10 +123,10 @@ func (p *processor) SearchGet(ctx context.Context, authed *oauth.Auth, searchQue
continue
}
results.Statuses = append(results.Statuses, *apiStatus)
searchResult.Statuses = append(searchResult.Statuses, *apiStatus)
}
return results, nil
return searchResult, nil
}
func (p *processor) searchStatusByURI(ctx context.Context, authed *oauth.Auth, uri *url.URL, resolve bool) (*gtsmodel.Status, error) {
@ -147,7 +165,10 @@ func (p *processor) searchAccountByURI(ctx context.Context, authed *oauth.Auth,
if resolve {
// we don't have it locally so try and dereference it
account, err := p.federator.GetRemoteAccount(ctx, authed.Account.Username, uri, true, true)
account, err := p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
RequestingUsername: authed.Account.Username,
RemoteAccountID: uri,
})
if err != nil {
return nil, fmt.Errorf("searchAccountByURI: error dereferencing account with uri %s: %s", uri.String(), err)
}
@ -156,18 +177,14 @@ func (p *processor) searchAccountByURI(ctx context.Context, authed *oauth.Auth,
return nil, nil
}
func (p *processor) searchAccountByMention(ctx context.Context, authed *oauth.Auth, mention string, resolve bool) (*gtsmodel.Account, error) {
// query is for a remote account
username, domain, err := util.ExtractMentionParts(mention)
if err != nil {
return nil, fmt.Errorf("searchAccountByMention: error extracting mention parts: %s", err)
}
func (p *processor) searchAccountByMention(ctx context.Context, authed *oauth.Auth, username string, domain string, resolve bool) (*gtsmodel.Account, error) {
maybeAcct := &gtsmodel.Account{}
var err error
// if it's a local account we can skip a whole bunch of stuff
maybeAcct := &gtsmodel.Account{}
if domain == config.GetHost() {
if domain == config.GetHost() || domain == config.GetAccountDomain() || domain == "" {
maybeAcct, err = p.db.GetLocalAccountByUsername(ctx, username)
if err != nil {
if err != nil && err != db.ErrNoEntries {
return nil, fmt.Errorf("searchAccountByMention: error getting local account by username: %s", err)
}
return maybeAcct, nil
@ -191,22 +208,15 @@ func (p *processor) searchAccountByMention(ctx context.Context, authed *oauth.Au
// we got a db.ErrNoEntries, so we just don't have the account locally stored -- check if we can dereference it
if resolve {
// we're allowed to resolve it so let's try
// first we need to webfinger the remote account to convert the username and domain into the activitypub URI for the account
acctURI, err := p.federator.FingerRemoteAccount(ctx, authed.Account.Username, username, domain)
maybeAcct, err = p.federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
RequestingUsername: authed.Account.Username,
RemoteAccountUsername: username,
RemoteAccountHost: domain,
})
if err != nil {
// something went wrong doing the webfinger lookup so we can't process the request
return nil, fmt.Errorf("error fingering remote account with username %s and domain %s: %s", username, domain, err)
}
if acctURI.Scheme == "https" || acctURI.Scheme == "http" {
acct, err := p.federator.GetRemoteAccount(ctx, authed.Account.Username, acctURI, true, true)
if err != nil {
logrus.Debugf("could not get remote account by mention %s with uri %s: %s", mention, acctURI, err)
return nil, err
}
return acct, nil
return nil, fmt.Errorf("searchAccountByMention: error getting remote account: %s", err)
}
return maybeAcct, nil
}
return nil, nil

2
internal/processing/status/status_test.go
View file

@ -80,7 +80,7 @@ func (suite *StatusStandardTestSuite) SetupTest() {
suite.db = testrig.NewTestDB()
suite.typeConverter = testrig.NewTestTypeConverter(suite.db)
suite.clientWorker = concurrency.NewWorkerPool[messages.FromClientAPI](-1, -1)
suite.tc = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil), suite.db, fedWorker)
suite.tc = testrig.NewTestTransportController(testrig.NewMockHTTPClient(nil, "../../../testrig/media"), suite.db, fedWorker)
suite.storage = testrig.NewTestStorage()
suite.mediaManager = testrig.NewTestMediaManager(suite.db, suite.storage)
suite.federator = testrig.NewTestFederator(suite.db, suite.tc, suite.storage, suite.mediaManager, fedWorker)

101
internal/processing/util.go
View file

@ -21,12 +21,14 @@ package processing
import (
"context"
"fmt"
"strings"
"github.com/superseriousbusiness/gotosocial/internal/config"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/federation"
"github.com/superseriousbusiness/gotosocial/internal/federation/dereferencing"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/id"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
func GetParseMentionFunc(dbConn db.DB, federator federation.Federator) gtsmodel.ParseMentionFunc {
@ -37,98 +39,35 @@ func GetParseMentionFunc(dbConn db.DB, federator federation.Federator) gtsmodel.
return nil, fmt.Errorf("couldn't get mention origin account with id %s", originAccountID)
}
// A mentioned account looks like "@test@example.org" or just "@test" for a local account
// -- we can guarantee this from the regex that targetAccounts should have been derived from.
// But we still need to do a bit of fiddling to get what we need here -- the username and domain (if given).
// 1. trim off the first @
trimmed := strings.TrimPrefix(targetAccount, "@")
// 2. split the username and domain
split := strings.Split(trimmed, "@")
// 3. if it's length 1 it's a local account, length 2 means remote, anything else means something is wrong
var local bool
switch len(split) {
case 1:
local = true
case 2:
local = false
default:
return nil, fmt.Errorf("mentioned account format '%s' was not valid", targetAccount)
}
var username, domain string
username = split[0]
if !local {
domain = split[1]
}
// 4. check we now have a proper username and domain
if username == "" || (!local && domain == "") {
return nil, fmt.Errorf("username or domain for '%s' was nil", targetAccount)
username, domain, err := util.ExtractNamestringParts(targetAccount)
if err != nil {
return nil, fmt.Errorf("couldn't extract namestring parts from %s: %s", targetAccount, err)
}
var mentionedAccount *gtsmodel.Account
if local {
if domain == "" || domain == config.GetHost() || domain == config.GetAccountDomain() {
localAccount, err := dbConn.GetLocalAccountByUsername(ctx, username)
if err != nil {
return nil, err
}
mentionedAccount = localAccount
} else {
remoteAccount := &gtsmodel.Account{}
where := []db.Where{
{
Key: "username",
Value: username,
CaseInsensitive: true,
},
{
Key: "domain",
Value: domain,
CaseInsensitive: true,
},
var requestingUsername string
if originAccount.Domain == "" {
requestingUsername = originAccount.Username
}
remoteAccount, err := federator.GetRemoteAccount(ctx, dereferencing.GetRemoteAccountParams{
RequestingUsername: requestingUsername,
RemoteAccountUsername: username,
RemoteAccountHost: domain,
})
if err != nil {
return nil, fmt.Errorf("error dereferencing account: %s", err)
}
err := dbConn.GetWhere(ctx, where, remoteAccount)
if err == nil {
// the account was already in the database
mentionedAccount = remoteAccount
} else {
// we couldn't get it from the database
if err != db.ErrNoEntries {
// a serious error has happened so bail
return nil, fmt.Errorf("error getting account with username '%s' and domain '%s': %s", username, domain, err)
}
// we were able to resolve it!
mentionedAccount = remoteAccount
// We just don't have the account, so try webfingering it.
//
// If the mention originates from our instance we should use the username of the origin account to do the dereferencing,
// otherwise we should just use our instance account (that is, provide an empty string), since obviously we can't use
// a remote account to do remote dereferencing!
var fingeringUsername string
if originAccount.Domain == "" {
fingeringUsername = originAccount.Username
}
acctURI, err := federator.FingerRemoteAccount(ctx, fingeringUsername, username, domain)
if err != nil {
// something went wrong doing the webfinger lookup so we can't process the request
return nil, fmt.Errorf("error fingering remote account with username %s and domain %s: %s", username, domain, err)
}
resolvedAccount, err := federator.GetRemoteAccount(ctx, fingeringUsername, acctURI, true, true)
if err != nil {
return nil, fmt.Errorf("error dereferencing account with uri %s: %s", acctURI.String(), err)
}
// we were able to resolve it!
mentionedAccount = resolvedAccount
}
}
mentionID, err := id.NewRandomULID()

2
internal/regexes/regexes.go
View file

@ -61,7 +61,7 @@ var (
return rgx
}()
mentionName = `^@(\w+)(?:@([a-zA-Z0-9_\-\.:]+))?$`
mentionName = `^@([\w\-\.]+)(?:@([\w\-\.:]+))?$`
// MentionName captures the username and domain part from a mention string
// such as @whatever_user@example.org, returning whatever_user and example.org (without the @ symbols)
MentionName = regexp.MustCompile(mentionName)

2
internal/transport/derefinstance.go
View file

@ -206,7 +206,7 @@ func dereferenceByNodeInfo(c context.Context, t *transport, iri *url.URL) (*gtsm
// see if there's a 'name' in the map
if name, present := v["name"]; present {
// name could be just a username, or could be a mention string eg @whatever@aaaa.com
username, _, err := util.ExtractMentionParts(name)
username, _, err := util.ExtractNamestringParts(name)
if err == nil {
// it was a mention string
contactAccountUsername = username

8
internal/typeutils/astointernal.go
View file

@ -30,7 +30,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
)
func (c *converter) ASRepresentationToAccount(ctx context.Context, accountable ap.Accountable, update bool) (*gtsmodel.Account, error) {
func (c *converter) ASRepresentationToAccount(ctx context.Context, accountable ap.Accountable, accountDomain string, update bool) (*gtsmodel.Account, error) {
// first check if we actually already know this account
uriProp := accountable.GetJSONLDId()
if uriProp == nil || !uriProp.IsIRI() {
@ -63,7 +63,11 @@ func (c *converter) ASRepresentationToAccount(ctx context.Context, accountable a
acct.Username = username
// Domain
acct.Domain = uri.Host
if accountDomain != "" {
acct.Domain = accountDomain
} else {
acct.Domain = uri.Host
}
// avatar aka icon
// if this one isn't extractable in a format we recognise we'll just skip it

6
internal/typeutils/astointernal_test.go
View file

@ -38,7 +38,7 @@ type ASToInternalTestSuite struct {
func (suite *ASToInternalTestSuite) TestParsePerson() {
testPerson := suite.testPeople["https://unknown-instance.com/users/brand_new_person"]
acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), testPerson, false)
acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), testPerson, "", false)
suite.NoError(err)
suite.Equal("https://unknown-instance.com/users/brand_new_person", acct.URI)
@ -106,7 +106,7 @@ func (suite *ASToInternalTestSuite) TestParseGargron() {
rep, ok := t.(ap.Accountable)
suite.True(ok)
acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), rep, false)
acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), rep, "", false)
suite.NoError(err)
fmt.Printf("%+v", acct)
@ -168,7 +168,7 @@ func (suite *ASToInternalTestSuite) TestParseOwncastService() {
rep, ok := t.(ap.Accountable)
suite.True(ok)
acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), rep, false)
acct, err := suite.typeconverter.ASRepresentationToAccount(context.Background(), rep, "", false)
suite.NoError(err)
suite.Equal("rgh", acct.Username)

5
internal/typeutils/converter.go
View file

@ -99,7 +99,10 @@ type TypeConverter interface {
// If update is false, and the account is already known in the database, then the existing account entry will be returned.
// If update is true, then even if the account is already known, all fields in the accountable will be parsed and a new *gtsmodel.Account
// will be generated. This is useful when one needs to force refresh of an account, eg., during an Update of a Profile.
ASRepresentationToAccount(ctx context.Context, accountable ap.Accountable, update bool) (*gtsmodel.Account, error)
//
// If accountDomain is set (not an empty string) then this value will be used as the account's Domain. If not set,
// then the Host of the accountable's AP ID will be used instead.
ASRepresentationToAccount(ctx context.Context, accountable ap.Accountable, accountDomain string, update bool) (*gtsmodel.Account, error)
// ASStatus converts a remote activitystreams 'status' representation into a gts model status.
ASStatusToStatus(ctx context.Context, statusable ap.Statusable) (*gtsmodel.Status, error)
// ASFollowToFollowRequest converts a remote activitystreams `follow` representation into gts model follow request.

59
internal/util/namestring.go Normal file
View file

@ -0,0 +1,59 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package util
import (
"fmt"
"strings"
"github.com/superseriousbusiness/gotosocial/internal/regexes"
)
// ExtractNamestringParts extracts the username test_user and
// the domain example.org from a string like @test_user@example.org.
//
// If nothing is matched, it will return an error.
func ExtractNamestringParts(mention string) (username, host string, err error) {
matches := regexes.MentionName.FindStringSubmatch(mention)
switch len(matches) {
case 2:
return matches[1], "", nil
case 3:
return matches[1], matches[2], nil
default:
return "", "", fmt.Errorf("couldn't match mention %s", mention)
}
}
// ExtractWebfingerParts returns username test_user and
// domain example.org from a string like acct:test_user@example.org,
// or acct:@test_user@example.org.
//
// If nothing is extracted, it will return an error.
func ExtractWebfingerParts(webfinger string) (username, host string, err error) {
// remove the acct: prefix if it's present
webfinger = strings.TrimPrefix(webfinger, "acct:")
// prepend an @ if necessary
if webfinger[0] != '@' {
webfinger = "@" + webfinger
}
return ExtractNamestringParts(webfinger)
}

118
internal/util/namestring_test.go Normal file
View file

@ -0,0 +1,118 @@
/*
GoToSocial
Copyright (C) 2021-2022 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package util_test
import (
"testing"
"github.com/stretchr/testify/suite"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
type NamestringSuite struct {
suite.Suite
}
func (suite *NamestringSuite) TestExtractWebfingerParts1() {
webfinger := "acct:stonerkitty.monster@stonerkitty.monster"
username, host, err := util.ExtractWebfingerParts(webfinger)
suite.NoError(err)
suite.Equal("stonerkitty.monster", username)
suite.Equal("stonerkitty.monster", host)
}
func (suite *NamestringSuite) TestExtractWebfingerParts2() {
webfinger := "@stonerkitty.monster@stonerkitty.monster"
username, host, err := util.ExtractWebfingerParts(webfinger)
suite.NoError(err)
suite.Equal("stonerkitty.monster", username)
suite.Equal("stonerkitty.monster", host)
}
func (suite *NamestringSuite) TestExtractWebfingerParts3() {
webfinger := "acct:someone@somewhere"
username, host, err := util.ExtractWebfingerParts(webfinger)
suite.NoError(err)
suite.Equal("someone", username)
suite.Equal("somewhere", host)
}
func (suite *NamestringSuite) TestExtractWebfingerParts4() {
webfinger := "@stoner-kitty.monster@stonerkitty.monster"
username, host, err := util.ExtractWebfingerParts(webfinger)
suite.NoError(err)
suite.Equal("stoner-kitty.monster", username)
suite.Equal("stonerkitty.monster", host)
}
func (suite *NamestringSuite) TestExtractWebfingerParts5() {
webfinger := "@stonerkitty.monster"
username, host, err := util.ExtractWebfingerParts(webfinger)
suite.NoError(err)
suite.Equal("stonerkitty.monster", username)
suite.Empty(host)
}
func (suite *NamestringSuite) TestExtractWebfingerParts6() {
webfinger := "@@stonerkitty.monster"
_, _, err := util.ExtractWebfingerParts(webfinger)
suite.EqualError(err, "couldn't match mention @@stonerkitty.monster")
}
func (suite *NamestringSuite) TestExtractNamestringParts1() {
namestring := "@stonerkitty.monster@stonerkitty.monster"
username, host, err := util.ExtractNamestringParts(namestring)
suite.NoError(err)
suite.Equal("stonerkitty.monster", username)
suite.Equal("stonerkitty.monster", host)
}
func (suite *NamestringSuite) TestExtractNamestringParts2() {
namestring := "@stonerkitty.monster"
username, host, err := util.ExtractNamestringParts(namestring)
suite.NoError(err)
suite.Equal("stonerkitty.monster", username)
suite.Empty(host)
}
func (suite *NamestringSuite) TestExtractNamestringParts3() {
namestring := "@someone@somewhere"
username, host, err := util.ExtractWebfingerParts(namestring)
suite.NoError(err)
suite.Equal("someone", username)
suite.Equal("somewhere", host)
}
func (suite *NamestringSuite) TestExtractNamestringParts4() {
namestring := ""
_, _, err := util.ExtractNamestringParts(namestring)
suite.EqualError(err, "couldn't match mention ")
}
func TestNamestringSuite(t *testing.T) {
suite.Run(t, &NamestringSuite{})
}

17
internal/util/statustools.go
View file

@ -19,7 +19,6 @@
package util
import (
"fmt"
"strings"
"github.com/superseriousbusiness/gotosocial/internal/regexes"
@ -59,19 +58,3 @@ func DeriveEmojisFromText(text string) []string {
}
return UniqueStrings(emojis)
}
// ExtractMentionParts extracts the username test_user and the domain example.org
// from a mention string like @test_user@example.org.
//
// If nothing is matched, it will return an error.
func ExtractMentionParts(mention string) (username, domain string, err error) {
matches := regexes.MentionName.FindStringSubmatch(mention)
switch len(matches) {
case 2:
return matches[1], "", nil
case 3:
return matches[1], matches[2], nil
default:
return "", "", fmt.Errorf("couldn't match mention %s", mention)
}
}

19
testrig/testmodels.go
View file

@ -1909,6 +1909,25 @@ func NewTestFediAttachments(relativePath string) map[string]RemoteAttachmentFile
func NewTestFediStatuses() map[string]vocab.ActivityStreamsNote {
return map[string]vocab.ActivityStreamsNote{
"http://example.org/users/some_user/statuses/afaba698-5740-4e32-a702-af61aa543bc1": NewAPNote(
URLMustParse("http://example.org/users/some_user/statuses/afaba698-5740-4e32-a702-af61aa543bc1"),
URLMustParse("http://example.org/@some_user/afaba698-5740-4e32-a702-af61aa543bc1"),
time.Now(),
"this is a public status, please forward it!",
"",
URLMustParse("http://example.org/users/some_user"),
[]*url.URL{URLMustParse(pub.PublicActivityPubIRI)},
nil,
false,
[]vocab.ActivityStreamsMention{},
[]vocab.ActivityStreamsImage{
newAPImage(
URLMustParse("http://example.org/users/some_user/statuses/afaba698-5740-4e32-a702-af61aa543bc1/attachment1.jpeg"),
"image/jpeg",
"trent reznor looking handsome as balls",
"LEDara58O=t5EMSOENEN9]}?aK%0"),
},
),
"https://unknown-instance.com/users/brand_new_person/statuses/01FE4NTHKWW7THT67EF10EB839": NewAPNote(
URLMustParse("https://unknown-instance.com/users/brand_new_person/statuses/01FE4NTHKWW7THT67EF10EB839"),
URLMustParse("https://unknown-instance.com/users/@brand_new_person/01FE4NTHKWW7THT67EF10EB839"),

252
testrig/transportcontroller.go
View file

@ -20,10 +20,16 @@ package testrig
import (
"bytes"
"io/ioutil"
"encoding/json"
"io"
"net/http"
"strings"
"github.com/sirupsen/logrus"
"github.com/superseriousbusiness/activity/pub"
"github.com/superseriousbusiness/activity/streams"
"github.com/superseriousbusiness/activity/streams/vocab"
apimodel "github.com/superseriousbusiness/gotosocial/internal/api/model"
"github.com/superseriousbusiness/gotosocial/internal/concurrency"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/federation"
@ -31,6 +37,9 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/transport"
)
const applicationJSON = "application/json"
const applicationActivityJSON = "application/activity+json"
// NewTestTransportController returns a test transport controller with the given http client.
//
// Obviously for testing purposes you should not be making actual http calls to other servers.
@ -44,33 +53,232 @@ func NewTestTransportController(client pub.HttpClient, db db.DB, fedWorker *conc
return transport.NewController(db, NewTestFederatingDB(db, fedWorker), &federation.Clock{}, client)
}
// NewMockHTTPClient returns a client that conforms to the pub.HttpClient interface,
// but will always just execute the given `do` function, allowing responses to be mocked.
type MockHTTPClient struct {
do func(req *http.Request) (*http.Response, error)
testRemoteStatuses map[string]vocab.ActivityStreamsNote
testRemotePeople map[string]vocab.ActivityStreamsPerson
testRemoteGroups map[string]vocab.ActivityStreamsGroup
testRemoteServices map[string]vocab.ActivityStreamsService
testRemoteAttachments map[string]RemoteAttachmentFile
SentMessages map[string][]byte
}
// NewMockHTTPClient returns a client that conforms to the pub.HttpClient interface.
//
// If 'do' is nil, then a no-op function will be used instead, that just returns status 200.
// If do is nil, then a standard response set will be mocked out, which includes models stored in the
// testrig, and webfinger responses as well.
//
// If do is not nil, then the given do function will always be used, which allows callers
// to customize how the client is mocked.
//
// Note that you should never ever make ACTUAL http calls with this thing.
func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error)) pub.HttpClient {
if do == nil {
return &mockHTTPClient{
do: func(req *http.Request) (*http.Response, error) {
r := ioutil.NopCloser(bytes.NewReader([]byte{}))
return &http.Response{
StatusCode: 200,
Body: r,
}, nil
func NewMockHTTPClient(do func(req *http.Request) (*http.Response, error), relativeMediaPath string) *MockHTTPClient {
mockHTTPClient := &MockHTTPClient{}
if do != nil {
mockHTTPClient.do = do
return mockHTTPClient
}
mockHTTPClient.testRemoteStatuses = NewTestFediStatuses()
mockHTTPClient.testRemotePeople = NewTestFediPeople()
mockHTTPClient.testRemoteGroups = NewTestFediGroups()
mockHTTPClient.testRemoteServices = NewTestFediServices()
mockHTTPClient.testRemoteAttachments = NewTestFediAttachments(relativeMediaPath)
mockHTTPClient.SentMessages = make(map[string][]byte)
mockHTTPClient.do = func(req *http.Request) (*http.Response, error) {
responseCode := http.StatusNotFound
responseBytes := []byte(`{"error":"404 not found"}`)
responseContentType := applicationJSON
responseContentLength := len(responseBytes)
if req.Method == http.MethodPost {
b, err := io.ReadAll(req.Body)
if err != nil {
panic(err)
}
mockHTTPClient.SentMessages[req.URL.String()] = b
responseCode = http.StatusOK
responseBytes = []byte(`{"ok":"accepted"}`)
responseContentType = applicationJSON
responseContentLength = len(responseBytes)
} else if strings.Contains(req.URL.String(), ".well-known/webfinger") {
responseCode, responseBytes, responseContentType, responseContentLength = WebfingerResponse(req)
} else if note, ok := mockHTTPClient.testRemoteStatuses[req.URL.String()]; ok {
// the request is for a note that we have stored
noteI, err := streams.Serialize(note)
if err != nil {
panic(err)
}
noteJSON, err := json.Marshal(noteI)
if err != nil {
panic(err)
}
responseCode = http.StatusOK
responseBytes = noteJSON
responseContentType = applicationActivityJSON
responseContentLength = len(noteJSON)
} else if person, ok := mockHTTPClient.testRemotePeople[req.URL.String()]; ok {
// the request is for a person that we have stored
personI, err := streams.Serialize(person)
if err != nil {
panic(err)
}
personJSON, err := json.Marshal(personI)
if err != nil {
panic(err)
}
responseCode = http.StatusOK
responseBytes = personJSON
responseContentType = applicationActivityJSON
responseContentLength = len(personJSON)
} else if group, ok := mockHTTPClient.testRemoteGroups[req.URL.String()]; ok {
// the request is for a person that we have stored
groupI, err := streams.Serialize(group)
if err != nil {
panic(err)
}
groupJSON, err := json.Marshal(groupI)
if err != nil {
panic(err)
}
responseCode = http.StatusOK
responseBytes = groupJSON
responseContentType = applicationActivityJSON
responseContentLength = len(groupJSON)
} else if service, ok := mockHTTPClient.testRemoteServices[req.URL.String()]; ok {
serviceI, err := streams.Serialize(service)
if err != nil {
panic(err)
}
serviceJSON, err := json.Marshal(serviceI)
if err != nil {
panic(err)
}
responseCode = http.StatusOK
responseBytes = serviceJSON
responseContentType = applicationActivityJSON
responseContentLength = len(serviceJSON)
} else if attachment, ok := mockHTTPClient.testRemoteAttachments[req.URL.String()]; ok {
responseCode = http.StatusOK
responseBytes = attachment.Data
responseContentType = attachment.ContentType
responseContentLength = len(attachment.Data)
}
logrus.Debugf("returning response %s", string(responseBytes))
reader := bytes.NewReader(responseBytes)
readCloser := io.NopCloser(reader)
return &http.Response{
StatusCode: responseCode,
Body: readCloser,
ContentLength: int64(responseContentLength),
Header: http.Header{
"content-type": {responseContentType},
},
}, nil
}
return mockHTTPClient
}
func (m *MockHTTPClient) Do(req *http.Request) (*http.Response, error) {
return m.do(req)
}
func WebfingerResponse(req *http.Request) (responseCode int, responseBytes []byte, responseContentType string, responseContentLength int) {
var wfr *apimodel.WellKnownResponse
switch req.URL.String() {
case "https://unknown-instance.com/.well-known/webfinger?resource=acct:some_group@unknown-instance.com":
wfr = &apimodel.WellKnownResponse{
Subject: "acct:some_group@unknown-instance.com",
Links: []apimodel.Link{
{
Rel: "self",
Type: applicationActivityJSON,
Href: "https://unknown-instance.com/groups/some_group",
},
},
}
case "https://owncast.example.org/.well-known/webfinger?resource=acct:rgh@owncast.example.org":
wfr = &apimodel.WellKnownResponse{
Subject: "acct:rgh@example.org",
Links: []apimodel.Link{
{
Rel: "self",
Type: applicationActivityJSON,
Href: "https://owncast.example.org/federation/user/rgh",
},
},
}
case "https://unknown-instance.com/.well-known/webfinger?resource=acct:brand_new_person@unknown-instance.com":
wfr = &apimodel.WellKnownResponse{
Subject: "acct:brand_new_person@unknown-instance.com",
Links: []apimodel.Link{
{
Rel: "self",
Type: applicationActivityJSON,
Href: "https://unknown-instance.com/users/brand_new_person",
},
},
}
case "https://turnip.farm/.well-known/webfinger?resource=acct:turniplover6969@turnip.farm":
wfr = &apimodel.WellKnownResponse{
Subject: "acct:turniplover6969@turnip.farm",
Links: []apimodel.Link{
{
Rel: "self",
Type: applicationActivityJSON,
Href: "https://turnip.farm/users/turniplover6969",
},
},
}
case "https://fossbros-anonymous.io/.well-known/webfinger?resource=acct:foss_satan@fossbros-anonymous.io":
wfr = &apimodel.WellKnownResponse{
Subject: "acct:foss_satan@fossbros-anonymous.io",
Links: []apimodel.Link{
{
Rel: "self",
Type: applicationActivityJSON,
Href: "https://fossbros-anonymous.io/users/foss_satan",
},
},
}
case "https://example.org/.well-known/webfinger?resource=acct:some_user@example.org":
wfr = &apimodel.WellKnownResponse{
Subject: "acct:some_user@example.org",
Links: []apimodel.Link{
{
Rel: "self",
Type: applicationActivityJSON,
Href: "https://example.org/users/some_user",
},
},
}
}
return &mockHTTPClient{
do: do,
if wfr == nil {
logrus.Debugf("webfinger response not available for %s", req.URL)
responseCode = http.StatusNotFound
responseBytes = []byte(`{"error":"not found"}`)
responseContentType = applicationJSON
responseContentLength = len(responseBytes)
return
}
}
type mockHTTPClient struct {
do func(req *http.Request) (*http.Response, error)
}
func (m *mockHTTPClient) Do(req *http.Request) (*http.Response, error) {
return m.do(req)
wfrJSON, err := json.Marshal(wfr)
if err != nil {
panic(err)
}
responseCode = http.StatusOK
responseBytes = wfrJSON
responseContentType = applicationJSON
responseContentLength = len(wfrJSON)
return
}