From 329a5e8144eea78e607c8a218ae78ae8f346f2e8 Mon Sep 17 00:00:00 2001
From: Tobi Smethurst <31960611+tsmethurst@users.noreply.github.com>
Date: Wed, 11 Aug 2021 16:54:54 +0200
Subject: [PATCH] Text duplication fix (#137)
* start testing text duplication
* tests
* fixes + tests
---
internal/processing/status/create.go | 16 +-
internal/processing/status/status.go | 13 +
internal/processing/status/status_test.go | 54 ++++
internal/processing/status/util.go | 23 +-
internal/processing/status/util_test.go | 349 ++++++++++++++++++++++
internal/text/common.go | 42 ++-
internal/text/common_test.go | 116 +++++++
internal/text/formatter_test.go | 1 +
internal/text/plain_test.go | 26 ++
internal/util/statustools.go | 16 +-
internal/util/statustools_test.go | 22 +-
testrig/db.go | 6 +
testrig/testmodels.go | 54 ++++
13 files changed, 696 insertions(+), 42 deletions(-)
create mode 100644 internal/processing/status/status_test.go
create mode 100644 internal/processing/status/util_test.go
create mode 100644 internal/text/common_test.go
diff --git a/internal/processing/status/create.go b/internal/processing/status/create.go
index 7480efd60..0e99b5f4a 100644
--- a/internal/processing/status/create.go
+++ b/internal/processing/status/create.go
@@ -39,39 +39,39 @@ func (p *processor) Create(account *gtsmodel.Account, application *gtsmodel.Appl
}
// check if replyToID is ok
- if err := p.processReplyToID(form, account.ID, newStatus); err != nil {
+ if err := p.ProcessReplyToID(form, account.ID, newStatus); err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
// check if mediaIDs are ok
- if err := p.processMediaIDs(form, account.ID, newStatus); err != nil {
+ if err := p.ProcessMediaIDs(form, account.ID, newStatus); err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
// check if visibility settings are ok
- if err := p.processVisibility(form, account.Privacy, newStatus); err != nil {
+ if err := p.ProcessVisibility(form, account.Privacy, newStatus); err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
// handle language settings
- if err := p.processLanguage(form, account.Language, newStatus); err != nil {
+ if err := p.ProcessLanguage(form, account.Language, newStatus); err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
// handle mentions
- if err := p.processMentions(form, account.ID, newStatus); err != nil {
+ if err := p.ProcessMentions(form, account.ID, newStatus); err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
- if err := p.processTags(form, account.ID, newStatus); err != nil {
+ if err := p.ProcessTags(form, account.ID, newStatus); err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
- if err := p.processEmojis(form, account.ID, newStatus); err != nil {
+ if err := p.ProcessEmojis(form, account.ID, newStatus); err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
- if err := p.processContent(form, account.ID, newStatus); err != nil {
+ if err := p.ProcessContent(form, account.ID, newStatus); err != nil {
return nil, gtserror.NewErrorInternalError(err)
}
diff --git a/internal/processing/status/status.go b/internal/processing/status/status.go
index 0073e254b..038ca005e 100644
--- a/internal/processing/status/status.go
+++ b/internal/processing/status/status.go
@@ -34,6 +34,19 @@ type Processor interface {
Unfave(account *gtsmodel.Account, targetStatusID string) (*apimodel.Status, gtserror.WithCode)
// Context returns the context (previous and following posts) from the given status ID
Context(account *gtsmodel.Account, targetStatusID string) (*apimodel.Context, gtserror.WithCode)
+
+ /*
+ PROCESSING UTILS
+ */
+
+ ProcessVisibility(form *apimodel.AdvancedStatusCreateForm, accountDefaultVis gtsmodel.Visibility, status *gtsmodel.Status) error
+ ProcessReplyToID(form *apimodel.AdvancedStatusCreateForm, thisAccountID string, status *gtsmodel.Status) error
+ ProcessMediaIDs(form *apimodel.AdvancedStatusCreateForm, thisAccountID string, status *gtsmodel.Status) error
+ ProcessLanguage(form *apimodel.AdvancedStatusCreateForm, accountDefaultLanguage string, status *gtsmodel.Status) error
+ ProcessMentions(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error
+ ProcessTags(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error
+ ProcessEmojis(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error
+ ProcessContent(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error
}
type processor struct {
diff --git a/internal/processing/status/status_test.go b/internal/processing/status/status_test.go
new file mode 100644
index 000000000..ba95a96a8
--- /dev/null
+++ b/internal/processing/status/status_test.go
@@ -0,0 +1,54 @@
+/*
+ GoToSocial
+ Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package status_test
+
+import (
+ "github.com/sirupsen/logrus"
+ "github.com/stretchr/testify/suite"
+ "github.com/superseriousbusiness/gotosocial/internal/config"
+ "github.com/superseriousbusiness/gotosocial/internal/db"
+ "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+ "github.com/superseriousbusiness/gotosocial/internal/oauth"
+ "github.com/superseriousbusiness/gotosocial/internal/processing/status"
+ "github.com/superseriousbusiness/gotosocial/internal/typeutils"
+)
+
+// nolint
+type StatusStandardTestSuite struct {
+ suite.Suite
+ config *config.Config
+ db db.DB
+ log *logrus.Logger
+ typeConverter typeutils.TypeConverter
+ fromClientAPIChan chan gtsmodel.FromClientAPI
+
+ // standard suite models
+ testTokens map[string]*oauth.Token
+ testClients map[string]*oauth.Client
+ testApplications map[string]*gtsmodel.Application
+ testUsers map[string]*gtsmodel.User
+ testAccounts map[string]*gtsmodel.Account
+ testAttachments map[string]*gtsmodel.MediaAttachment
+ testStatuses map[string]*gtsmodel.Status
+ testTags map[string]*gtsmodel.Tag
+ testMentions map[string]*gtsmodel.Mention
+
+ // module being tested
+ status status.Processor
+}
diff --git a/internal/processing/status/util.go b/internal/processing/status/util.go
index 31541ce71..3be53591b 100644
--- a/internal/processing/status/util.go
+++ b/internal/processing/status/util.go
@@ -12,7 +12,7 @@ import (
"github.com/superseriousbusiness/gotosocial/internal/util"
)
-func (p *processor) processVisibility(form *apimodel.AdvancedStatusCreateForm, accountDefaultVis gtsmodel.Visibility, status *gtsmodel.Status) error {
+func (p *processor) ProcessVisibility(form *apimodel.AdvancedStatusCreateForm, accountDefaultVis gtsmodel.Visibility, status *gtsmodel.Status) error {
// by default all flags are set to true
gtsAdvancedVis := >smodel.VisibilityAdvanced{
Federated: true,
@@ -83,7 +83,7 @@ func (p *processor) processVisibility(form *apimodel.AdvancedStatusCreateForm, a
return nil
}
-func (p *processor) processReplyToID(form *apimodel.AdvancedStatusCreateForm, thisAccountID string, status *gtsmodel.Status) error {
+func (p *processor) ProcessReplyToID(form *apimodel.AdvancedStatusCreateForm, thisAccountID string, status *gtsmodel.Status) error {
if form.InReplyToID == "" {
return nil
}
@@ -132,7 +132,7 @@ func (p *processor) processReplyToID(form *apimodel.AdvancedStatusCreateForm, th
return nil
}
-func (p *processor) processMediaIDs(form *apimodel.AdvancedStatusCreateForm, thisAccountID string, status *gtsmodel.Status) error {
+func (p *processor) ProcessMediaIDs(form *apimodel.AdvancedStatusCreateForm, thisAccountID string, status *gtsmodel.Status) error {
if form.MediaIDs == nil {
return nil
}
@@ -161,7 +161,7 @@ func (p *processor) processMediaIDs(form *apimodel.AdvancedStatusCreateForm, thi
return nil
}
-func (p *processor) processLanguage(form *apimodel.AdvancedStatusCreateForm, accountDefaultLanguage string, status *gtsmodel.Status) error {
+func (p *processor) ProcessLanguage(form *apimodel.AdvancedStatusCreateForm, accountDefaultLanguage string, status *gtsmodel.Status) error {
if form.Language != "" {
status.Language = form.Language
} else {
@@ -173,7 +173,7 @@ func (p *processor) processLanguage(form *apimodel.AdvancedStatusCreateForm, acc
return nil
}
-func (p *processor) processMentions(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
+func (p *processor) ProcessMentions(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
menchies := []string{}
gtsMenchies, err := p.db.MentionStringsToMentions(util.DeriveMentionsFromStatus(form.Status), accountID, status.ID)
if err != nil {
@@ -198,7 +198,7 @@ func (p *processor) processMentions(form *apimodel.AdvancedStatusCreateForm, acc
return nil
}
-func (p *processor) processTags(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
+func (p *processor) ProcessTags(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
tags := []string{}
gtsTags, err := p.db.TagStringsToTags(util.DeriveHashtagsFromStatus(form.Status), accountID, status.ID)
if err != nil {
@@ -217,7 +217,7 @@ func (p *processor) processTags(form *apimodel.AdvancedStatusCreateForm, account
return nil
}
-func (p *processor) processEmojis(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
+func (p *processor) ProcessEmojis(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
emojis := []string{}
gtsEmojis, err := p.db.EmojiStringsToEmojis(util.DeriveEmojisFromStatus(form.Status), accountID, status.ID)
if err != nil {
@@ -233,7 +233,7 @@ func (p *processor) processEmojis(form *apimodel.AdvancedStatusCreateForm, accou
return nil
}
-func (p *processor) processContent(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
+func (p *processor) ProcessContent(form *apimodel.AdvancedStatusCreateForm, accountID string, status *gtsmodel.Status) error {
// if there's nothing in the status at all we can just return early
if form.Status == "" {
status.Content = ""
@@ -249,15 +249,16 @@ func (p *processor) processContent(form *apimodel.AdvancedStatusCreateForm, acco
content := text.RemoveHTML(form.Status)
// parse content out of the status depending on what format has been submitted
+ var formatted string
switch form.Format {
case apimodel.StatusFormatPlain:
- content = p.formatter.FromPlain(content, status.GTSMentions, status.GTSTags)
+ formatted = p.formatter.FromPlain(content, status.GTSMentions, status.GTSTags)
case apimodel.StatusFormatMarkdown:
- content = p.formatter.FromMarkdown(content, status.GTSMentions, status.GTSTags)
+ formatted = p.formatter.FromMarkdown(content, status.GTSMentions, status.GTSTags)
default:
return fmt.Errorf("format %s not recognised as a valid status format", form.Format)
}
- status.Content = content
+ status.Content = formatted
return nil
}
diff --git a/internal/processing/status/util_test.go b/internal/processing/status/util_test.go
new file mode 100644
index 000000000..9a4bd6515
--- /dev/null
+++ b/internal/processing/status/util_test.go
@@ -0,0 +1,349 @@
+package status_test
+
+import (
+ "fmt"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/suite"
+ "github.com/superseriousbusiness/gotosocial/internal/api/model"
+ "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+ "github.com/superseriousbusiness/gotosocial/internal/processing/status"
+ "github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+const statusText1 = `Another test @foss_satan@fossbros-anonymous.io
+
+#Hashtag
+
+Text`
+const statusText1ExpectedFull = `<p>Another test <span class="h-card"><a href="http://fossbros-anonymous.io/@foss_satan" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>foss_satan</span></a></span><br/><br/><a href="http://localhost:8080/tags/Hashtag" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>Hashtag</span></a><br/><br/>Text</p>`
+const statusText1ExpectedPartial = `<p>Another test <span class="h-card"><a href="http://fossbros-anonymous.io/@foss_satan" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>foss_satan</span></a></span><br/><br/>#Hashtag<br/><br/>Text</p>`
+
+const statusText2 = `Another test @foss_satan@fossbros-anonymous.io
+
+#Hashtag
+
+#hashTAG`
+
+const status2TextExpectedFull = `<p>Another test <span class="h-card"><a href="http://fossbros-anonymous.io/@foss_satan" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>foss_satan</span></a></span><br/><br/><a href="http://localhost:8080/tags/Hashtag" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>Hashtag</span></a><br/><br/><a href="http://localhost:8080/tags/Hashtag" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>hashTAG</span></a></p>`
+
+type UtilTestSuite struct {
+ StatusStandardTestSuite
+}
+
+func (suite *UtilTestSuite) SetupSuite() {
+ suite.testTokens = testrig.NewTestTokens()
+ suite.testClients = testrig.NewTestClients()
+ suite.testApplications = testrig.NewTestApplications()
+ suite.testUsers = testrig.NewTestUsers()
+ suite.testAccounts = testrig.NewTestAccounts()
+ suite.testAttachments = testrig.NewTestAttachments()
+ suite.testStatuses = testrig.NewTestStatuses()
+ suite.testTags = testrig.NewTestTags()
+ suite.testMentions = testrig.NewTestMentions()
+}
+
+func (suite *UtilTestSuite) SetupTest() {
+ suite.config = testrig.NewTestConfig()
+ suite.db = testrig.NewTestDB()
+ suite.log = testrig.NewTestLog()
+ suite.typeConverter = testrig.NewTestTypeConverter(suite.db)
+ suite.fromClientAPIChan = make(chan gtsmodel.FromClientAPI, 100)
+ suite.status = status.New(suite.db, suite.typeConverter, suite.config, suite.fromClientAPIChan, suite.log)
+
+ testrig.StandardDBSetup(suite.db, nil)
+}
+
+func (suite *UtilTestSuite) TearDownTest() {
+ testrig.StandardDBTeardown(suite.db)
+}
+
+func (suite *UtilTestSuite) TestProcessMentions1() {
+ creatingAccount := suite.testAccounts["local_account_1"]
+ mentionedAccount := suite.testAccounts["remote_account_1"]
+
+ form := &model.AdvancedStatusCreateForm{
+ StatusCreateRequest: model.StatusCreateRequest{
+ Status: statusText1,
+ MediaIDs: []string{},
+ Poll: nil,
+ InReplyToID: "",
+ Sensitive: false,
+ SpoilerText: "",
+ Visibility: model.VisibilityPublic,
+ ScheduledAt: "",
+ Language: "en",
+ Format: model.StatusFormatPlain,
+ },
+ AdvancedVisibilityFlagsForm: model.AdvancedVisibilityFlagsForm{
+ Federated: nil,
+ Boostable: nil,
+ Replyable: nil,
+ Likeable: nil,
+ },
+ }
+
+ status := >smodel.Status{
+ ID: "01FCTDD78JJMX3K9KPXQ7ZQ8BJ",
+ }
+
+ err := suite.status.ProcessMentions(form, creatingAccount.ID, status)
+ assert.NoError(suite.T(), err)
+
+ assert.Len(suite.T(), status.GTSMentions, 1)
+ newMention := status.GTSMentions[0]
+ assert.Equal(suite.T(), mentionedAccount.ID, newMention.TargetAccountID)
+ assert.Equal(suite.T(), creatingAccount.ID, newMention.OriginAccountID)
+ assert.Equal(suite.T(), creatingAccount.URI, newMention.OriginAccountURI)
+ assert.Equal(suite.T(), status.ID, newMention.StatusID)
+ assert.Equal(suite.T(), fmt.Sprintf("@%s@%s", mentionedAccount.Username, mentionedAccount.Domain), newMention.NameString)
+ assert.Equal(suite.T(), mentionedAccount.URI, newMention.MentionedAccountURI)
+ assert.Equal(suite.T(), mentionedAccount.URL, newMention.MentionedAccountURL)
+ assert.NotNil(suite.T(), newMention.GTSAccount)
+
+ assert.Len(suite.T(), status.Mentions, 1)
+ assert.Equal(suite.T(), newMention.ID, status.Mentions[0])
+}
+
+func (suite *UtilTestSuite) TestProcessContentFull1() {
+
+ /*
+ TEST PREPARATION
+ */
+ // we need to partially process the status first since processContent expects a status with some stuff already set on it
+ creatingAccount := suite.testAccounts["local_account_1"]
+ form := &model.AdvancedStatusCreateForm{
+ StatusCreateRequest: model.StatusCreateRequest{
+ Status: statusText1,
+ MediaIDs: []string{},
+ Poll: nil,
+ InReplyToID: "",
+ Sensitive: false,
+ SpoilerText: "",
+ Visibility: model.VisibilityPublic,
+ ScheduledAt: "",
+ Language: "en",
+ Format: model.StatusFormatPlain,
+ },
+ AdvancedVisibilityFlagsForm: model.AdvancedVisibilityFlagsForm{
+ Federated: nil,
+ Boostable: nil,
+ Replyable: nil,
+ Likeable: nil,
+ },
+ }
+
+ status := >smodel.Status{
+ ID: "01FCTDD78JJMX3K9KPXQ7ZQ8BJ",
+ }
+
+ err := suite.status.ProcessMentions(form, creatingAccount.ID, status)
+ assert.NoError(suite.T(), err)
+ assert.Empty(suite.T(), status.Content) // shouldn't be set yet
+
+ err = suite.status.ProcessTags(form, creatingAccount.ID, status)
+ assert.NoError(suite.T(), err)
+ assert.Empty(suite.T(), status.Content) // shouldn't be set yet
+
+ /*
+ ACTUAL TEST
+ */
+
+ err = suite.status.ProcessContent(form, creatingAccount.ID, status)
+ assert.NoError(suite.T(), err)
+ assert.Equal(suite.T(), statusText1ExpectedFull, status.Content)
+}
+
+func (suite *UtilTestSuite) TestProcessContentPartial1() {
+
+ /*
+ TEST PREPARATION
+ */
+ // we need to partially process the status first since processContent expects a status with some stuff already set on it
+ creatingAccount := suite.testAccounts["local_account_1"]
+ form := &model.AdvancedStatusCreateForm{
+ StatusCreateRequest: model.StatusCreateRequest{
+ Status: statusText1,
+ MediaIDs: []string{},
+ Poll: nil,
+ InReplyToID: "",
+ Sensitive: false,
+ SpoilerText: "",
+ Visibility: model.VisibilityPublic,
+ ScheduledAt: "",
+ Language: "en",
+ Format: model.StatusFormatPlain,
+ },
+ AdvancedVisibilityFlagsForm: model.AdvancedVisibilityFlagsForm{
+ Federated: nil,
+ Boostable: nil,
+ Replyable: nil,
+ Likeable: nil,
+ },
+ }
+
+ status := >smodel.Status{
+ ID: "01FCTDD78JJMX3K9KPXQ7ZQ8BJ",
+ }
+
+ err := suite.status.ProcessMentions(form, creatingAccount.ID, status)
+ assert.NoError(suite.T(), err)
+ assert.Empty(suite.T(), status.Content) // shouldn't be set yet
+
+ /*
+ ACTUAL TEST
+ */
+
+ err = suite.status.ProcessContent(form, creatingAccount.ID, status)
+ assert.NoError(suite.T(), err)
+ assert.Equal(suite.T(), statusText1ExpectedPartial, status.Content)
+}
+
+func (suite *UtilTestSuite) TestProcessMentions2() {
+ creatingAccount := suite.testAccounts["local_account_1"]
+ mentionedAccount := suite.testAccounts["remote_account_1"]
+
+ form := &model.AdvancedStatusCreateForm{
+ StatusCreateRequest: model.StatusCreateRequest{
+ Status: statusText2,
+ MediaIDs: []string{},
+ Poll: nil,
+ InReplyToID: "",
+ Sensitive: false,
+ SpoilerText: "",
+ Visibility: model.VisibilityPublic,
+ ScheduledAt: "",
+ Language: "en",
+ Format: model.StatusFormatPlain,
+ },
+ AdvancedVisibilityFlagsForm: model.AdvancedVisibilityFlagsForm{
+ Federated: nil,
+ Boostable: nil,
+ Replyable: nil,
+ Likeable: nil,
+ },
+ }
+
+ status := >smodel.Status{
+ ID: "01FCTDD78JJMX3K9KPXQ7ZQ8BJ",
+ }
+
+ err := suite.status.ProcessMentions(form, creatingAccount.ID, status)
+ assert.NoError(suite.T(), err)
+
+ assert.Len(suite.T(), status.GTSMentions, 1)
+ newMention := status.GTSMentions[0]
+ assert.Equal(suite.T(), mentionedAccount.ID, newMention.TargetAccountID)
+ assert.Equal(suite.T(), creatingAccount.ID, newMention.OriginAccountID)
+ assert.Equal(suite.T(), creatingAccount.URI, newMention.OriginAccountURI)
+ assert.Equal(suite.T(), status.ID, newMention.StatusID)
+ assert.Equal(suite.T(), fmt.Sprintf("@%s@%s", mentionedAccount.Username, mentionedAccount.Domain), newMention.NameString)
+ assert.Equal(suite.T(), mentionedAccount.URI, newMention.MentionedAccountURI)
+ assert.Equal(suite.T(), mentionedAccount.URL, newMention.MentionedAccountURL)
+ assert.NotNil(suite.T(), newMention.GTSAccount)
+
+ assert.Len(suite.T(), status.Mentions, 1)
+ assert.Equal(suite.T(), newMention.ID, status.Mentions[0])
+}
+
+func (suite *UtilTestSuite) TestProcessContentFull2() {
+
+ /*
+ TEST PREPARATION
+ */
+ // we need to partially process the status first since processContent expects a status with some stuff already set on it
+ creatingAccount := suite.testAccounts["local_account_1"]
+ form := &model.AdvancedStatusCreateForm{
+ StatusCreateRequest: model.StatusCreateRequest{
+ Status: statusText2,
+ MediaIDs: []string{},
+ Poll: nil,
+ InReplyToID: "",
+ Sensitive: false,
+ SpoilerText: "",
+ Visibility: model.VisibilityPublic,
+ ScheduledAt: "",
+ Language: "en",
+ Format: model.StatusFormatPlain,
+ },
+ AdvancedVisibilityFlagsForm: model.AdvancedVisibilityFlagsForm{
+ Federated: nil,
+ Boostable: nil,
+ Replyable: nil,
+ Likeable: nil,
+ },
+ }
+
+ status := >smodel.Status{
+ ID: "01FCTDD78JJMX3K9KPXQ7ZQ8BJ",
+ }
+
+ err := suite.status.ProcessMentions(form, creatingAccount.ID, status)
+ assert.NoError(suite.T(), err)
+ assert.Empty(suite.T(), status.Content) // shouldn't be set yet
+
+ err = suite.status.ProcessTags(form, creatingAccount.ID, status)
+ assert.NoError(suite.T(), err)
+ assert.Empty(suite.T(), status.Content) // shouldn't be set yet
+
+ /*
+ ACTUAL TEST
+ */
+
+ err = suite.status.ProcessContent(form, creatingAccount.ID, status)
+ assert.NoError(suite.T(), err)
+
+ assert.Equal(suite.T(), status2TextExpectedFull, status.Content)
+}
+
+func (suite *UtilTestSuite) TestProcessContentPartial2() {
+
+ /*
+ TEST PREPARATION
+ */
+ // we need to partially process the status first since processContent expects a status with some stuff already set on it
+ creatingAccount := suite.testAccounts["local_account_1"]
+ form := &model.AdvancedStatusCreateForm{
+ StatusCreateRequest: model.StatusCreateRequest{
+ Status: statusText2,
+ MediaIDs: []string{},
+ Poll: nil,
+ InReplyToID: "",
+ Sensitive: false,
+ SpoilerText: "",
+ Visibility: model.VisibilityPublic,
+ ScheduledAt: "",
+ Language: "en",
+ Format: model.StatusFormatPlain,
+ },
+ AdvancedVisibilityFlagsForm: model.AdvancedVisibilityFlagsForm{
+ Federated: nil,
+ Boostable: nil,
+ Replyable: nil,
+ Likeable: nil,
+ },
+ }
+
+ status := >smodel.Status{
+ ID: "01FCTDD78JJMX3K9KPXQ7ZQ8BJ",
+ }
+
+ err := suite.status.ProcessMentions(form, creatingAccount.ID, status)
+ assert.NoError(suite.T(), err)
+ assert.Empty(suite.T(), status.Content) // shouldn't be set yet
+
+ /*
+ ACTUAL TEST
+ */
+
+ err = suite.status.ProcessContent(form, creatingAccount.ID, status)
+ assert.NoError(suite.T(), err)
+
+ fmt.Println(status.Content)
+ // assert.Equal(suite.T(), statusText2ExpectedPartial, status.Content)
+}
+
+func TestUtilTestSuite(t *testing.T) {
+ suite.Run(t, new(UtilTestSuite))
+}
diff --git a/internal/text/common.go b/internal/text/common.go
index 98ec892a7..4f0bad9dc 100644
--- a/internal/text/common.go
+++ b/internal/text/common.go
@@ -50,26 +50,54 @@ func postformat(in string) string {
func (f *formatter) ReplaceTags(in string, tags []*gtsmodel.Tag) string {
return util.HashtagFinderRegex.ReplaceAllStringFunc(in, func(match string) string {
+ // we have a match
+ matchTrimmed := strings.TrimSpace(match)
+ tagAsEntered := strings.Split(matchTrimmed, "#")[1]
+
+ // check through the tags to find what we're matching
for _, tag := range tags {
- if strings.TrimSpace(match) == fmt.Sprintf("#%s", tag.Name) {
- tagContent := fmt.Sprintf(`<a href="%s" class="mention hashtag" rel="tag">#<span>%s</span></a>`, tag.URL, tag.Name)
+
+ if strings.EqualFold(matchTrimmed, fmt.Sprintf("#%s", tag.Name)) {
+ // replace the #tag with the formatted tag content
+ tagContent := fmt.Sprintf(`<a href="%s" class="mention hashtag" rel="tag">#<span>%s</span></a>`, tag.URL, tagAsEntered)
+
+ // in case the match picked up any previous space or newlines (thanks to the regex), include them as well
if strings.HasPrefix(match, " ") {
tagContent = " " + tagContent
+ } else if strings.HasPrefix(match, "\n") {
+ tagContent = "\n" + tagContent
}
+
+ // done
return tagContent
}
}
- return in
+ // the match wasn't in the list of tags for whatever reason, so just return the match as we found it so nothing changes
+ return match
})
}
func (f *formatter) ReplaceMentions(in string, mentions []*gtsmodel.Mention) string {
for _, menchie := range mentions {
- targetAccount := >smodel.Account{}
- if err := f.db.GetByID(menchie.TargetAccountID, targetAccount); err == nil {
- mentionContent := fmt.Sprintf(`<span class="h-card"><a href="%s" class="u-url mention">@<span>%s</span></a></span>`, targetAccount.URL, targetAccount.Username)
- in = strings.ReplaceAll(in, menchie.NameString, mentionContent)
+ // make sure we have a target account, either by getting one pinned on the mention,
+ // or by pulling it from the database
+ var targetAccount *gtsmodel.Account
+ if menchie.GTSAccount != nil {
+ // got it from the mention
+ targetAccount = menchie.GTSAccount
+ } else {
+ a := >smodel.Account{}
+ if err := f.db.GetByID(menchie.TargetAccountID, a); err == nil {
+ // got it from the db
+ targetAccount = a
+ } else {
+ // couldn't get it so we can't do replacement
+ return in
+ }
}
+
+ mentionContent := fmt.Sprintf(`<span class="h-card"><a href="%s" class="u-url mention">@<span>%s</span></a></span>`, targetAccount.URL, targetAccount.Username)
+ in = strings.ReplaceAll(in, menchie.NameString, mentionContent)
}
return in
}
diff --git a/internal/text/common_test.go b/internal/text/common_test.go
new file mode 100644
index 000000000..69fe7d446
--- /dev/null
+++ b/internal/text/common_test.go
@@ -0,0 +1,116 @@
+/*
+ GoToSocial
+ Copyright (C) 2021 GoToSocial Authors admin@gotosocial.org
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+package text_test
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/suite"
+ "github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
+ "github.com/superseriousbusiness/gotosocial/internal/text"
+ "github.com/superseriousbusiness/gotosocial/testrig"
+)
+
+const (
+ replaceMentionsString = `Another test @foss_satan@fossbros-anonymous.io
+
+#Hashtag
+
+Text`
+ replaceMentionsExpected = `Another test <span class="h-card"><a href="http://fossbros-anonymous.io/@foss_satan" class="u-url mention">@<span>foss_satan</span></a></span>
+
+#Hashtag
+
+Text`
+
+ replaceHashtagsExpected = `Another test @foss_satan@fossbros-anonymous.io
+
+<a href="http://localhost:8080/tags/Hashtag" class="mention hashtag" rel="tag">#<span>Hashtag</span></a>
+
+Text`
+
+ replaceHashtagsAfterMentionsExpected = `Another test <span class="h-card"><a href="http://fossbros-anonymous.io/@foss_satan" class="u-url mention">@<span>foss_satan</span></a></span>
+
+<a href="http://localhost:8080/tags/Hashtag" class="mention hashtag" rel="tag">#<span>Hashtag</span></a>
+
+Text`
+)
+
+type CommonTestSuite struct {
+ TextStandardTestSuite
+}
+
+func (suite *CommonTestSuite) SetupSuite() {
+ suite.testTokens = testrig.NewTestTokens()
+ suite.testClients = testrig.NewTestClients()
+ suite.testApplications = testrig.NewTestApplications()
+ suite.testUsers = testrig.NewTestUsers()
+ suite.testAccounts = testrig.NewTestAccounts()
+ suite.testAttachments = testrig.NewTestAttachments()
+ suite.testStatuses = testrig.NewTestStatuses()
+ suite.testTags = testrig.NewTestTags()
+ suite.testMentions = testrig.NewTestMentions()
+}
+
+func (suite *CommonTestSuite) SetupTest() {
+ suite.config = testrig.NewTestConfig()
+ suite.db = testrig.NewTestDB()
+ suite.log = testrig.NewTestLog()
+ suite.formatter = text.NewFormatter(suite.config, suite.db, suite.log)
+
+ testrig.StandardDBSetup(suite.db, nil)
+}
+
+func (suite *CommonTestSuite) TearDownTest() {
+ testrig.StandardDBTeardown(suite.db)
+}
+
+func (suite *CommonTestSuite) TestReplaceMentions() {
+ foundMentions := []*gtsmodel.Mention{
+ suite.testMentions["zork_mention_foss_satan"],
+ }
+
+ f := suite.formatter.ReplaceMentions(replaceMentionsString, foundMentions)
+ assert.Equal(suite.T(), replaceMentionsExpected, f)
+}
+
+func (suite *CommonTestSuite) TestReplaceHashtags() {
+ foundTags := []*gtsmodel.Tag{
+ suite.testTags["Hashtag"],
+ }
+
+ f := suite.formatter.ReplaceTags(replaceMentionsString, foundTags)
+
+ assert.Equal(suite.T(), replaceHashtagsExpected, f)
+}
+
+func (suite *CommonTestSuite) TestReplaceHashtagsAfterReplaceMentions() {
+ foundTags := []*gtsmodel.Tag{
+ suite.testTags["Hashtag"],
+ }
+
+ f := suite.formatter.ReplaceTags(replaceMentionsExpected, foundTags)
+
+ assert.Equal(suite.T(), replaceHashtagsAfterMentionsExpected, f)
+}
+
+func TestCommonTestSuite(t *testing.T) {
+ suite.Run(t, new(CommonTestSuite))
+}
diff --git a/internal/text/formatter_test.go b/internal/text/formatter_test.go
index 2c9c18546..803088794 100644
--- a/internal/text/formatter_test.go
+++ b/internal/text/formatter_test.go
@@ -45,6 +45,7 @@ type TextStandardTestSuite struct {
testAttachments map[string]*gtsmodel.MediaAttachment
testStatuses map[string]*gtsmodel.Status
testTags map[string]*gtsmodel.Tag
+ testMentions map[string]*gtsmodel.Mention
// module being tested
formatter text.Formatter
diff --git a/internal/text/plain_test.go b/internal/text/plain_test.go
index 183ccc478..2f9eb3a29 100644
--- a/internal/text/plain_test.go
+++ b/internal/text/plain_test.go
@@ -19,6 +19,7 @@
package text_test
import (
+ "fmt"
"testing"
"github.com/stretchr/testify/assert"
@@ -34,6 +35,13 @@ const (
withTag = "this is a simple status that uses hashtag #welcome!"
withTagExpected = "<p>this is a simple status that uses hashtag <a href=\"http://localhost:8080/tags/welcome\" class=\"mention hashtag\" rel=\"tag nofollow noreferrer noopener\" target=\"_blank\">#<span>welcome</span></a>!</p>"
+
+ moreComplex = `Another test @foss_satan@fossbros-anonymous.io
+
+#Hashtag
+
+Text`
+ moreComplexExpected = `<p>Another test <span class="h-card"><a href="http://fossbros-anonymous.io/@foss_satan" class="u-url mention" rel="nofollow noreferrer noopener" target="_blank">@<span>foss_satan</span></a></span><br/><br/><a href="http://localhost:8080/tags/Hashtag" class="mention hashtag" rel="tag nofollow noreferrer noopener" target="_blank">#<span>Hashtag</span></a><br/><br/>Text</p>`
)
type PlainTestSuite struct {
@@ -49,6 +57,7 @@ func (suite *PlainTestSuite) SetupSuite() {
suite.testAttachments = testrig.NewTestAttachments()
suite.testStatuses = testrig.NewTestStatuses()
suite.testTags = testrig.NewTestTags()
+ suite.testMentions = testrig.NewTestMentions()
}
func (suite *PlainTestSuite) SetupTest() {
@@ -79,6 +88,23 @@ func (suite *PlainTestSuite) TestParseWithTag() {
assert.Equal(suite.T(), withTagExpected, f)
}
+func (suite *PlainTestSuite) TestParseMoreComplex() {
+
+ foundTags := []*gtsmodel.Tag{
+ suite.testTags["Hashtag"],
+ }
+
+ foundMentions := []*gtsmodel.Mention{
+ suite.testMentions["zork_mention_foss_satan"],
+ }
+
+ f := suite.formatter.FromPlain(moreComplex, foundMentions, foundTags)
+
+ fmt.Println(f)
+
+ assert.Equal(suite.T(), moreComplexExpected, f)
+}
+
func TestPlainTestSuite(t *testing.T) {
suite.Run(t, new(PlainTestSuite))
}
diff --git a/internal/util/statustools.go b/internal/util/statustools.go
index 93294da68..ce5860c6d 100644
--- a/internal/util/statustools.go
+++ b/internal/util/statustools.go
@@ -46,7 +46,7 @@ func DeriveHashtagsFromStatus(status string) []string {
for _, m := range HashtagFinderRegex.FindAllStringSubmatch(status, -1) {
tags = append(tags, strings.TrimPrefix(m[1], "#"))
}
- return uniqueLower(tags)
+ return unique(tags)
}
// DeriveEmojisFromStatus takes a plaintext (ie., not html-formatted) status,
@@ -92,17 +92,3 @@ func unique(s []string) []string {
}
return list
}
-
-// uniqueLower returns a deduplicated version of a given string slice, with all entries converted to lowercase
-func uniqueLower(s []string) []string {
- keys := make(map[string]bool)
- list := []string{}
- for _, entry := range s {
- eLower := strings.ToLower(entry)
- if _, value := keys[eLower]; !value {
- keys[eLower] = true
- list = append(list, eLower)
- }
- }
- return list
-}
diff --git a/internal/util/statustools_test.go b/internal/util/statustools_test.go
index 5bdce2d5a..0ec2719f5 100644
--- a/internal/util/statustools_test.go
+++ b/internal/util/statustools_test.go
@@ -79,7 +79,7 @@ func (suite *StatusTestSuite) TestDeriveHashtagsOK() {
assert.Equal(suite.T(), "testing123", tags[0])
assert.Equal(suite.T(), "also", tags[1])
assert.Equal(suite.T(), "thisshouldwork", tags[2])
- assert.Equal(suite.T(), "thisshouldalsowork", tags[3])
+ assert.Equal(suite.T(), "ThisShouldAlsoWork", tags[3])
assert.Equal(suite.T(), "111111", tags[4])
}
@@ -108,6 +108,26 @@ Here's some normal text with an :emoji: at the end
assert.Equal(suite.T(), "underscores_ok_too", tags[6])
}
+func (suite *StatusTestSuite) TestDeriveMultiple() {
+ statusText := `Another test @foss_satan@fossbros-anonymous.io
+
+ #Hashtag
+
+ Text`
+
+ ms := util.DeriveMentionsFromStatus(statusText)
+ hs := util.DeriveHashtagsFromStatus(statusText)
+ es := util.DeriveEmojisFromStatus(statusText)
+
+ assert.Len(suite.T(), ms, 1)
+ assert.Equal(suite.T(), "@foss_satan@fossbros-anonymous.io", ms[0])
+
+ assert.Len(suite.T(), hs, 1)
+ assert.Equal(suite.T(), "Hashtag", hs[0])
+
+ assert.Len(suite.T(), es, 0)
+}
+
func TestStatusTestSuite(t *testing.T) {
suite.Run(t, new(StatusTestSuite))
}
diff --git a/testrig/db.go b/testrig/db.go
index fe38c3164..f34f7936b 100644
--- a/testrig/db.go
+++ b/testrig/db.go
@@ -141,6 +141,12 @@ func StandardDBSetup(db db.DB, accounts map[string]*gtsmodel.Account) {
}
}
+ for _, v := range NewTestMentions() {
+ if err := db.Put(v); err != nil {
+ panic(err)
+ }
+ }
+
for _, v := range NewTestFaves() {
if err := db.Put(v); err != nil {
panic(err)
diff --git a/testrig/testmodels.go b/testrig/testmodels.go
index da5cbe7af..77274474c 100644
--- a/testrig/testmodels.go
+++ b/testrig/testmodels.go
@@ -937,6 +937,31 @@ func NewTestStatuses() map[string]*gtsmodel.Status {
},
ActivityStreamsType: gtsmodel.ActivityStreamsNote,
},
+ "local_account_1_status_5": {
+ ID: "01FCTA44PW9H1TB328S9AQXKDS",
+ URI: "http://localhost:8080/users/the_mighty_zork/statuses/01FCTA44PW9H1TB328S9AQXKDS",
+ URL: "http://localhost:8080/@the_mighty_zork/statuses/01FCTA44PW9H1TB328S9AQXKDS",
+ Content: "hi!",
+ Attachments: []string{},
+ CreatedAt: time.Now().Add(-1 * time.Minute),
+ UpdatedAt: time.Now().Add(-1 * time.Minute),
+ Local: true,
+ AccountID: "01F8MH1H7YV1Z7D2C8K2730QBF",
+ InReplyToID: "",
+ BoostOfID: "",
+ ContentWarning: "",
+ Visibility: gtsmodel.VisibilityMutualsOnly,
+ Sensitive: false,
+ Language: "en",
+ CreatedWithApplicationID: "01F8MGY43H3N2C8EWPR2FPYEXG",
+ VisibilityAdvanced: >smodel.VisibilityAdvanced{
+ Federated: true,
+ Boostable: true,
+ Replyable: true,
+ Likeable: true,
+ },
+ ActivityStreamsType: gtsmodel.ActivityStreamsNote,
+ },
"local_account_2_status_1": {
ID: "01F8MHBQCBTDKN6X5VHGMMN4MA",
URI: "http://localhost:8080/users/1happyturtle/statuses/01F8MHBQCBTDKN6X5VHGMMN4MA",
@@ -1076,6 +1101,35 @@ func NewTestTags() map[string]*gtsmodel.Tag {
Listable: true,
LastStatusAt: time.Now().Add(-71 * time.Hour),
},
+ "Hashtag": {
+ ID: "01FCT9SGYA71487N8D0S1M638G",
+ URL: "http://localhost:8080/tags/Hashtag",
+ Name: "Hashtag",
+ FirstSeenFromAccountID: "",
+ CreatedAt: time.Now().Add(-71 * time.Hour),
+ UpdatedAt: time.Now().Add(-71 * time.Hour),
+ Useable: true,
+ Listable: true,
+ LastStatusAt: time.Now().Add(-71 * time.Hour),
+ },
+ }
+}
+
+// NewTestMentions returns a map of gts model mentions keyed by their name.
+func NewTestMentions() map[string]*gtsmodel.Mention {
+ return map[string]*gtsmodel.Mention{
+ "zork_mention_foss_satan": {
+ ID: "01FCTA2Y6FGHXQA4ZE6N5NMNEX",
+ StatusID: "01FCTA44PW9H1TB328S9AQXKDS",
+ CreatedAt: time.Now().Add(-1 * time.Minute),
+ UpdatedAt: time.Now().Add(-1 * time.Minute),
+ OriginAccountID: "01F8MH1H7YV1Z7D2C8K2730QBF",
+ OriginAccountURI: "http://localhost:8080/users/the_mighty_zork",
+ TargetAccountID: "01F8MH5ZK5VRH73AKHQM6Y9VNX",
+ NameString: "@foss_satan@fossbros-anonymous.io",
+ MentionedAccountURI: "http://fossbros-anonymous.io/users/foss_satan",
+ MentionedAccountURL: "http://fossbros-anonymous.io/@foss_satan",
+ },
}
}