[feature] allow uncaching of other media types (#1234)

* simplify pruneRemote, remove unncecessary media trace logging, update RemoteOlderThan() to include headers/avis

Signed-off-by: kim <grufwub@gmail.com>

* cleanup pruneallmeta, add remote header to pruneremote tests

Signed-off-by: kim <grufwub@gmail.com>

* fix olderthan duration additions

Signed-off-by: kim <grufwub@gmail.com>

* fix broken test now that test model header changed

Signed-off-by: kim <grufwub@gmail.com>

* instead use new remote test account for new header model

Signed-off-by: kim <grufwub@gmail.com>

* use newer generated ULID for remote_account_3 to ensure it is sorted last

Signed-off-by: kim <grufwub@gmail.com>

* reorganize serialized keys to match expected test account model order

Signed-off-by: kim <grufwub@gmail.com>

Signed-off-by: kim <grufwub@gmail.com>

internal/db/bundb/media.go

@@ -57,8 +57,6 @@ func (m *mediaDB) GetRemoteOlderThan(ctx context.Context, olderThan time.Time, l
 		NewSelect().
 		Model(&attachments).
 		Where("? = ?", bun.Ident("media_attachment.cached"), true).
-		Where("? = ?", bun.Ident("media_attachment.avatar"), false).
-		Where("? = ?", bun.Ident("media_attachment.header"), false).
 		Where("? < ?", bun.Ident("media_attachment.created_at"), olderThan).
 		WhereGroup(" AND ", whereNotEmptyAndNotNull("media_attachment.remote_url")).
 		Order("media_attachment.created_at DESC")
@@ -70,6 +68,7 @@ func (m *mediaDB) GetRemoteOlderThan(ctx context.Context, olderThan time.Time, l
 	if err := q.Scan(ctx); err != nil {
 		return nil, m.conn.ProcessError(err)
 	}
+
 	return attachments, nil
 }
 

internal/db/bundb/media_test.go

@@ -41,7 +41,7 @@ func (suite *MediaTestSuite) TestGetAttachmentByID() {
 func (suite *MediaTestSuite) TestGetOlder() {
 	attachments, err := suite.db.GetRemoteOlderThan(context.Background(), time.Now(), 20)
 	suite.NoError(err)
-	suite.Len(attachments, 2)
+	suite.Len(attachments, 3)
 }
 
 func (suite *MediaTestSuite) TestGetAvisAndHeaders() {
@@ -49,7 +49,7 @@ func (suite *MediaTestSuite) TestGetAvisAndHeaders() {
 
 	attachments, err := suite.db.GetAvatarsAndHeaders(ctx, "", 20)
 	suite.NoError(err)
-	suite.Len(attachments, 2)
+	suite.Len(attachments, 3)
 }
 
 func (suite *MediaTestSuite) TestGetLocalUnattachedOlderThan() {

internal/db/media.go

@@ -29,18 +29,20 @@ import (
 type Media interface {
 	// GetAttachmentByID gets a single attachment by its ID
 	GetAttachmentByID(ctx context.Context, id string) (*gtsmodel.MediaAttachment, Error)
-	// GetRemoteOlderThan gets limit n remote media attachments older than the given olderThan time.
+
-	// These will be returned in order of attachment.created_at descending (newest to oldest in other words).
+	// GetRemoteOlderThan gets limit n remote media attachments (including avatars and headers) older than the given
+	// olderThan time. These will be returned in order of attachment.created_at descending (newest to oldest in other words).
 	//
 	// The selected media attachments will be those with both a URL and a RemoteURL filled in.
 	// In other words, media attachments that originated remotely, and that we currently have cached locally.
 	GetRemoteOlderThan(ctx context.Context, olderThan time.Time, limit int) ([]*gtsmodel.MediaAttachment, Error)
+
 	// GetAvatarsAndHeaders fetches limit n avatars and headers with an id < maxID. These headers
 	// and avis may be in use or not; the caller should check this if it's important.
 	GetAvatarsAndHeaders(ctx context.Context, maxID string, limit int) ([]*gtsmodel.MediaAttachment, Error)
-	// GetLocalUnattachedOlderThan fetches limit n local media attachments, older than the given time, which
+
-	// aren't header or avatars, and aren't attached to a status. In other words, attachments which were uploaded
+	// GetLocalUnattachedOlderThan fetches limit n local media attachments (including avatars and headers), older than
-	// but never used for whatever reason, or attachments that were attached to a status which was subsequently
+	// the given time, which aren't header or avatars, and aren't attached to a status. In other words, attachments which were
-	// deleted.
+	// uploaded but never used for whatever reason, or attachments that were attached to a status which was subsequently deleted.
 	GetLocalUnattachedOlderThan(ctx context.Context, olderThan time.Time, maxID string, limit int) ([]*gtsmodel.MediaAttachment, Error)
 }

internal/media/processingmedia.go

@@ -81,10 +81,8 @@ func (p *ProcessingMedia) AttachmentID() string {
 // LoadAttachment blocks until the thumbnail and fullsize content
 // has been processed, and then returns the completed attachment.
 func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAttachment, error) {
-	log.Tracef("LoadAttachment: getting lock for attachment %s", p.attachment.URL)
 	p.mu.Lock()
 	defer p.mu.Unlock()
-	log.Tracef("LoadAttachment: got lock for attachment %s", p.attachment.URL)
 
 	if err := p.store(ctx); err != nil {
 		return nil, err
@@ -98,23 +96,24 @@ func (p *ProcessingMedia) LoadAttachment(ctx context.Context) (*gtsmodel.MediaAt
 		return nil, err
 	}
 
-	// store the result in the database before returning it
 	if !p.insertedInDB {
 		if p.recache {
-			// if it's a recache we should only need to update
+			// This is an existing media attachment we're recaching, so only need to update it
 			if err := p.database.UpdateByID(ctx, p.attachment, p.attachment.ID); err != nil {
 				return nil, err
 			}
 		} else {
-			// otherwise we need to really PUT it
+			// This is a new media attachment we're caching for first time
 			if err := p.database.Put(ctx, p.attachment); err != nil {
 				return nil, err
 			}
 		}
+
+		// Mark this as stored in DB
 		p.insertedInDB = true
 	}
 
-	log.Tracef("LoadAttachment: finished, returning attachment %s", p.attachment.URL)
+	log.Tracef("finished loading attachment %s", p.attachment.URL)
 	return p.attachment, nil
 }
 
@@ -180,7 +179,7 @@ func (p *ProcessingMedia) loadThumb(ctx context.Context) error {
 
 		// we're done processing the thumbnail!
 		atomic.StoreInt32(&p.thumbState, int32(complete))
-		log.Tracef("loadThumb: finished processing thumbnail for attachment %s", p.attachment.URL)
+		log.Tracef("finished processing thumbnail for attachment %s", p.attachment.URL)
 		fallthrough
 	case complete:
 		return nil
@@ -241,7 +240,7 @@ func (p *ProcessingMedia) loadFullSize(ctx context.Context) error {
 
 		// we're done processing the full-size image
 		atomic.StoreInt32(&p.fullSizeState, int32(complete))
-		log.Tracef("loadFullSize: finished processing full size image for attachment %s", p.attachment.URL)
+		log.Tracef("finished processing full size image for attachment %s", p.attachment.URL)
 		fallthrough
 	case complete:
 		return nil
@@ -362,7 +361,7 @@ func (p *ProcessingMedia) store(ctx context.Context) error {
 	p.attachment.File.FileSize = int(fileSize)
 	p.read = true
 
-	log.Tracef("store: finished storing initial data for attachment %s", p.attachment.URL)
+	log.Tracef("finished storing initial data for attachment %s", p.attachment.URL)
 	return nil
 }
 

internal/media/prunemeta.go

@@ -20,6 +20,7 @@ package media
 
 import (
 	"context"
+	"errors"
 
 	"codeberg.org/gruf/go-store/v2/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -28,17 +29,23 @@ import (
 )
 
 func (m *manager) PruneAllMeta(ctx context.Context) (int, error) {
-	var totalPruned int
+	var (
-	var maxID string
+		totalPruned int
-	var attachments []*gtsmodel.MediaAttachment
+		maxID       string
-	var err error
+	)
+
+	for {
+		// select "selectPruneLimit" headers / avatars at a time for pruning
+		attachments, err := m.db.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit)
+		if err != nil && !errors.Is(err, db.ErrNoEntries) {
+			return totalPruned, err
+		} else if len(attachments) == 0 {
+			break
+		}
 
-	// select 20 attachments at a time and prune them
-	for attachments, err = m.db.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetAvatarsAndHeaders(ctx, maxID, selectPruneLimit) {
 		// use the id of the last attachment in the slice as the next 'maxID' value
-		l := len(attachments)
+		log.Tracef("PruneAllMeta: got %d attachments with maxID < %s", len(attachments), maxID)
-		log.Tracef("PruneAllMeta: got %d attachments with maxID < %s", l, maxID)
+		maxID = attachments[len(attachments)-1].ID
-		maxID = attachments[l-1].ID
 
 		// prune each attachment that meets one of the following criteria:
 		// - has no owning account in the database
@@ -56,11 +63,6 @@ func (m *manager) PruneAllMeta(ctx context.Context) (int, error) {
 		}
 	}
 
-	// make sure we don't have a real error when we leave the loop
-	if err != nil && err != db.ErrNoEntries {
-		return totalPruned, err
-	}
-
 	log.Infof("PruneAllMeta: finished pruning avatars + headers: pruned %d entries", totalPruned)
 	return totalPruned, nil
 }

internal/media/pruneremote.go

@@ -20,7 +20,8 @@ package media
 
 import (
 	"context"
-	"fmt"
+	"errors"
+	"time"
 
 	"codeberg.org/gruf/go-store/v2/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -31,21 +32,23 @@ import (
 func (m *manager) PruneAllRemote(ctx context.Context, olderThanDays int) (int, error) {
 	var totalPruned int
 
-	olderThan, err := parseOlderThan(olderThanDays)
+	olderThan := time.Now().Add(-time.Hour * 24 * time.Duration(olderThanDays))
-	if err != nil {
-		return totalPruned, fmt.Errorf("PruneAllRemote: error parsing olderThanDays %d: %s", olderThanDays, err)
-	}
 	log.Infof("PruneAllRemote: pruning media older than %s", olderThan)
 
-	// select 20 attachments at a time and prune them
+	for {
-	for attachments, err := m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit); err == nil && len(attachments) != 0; attachments, err = m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit) {
+		// Select "selectPruneLimit" status attacchments at a time for pruning
+		attachments, err := m.db.GetRemoteOlderThan(ctx, olderThan, selectPruneLimit)
+		if err != nil && !errors.Is(err, db.ErrNoEntries) {
+			return totalPruned, err
+		} else if len(attachments) == 0 {
+			break
+		}
 
-		// use the age of the oldest attachment (the last one in the slice) as the next 'older than' value
+		// use the age of the oldest attachment (last in slice) as the next 'olderThan' value
-		l := len(attachments)
+		log.Tracef("PruneAllRemote: got %d status attachments older than %s", len(attachments), olderThan)
-		log.Tracef("PruneAllRemote: got %d attachments older than %s", l, olderThan)
+		olderThan = attachments[len(attachments)-1].CreatedAt
-		olderThan = attachments[l-1].CreatedAt
 
-		// prune each attachment
+		// prune each status attachment
 		for _, attachment := range attachments {
 			if err := m.pruneOneRemote(ctx, attachment); err != nil {
 				return totalPruned, err
@@ -54,11 +57,6 @@ func (m *manager) PruneAllRemote(ctx context.Context, olderThanDays int) (int, e
 		}
 	}
 
-	// make sure we don't have a real error when we leave the loop
-	if err != nil && err != db.ErrNoEntries {
-		return totalPruned, err
-	}
-
 	log.Infof("PruneAllRemote: finished pruning remote media: pruned %d entries", totalPruned)
 	return totalPruned, nil
 }
@@ -69,7 +67,7 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media
 	if attachment.File.Path != "" {
 		// delete the full size attachment from storage
 		log.Tracef("pruneOneRemote: deleting %s", attachment.File.Path)
-		if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && err != storage.ErrNotFound {
+		if err := m.storage.Delete(ctx, attachment.File.Path); err != nil && !errors.Is(err, storage.ErrNotFound) {
 			return err
 		}
 		cached := false
@@ -80,7 +78,7 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media
 	if attachment.Thumbnail.Path != "" {
 		// delete the thumbnail from storage
 		log.Tracef("pruneOneRemote: deleting %s", attachment.Thumbnail.Path)
-		if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && err != storage.ErrNotFound {
+		if err := m.storage.Delete(ctx, attachment.Thumbnail.Path); err != nil && !errors.Is(err, storage.ErrNotFound) {
 			return err
 		}
 		cached := false
@@ -88,10 +86,10 @@ func (m *manager) pruneOneRemote(ctx context.Context, attachment *gtsmodel.Media
 		changed = true
 	}
 
-	// update the attachment to reflect that we no longer have it cached
+	if !changed {
-	if changed {
+		return nil
-		return m.db.UpdateByID(ctx, attachment, attachment.ID, "updated_at", "cached")
 	}
 
-	return nil
+	// update the attachment to reflect that we no longer have it cached
+	return m.db.UpdateByID(ctx, attachment, attachment.ID, "updated_at", "cached")
 }

internal/media/pruneremote_test.go

@@ -27,6 +27,7 @@ import (
 
 	"codeberg.org/gruf/go-store/v2/storage"
 	"github.com/stretchr/testify/suite"
+	"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
 )
 
 type PruneRemoteTestSuite struct {
@@ -34,24 +35,29 @@ type PruneRemoteTestSuite struct {
 }
 
 func (suite *PruneRemoteTestSuite) TestPruneRemote() {
-	testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
+	testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
-	suite.True(*testAttachment.Cached)
+	suite.True(*testStatusAttachment.Cached)
+
+	testHeader := suite.testAttachments["remote_account_3_header"]
+	suite.True(*testHeader.Cached)
 
 	totalPruned, err := suite.manager.PruneAllRemote(context.Background(), 1)
 	suite.NoError(err)
-	suite.Equal(2, totalPruned)
+	suite.Equal(3, totalPruned)
 
-	prunedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testAttachment.ID)
+	prunedAttachment, err := suite.db.GetAttachmentByID(context.Background(), testStatusAttachment.ID)
 	suite.NoError(err)
+	suite.False(*prunedAttachment.Cached)
 
-	// the media should no longer be cached
+	prunedAttachment, err = suite.db.GetAttachmentByID(context.Background(), testHeader.ID)
+	suite.NoError(err)
 	suite.False(*prunedAttachment.Cached)
 }
 
 func (suite *PruneRemoteTestSuite) TestPruneRemoteTwice() {
 	totalPruned, err := suite.manager.PruneAllRemote(context.Background(), 1)
 	suite.NoError(err)
-	suite.Equal(2, totalPruned)
+	suite.Equal(3, totalPruned)
 
 	// final prune should prune nothing, since the first prune already happened
 	totalPrunedAgain, err := suite.manager.PruneAllRemote(context.Background(), 1)
@@ -61,16 +67,21 @@ func (suite *PruneRemoteTestSuite) TestPruneRemoteTwice() {
 
 func (suite *PruneRemoteTestSuite) TestPruneAndRecache() {
 	ctx := context.Background()
-	testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
+	testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
+	testHeader := suite.testAttachments["remote_account_3_header"]
 
 	totalPruned, err := suite.manager.PruneAllRemote(ctx, 1)
 	suite.NoError(err)
-	suite.Equal(2, totalPruned)
+	suite.Equal(3, totalPruned)
 
 	// media should no longer be stored
-	_, err = suite.storage.Get(ctx, testAttachment.File.Path)
+	_, err = suite.storage.Get(ctx, testStatusAttachment.File.Path)
 	suite.ErrorIs(err, storage.ErrNotFound)
-	_, err = suite.storage.Get(ctx, testAttachment.Thumbnail.Path)
+	_, err = suite.storage.Get(ctx, testStatusAttachment.Thumbnail.Path)
+	suite.ErrorIs(err, storage.ErrNotFound)
+	_, err = suite.storage.Get(ctx, testHeader.File.Path)
+	suite.ErrorIs(err, storage.ErrNotFound)
+	_, err = suite.storage.Get(ctx, testHeader.Thumbnail.Path)
 	suite.ErrorIs(err, storage.ErrNotFound)
 
 	// now recache the image....
@@ -82,7 +93,12 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() {
 		}
 		return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil
 	}
-	processingRecache, err := suite.manager.RecacheMedia(ctx, data, nil, testAttachment.ID)
+
+	for _, original := range []*gtsmodel.MediaAttachment{
+		testStatusAttachment,
+		testHeader,
+	} {
+		processingRecache, err := suite.manager.RecacheMedia(ctx, data, nil, original.ID)
 		suite.NoError(err)
 
 		// synchronously load the recached attachment
@@ -92,24 +108,25 @@ func (suite *PruneRemoteTestSuite) TestPruneAndRecache() {
 
 		// recachedAttachment should be basically the same as the old attachment
 		suite.True(*recachedAttachment.Cached)
-	suite.Equal(testAttachment.ID, recachedAttachment.ID)
+		suite.Equal(original.ID, recachedAttachment.ID)
-	suite.Equal(testAttachment.File.Path, recachedAttachment.File.Path)           // file should be stored in the same place
+		suite.Equal(original.File.Path, recachedAttachment.File.Path)           // file should be stored in the same place
-	suite.Equal(testAttachment.Thumbnail.Path, recachedAttachment.Thumbnail.Path) // as should the thumbnail
+		suite.Equal(original.Thumbnail.Path, recachedAttachment.Thumbnail.Path) // as should the thumbnail
-	suite.EqualValues(testAttachment.FileMeta, recachedAttachment.FileMeta)       // and the filemeta should be the same
+		suite.EqualValues(original.FileMeta, recachedAttachment.FileMeta)       // and the filemeta should be the same
 
 		// recached files should be back in storage
 		_, err = suite.storage.Get(ctx, recachedAttachment.File.Path)
 		suite.NoError(err)
 		_, err = suite.storage.Get(ctx, recachedAttachment.Thumbnail.Path)
 		suite.NoError(err)
+	}
 }
 
 func (suite *PruneRemoteTestSuite) TestPruneOneNonExistent() {
 	ctx := context.Background()
-	testAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
+	testStatusAttachment := suite.testAttachments["remote_account_1_status_1_attachment_1"]
 
 	// Delete this attachment cached on disk
-	media, err := suite.db.GetAttachmentByID(ctx, testAttachment.ID)
+	media, err := suite.db.GetAttachmentByID(ctx, testStatusAttachment.ID)
 	suite.NoError(err)
 	suite.True(*media.Cached)
 	err = suite.storage.Delete(ctx, media.File.Path)
@@ -118,7 +135,7 @@ func (suite *PruneRemoteTestSuite) TestPruneOneNonExistent() {
 	// Now attempt to prune remote for item with db entry no file
 	totalPruned, err := suite.manager.PruneAllRemote(ctx, 1)
 	suite.NoError(err)
-	suite.Equal(2, totalPruned)
+	suite.Equal(3, totalPruned)
 }
 
 func TestPruneRemoteTestSuite(t *testing.T) {

internal/media/pruneunusedlocal.go

@@ -20,7 +20,7 @@ package media
 
 import (
 	"context"
-	"fmt"
+	"time"
 
 	"codeberg.org/gruf/go-store/v2/storage"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
@@ -34,10 +34,7 @@ func (m *manager) PruneUnusedLocalAttachments(ctx context.Context) (int, error)
 	var attachments []*gtsmodel.MediaAttachment
 	var err error
 
-	olderThan, err := parseOlderThan(UnusedLocalAttachmentCacheDays)
+	olderThan := time.Now().Add(-time.Hour * 24 * time.Duration(UnusedLocalAttachmentCacheDays))
-	if err != nil {
-		return totalPruned, fmt.Errorf("PruneUnusedLocalAttachments: error parsing olderThanDays %d: %s", UnusedLocalAttachmentCacheDays, err)
-	}
 	log.Infof("PruneUnusedLocalAttachments: pruning unused local attachments older than %s", olderThan)
 
 	// select 20 attachments at a time and prune them

internal/media/util.go

@@ -23,7 +23,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"time"
 
 	"github.com/h2non/filetype"
 	"github.com/superseriousbusiness/gotosocial/internal/log"
@@ -134,22 +133,6 @@ func (l *logrusWrapper) Error(err error, msg string, keysAndValues ...interface{
 	log.Error("media manager cron logger: ", err, msg, keysAndValues)
 }
 
-func parseOlderThan(olderThanDays int) (time.Time, error) {
-	// convert days into a duration string
-	olderThanHoursString := fmt.Sprintf("%dh", olderThanDays*24)
-
-	// parse the duration string into a duration
-	olderThanHours, err := time.ParseDuration(olderThanHoursString)
-	if err != nil {
-		return time.Time{}, err
-	}
-
-	// 'subtract' that from the time now to give our threshold
-	olderThan := time.Now().Add(-olderThanHours)
-
-	return olderThan, nil
-}
-
 // lengthReader wraps a reader and reads the length of total bytes written as it goes.
 type lengthReader struct {
 	source io.Reader

testrig/testmodels.go

@@ -573,6 +573,43 @@ func NewTestAccounts() map[string]*gtsmodel.Account {
 			HideCollections:       FalseBool(),
 			SuspensionOrigin:      "",
 		},
+		"remote_account_3": {
+			ID:                      "062G5WYKY35KKD12EMSM3F8PJ8",
+			Username:                "her_fuckin_maj",
+			Domain:                  "thequeenisstillalive.technology",
+			DisplayName:             "lizzzieeeeeeeeeeee",
+			Fields:                  []gtsmodel.Field{},
+			Note:                    "if i die blame charles don't let that fuck become king",
+			Memorial:                FalseBool(),
+			MovedToAccountID:        "",
+			CreatedAt:               TimeMustParse("2020-08-10T14:13:28+02:00"),
+			UpdatedAt:               TimeMustParse("2022-06-04T13:12:00Z"),
+			Bot:                     FalseBool(),
+			Locked:                  TrueBool(),
+			Discoverable:            TrueBool(),
+			Sensitive:               FalseBool(),
+			Language:                "en",
+			URI:                     "http://thequeenisstillalive.technology/users/her_fuckin_maj",
+			URL:                     "http://thequeenisstillalive.technology/@her_fuckin_maj",
+			LastWebfingeredAt:       time.Time{},
+			InboxURI:                "http://thequeenisstillalive.technology/users/her_fuckin_maj/inbox",
+			SharedInboxURI:          StringPtr(""),
+			OutboxURI:               "http://thequeenisstillalive.technology/users/her_fuckin_maj/outbox",
+			FollowersURI:            "http://thequeenisstillalive.technology/users/her_fuckin_maj/followers",
+			FollowingURI:            "http://thequeenisstillalive.technology/users/her_fuckin_maj/following",
+			FeaturedCollectionURI:   "http://thequeenisstillalive.technology/users/her_fuckin_maj/collections/featured",
+			ActorType:               ap.ActorPerson,
+			AlsoKnownAs:             "",
+			PrivateKey:              &rsa.PrivateKey{},
+			PublicKey:               &rsa.PublicKey{},
+			PublicKeyURI:            "http://thequeenisstillalive.technology/users/her_fuckin_maj#main-key",
+			SensitizedAt:            time.Time{},
+			SilencedAt:              time.Time{},
+			SuspendedAt:             time.Time{},
+			HideCollections:         FalseBool(),
+			SuspensionOrigin:        "",
+			HeaderMediaAttachmentID: "01PFPMWK2FF0D9WMHEJHR07C3R",
+		},
 	}
 
 	var accountsSorted []*gtsmodel.Account
@@ -585,6 +622,7 @@ func NewTestAccounts() map[string]*gtsmodel.Account {
 	})
 
 	preserializedKeys := []string{
+		"MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDA3bAoQMUofndXMXikEU2MOJbfI1uaZbIDrxW0bEO6IOhwe/J0jWJHL2fWc2mbp2NxAH4Db1kZIcl9D0owoRf2cT5k0Y2Dah86dGz4fIedkqGryoWAnEJ2hHKGXGQf2K9OS2L8eaDLGU4CBds0m80vrn153Uiyj7zxWDYqcySM0qQjSg+mvgqpBcxKpd+xACaWNDL8qWvDsBF1D0RuO8hUiXMIKOUoFAGbqe6qWGK0COrEYQTAMydoFuSaAccP70zKQslnSOCKvsOi/iPRKGDNqWINIC/lwqXEIpMj3K+b/A+x41zR7frTgHNLbe4yHWAVNPEwTFningbB/lIyyVmDAgMBAAECggEBALxwnipmRnyvPClMY+RiJ5PGwtqYcGsly82/pwRW98GHX7Rv1lA8x/ZnghxNPbVg0k9ZvMXcaICeu4BejQ2AiKo4sU7OVGc/K+3wTXxoKBU0bJQuV0x24JVuCXvwD7/x9i8Yh0nKCOoH+mkNkcUQKWXaJi0IoXwd5u0kVCAbym1vux/9DcwtydqT4P1EoxEHCXDuRorBP8vYWCZBwRY2etmdAEbHsVpVlNlXWfbGCNMf5e8AecOZre4No8UfTOZkM7YKgjryde3YCmY2zDQI9jExGD2L5nptLizODD5imdpp/IQ7qg6rR3XbIK6CDiKiePEFQibD8XWiz7XVD6JBRokCgYEA0jEAxZseHUyobh1ERHezs2vC2zbiTOfnOpFxhwtNt67dUQZDssTxXF+BymUL8yKi1bnheOTuyASxrgZ7BPdiFvJfhlelSxtxtt1RamY58E179uiel2NPRsR3SL2AsGg+jP+QjJpsJHvYIliXP38G7NVaqaSMFgXfXir7Ty7W0r0CgYEA6uYQWfjmaB66xPrL/oCBaJ+UWM/Zdfw4IETVnRVOxVqGE7AKqC+31fZQ5kIXnNcJNLJ0OJlhGH5vZYp/r4z6qly9BUVolCJcW2YLEOOnChOvKGwlDSXrdGty2f34RXdABwsf/pBHsdpJq70+SE01tTB/8P2NTnRafy9GL/FnwT8CgYEAjJ4D6i8wImHafHBP7441Rl9daNJ66wBqDSCoVrQVNkFiBoauW7at0iKC7ihTqkENtvY4BW0C4gVh6Q6k1lm54agch/+ysWCW3sOJaCkjscPknvZYwubJboqZUqyUn2/eCO4ggi/9ERtZKQEjjnMo6uCBWuSeY01iddlDb2HijfECgYBYQCM4ikiWKaVlyAvIDCOSWRH04/IBX8b+aJ4QrCayAraIwwTd9z+MBUSTnZUdebSdtcXwVb+i4i2b6pLaM48hXkItrswBi39DX20c5UqmgIq4Fxk8fVienpfByqbyAkFt5AIbM72b1jUDbs/tfgSFlDkdI0VpilFNo0ctT/b5JQKBgAxPGtVGzhSQUZWPXjhiBT7MM/1EiLBYhGVrymzd9dmBxj+UyifnRXfIQbOQm3EfI5Z8ZpyS6eqWdi9NTeZi8rg0WleMb/VbOMT3xvTO34vDXvwrQKhFMimX1tY7aKy1udnE2ON2/alq2zWo3zPZfYH1KFdDtGD08GW2M4OO1caa",
 		"MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDGj2wLnDIHnP6wjJ+WmIhp7NGAaKWwfxBWfdMFR+Y0ilkK5ld5igT45UHAmzN3v4HcwHGGpPITD9caDYj5YaGOX+dSdGLgXWwItR0j+ivrHEJmvz8hG6z9wKEZKUUrRw7Ob72S0LOsreq98bjdiWJKHNka27slqQjGyhLQtcg6pe1CLJtnuJH4GEMLj7jJB3/Mqv3vl5CQZ+Js0bXfgw5TF/x/Bzq/8qsxQ1vnmYHJsR0eLPEuDJOvoFPiJZytI09S7qBEJL5PDeVSfjQi3o71sqOzZlEL0b0Ny48rfo/mwJAdkmfcnydRDxeGUEqpAWICCOdUL0+W3/fCffaRZsk1AgMBAAECggEAUuyO6QJgeoF8dGsmMxSc0/ANRp1tpRpLznNZ77ipUYP9z+mG2sFjdjb4kOHASuB18aWFRAAbAQ76fGzuqYe2muk+iFcG/EDH35MUCnRuZxA0QwjX6pHOW2NZZFKyCnLwohJUj74Na65ufMk4tXysydrmaKsfq4i+m5bE6NkiOCtbXsjUGVdJKzkT6X1gEyEPEHgrgVZz9OpRY5nwjZBMcFI6EibFnWdehcuCQLESIX9ll/QzGvTJ1p8xeVJs2ktLWKQ38RewwucNYVLVJmxS1LCPP8x+yHVkOxD66eIncY26sjX+VbyICkaG/ZjKBuoOekOq/T+b6q5ESxWUNfcu+QKBgQDmt3WVBrW6EXKtN1MrVyBoSfn9WHyf8Rfb84t5iNtaWGSyPZK/arUw1DRbI0TdPjct//wMWoUU2/uqcPSzudTaPena3oxjKReXso1hcynHqboCaXJMxWSqDQLumbrVY05C1WFSyhRY0iQS5fIrNzD4+6rmeC2Aj5DKNW5Atda8dwKBgQDcUdhQfjL9SmzzIeAqJUBIfSSI2pSTsZrnrvMtSMkYJbzwYrUdhIVxaS4hXuQYmGgwonLctyvJxVxEMnf+U0nqPgJHE9nGQb5BbK6/LqxBWRJQlc+W6EYodIwvtE5B4JNkPE5757u+xlDdHe2zGUGXSIf4IjBNbSpCu6RcFsGOswKBgEnr4gqbmcJCMOH65fTu930yppxbq6J7Vs+sWrXX+aAazjilrc0S3XcFprjEth3E/10HtbQnlJg4W4wioOSs19wNFk6AG67xzZNXLCFbCrnkUarQKkUawcQSYywbqVcReFPFlmc2RAqpWdGMR2k9R72etQUe4EVeul9veyHUoTbFAoGBAKj3J9NLhaVVb8ri3vzThsJRHzTJlYrTeb5XIO5I1NhtEMK2oLobiQ+aH6O+F2Z5c+Zgn4CABdf/QSyYHAhzLcu0dKC4K5rtjpC0XiwHClovimk9C3BrgGrEP0LSn/XL2p3T1kkWRpkflKKPsl1ZcEEqggSdi7fFkdSN/ZYWaakbAoGBALWVGpA/vXmaZEV/hTDdtDnIHj6RXfKHCsfnyI7AdjUX4gokzdcEvFsEIoI+nnXR/PIAvwqvQw4wiUqQnp2VB8r73YZvW/0npnsidQw3ZjqnyvZ9X8y80nYs7DjSlaG0A8huy2TUdFnJyCMWby30g82kf0b/lhotJg4d3fIDou51",
 		"MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC6q61hiC7OhlMz7JNnLiL/RwOaFC8955GDvwSMH9Zw3oguWH9nLqkmlJ98cnqRG9ZC0qVo6Gagl7gv6yOHDwD4xZI8JoV2ZfNdDzq4QzoBIzMtRsbSS4IvrF3JP+kDH1tim+CbRMBxiFJgLgS6yeeQlLNvBW+CIYzmeCimZ6CWCr91rZPIprUIdjvhxrM9EQU072Pmzn2gpGM6K5gAReN+LtP+VSBC61x7GQJxBaJNtk11PXkgG99EdFi9vvgEBbM9bdcawvf8jxvjgsgdaDx/1cypDdnaL8eistmyv1YI67bKvrSPCEh55b90hl3o3vW4W5G4gcABoyORON96Y+i9AgMBAAECggEBAKp+tyNH0QiMo13fjFpHR2vFnsKSAPwXj063nx2kzqXUeqlp5yOE+LXmNSzjGpOCy1XJM474BRRUvsP1jkODLq4JNiF+RZP4Vij/CfDWZho33jxSUrIsiUGluxtfJiHV+A++s4zdZK/NhP+XyHYah0gEqUaTvl8q6Zhu0yH5sDCZHDLxDBpgiT5qD3lli8/o2xzzBdaibZdjQyHi9v5Yi3+ysly1tmfmqnkXSsevAubwJu504WxvDUSo7hPpG4a8Xb8ODqL738GIF2UY/olCcGkWqTQEr2pOqG9XbMmlUWnxG62GCfK6KtGfIzCyBBkGO2PZa9aPhVnv2bkYxI4PkLkCgYEAzAp7xH88UbSX31suDRa4jZwgtzhJLeyc3YxO5C4XyWZ89oWrA30V1KvfVwFRavYRJW07a+r0moba+0E1Nj5yZVXPOVu0bWd9ZyMbdH2L6MRZoJWU5bUOwyruulRCkqASZbWo4G05NOVesOyY1bhZGE7RyUW0vOo8tSyyRQ8nUGMCgYEA6jTQbDry4QkUP9tDhvc8+LsobIF1mPLEJui+mT98+9IGar6oeVDKekmNDO0Dx2+miLfjMNhCb5qUc8g036ZsekHt2WuQKunADua0coB00CebMdr6AQFf7QOQ/RuA+/gPJ5G0GzWB3YOQ5gE88tTCO/jBfmikVOZvLtgXUGjo3F8CgYEAl2poMoehQZjc41mMsRXdWukztgPE+pmORzKqENbLvB+cOG01XV9j5fCtyqklvFRioP2QjSNM5aeRtcbMMDbjOaQWJaCSImYcP39kDmxkeRXM1UhruJNGIzsm8Ys55Al53ZSTgAhN3Z0hSfYp7N/i7hD/yXc7Cr5g0qoamPkH2bUCgYApf0oeoyM9tDoeRl9knpHzEFZNQ3LusrUGn96FkLY4eDIi371CIYp+uGGBlM1CnQnI16wtj2PWGnGLQkH8DqTR1LSr/V8B+4DIIyB92TzZVOsunjoFy5SPjj42WpU0D/O/cxWSbJyh/xnBZx7Bd+kibyT5nNjhIiM5DZiz6qK3yQKBgAOO/MFKHKpKOXrtafbqCyculG/ope2u4eBveHKO6ByWcUSbuD9ebtr7Lu5AC5tKUJLkSyRx4EHk71bqP1yOITj8z9wQWdVyLxtVtyj9SUkUNvGwIj+F7NJ5VgHzWVZtvYWDCzrfxkEhKk3DRIIVjqmEohJcaOZoZ2Q/f8sjlId6",
 		"MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC1NzommDoutE+FVAbgovPb5ioRRS1k93hhH5Mpe4KfAQb1k0aGA/TjrFr2HcRbOtldo6Fe+RRCAm5Go+sBx829zyMEdXbGtR4Pym78xYoRpsCwD+2AK/edQLjsdDf9zXZod9ig/Pe59awYaeuSFyK/w9q94ncuiE7m+MKfXJnTS/qiwwkxWIRm9lBprPIT0DwXCtoh7FdpsOmjLu2QdGADV+9KSDgV5IbVcxwjPY03vHJS4UAIP5eS46TtSrNF3hyM9Q8vGIPAixOVyAY53cRQUxZWU/FIaNjaEgpreUQfK1pgr0gxh1K7IKwmyF3f/JgL0urFljYp2UonzRU5XKHJAgMBAAECggEBAKVT2HLDqTlY+b/LNGcXY+H4b+LHuS2HdUUuuGU9MKN+HWpIziuQSoi4g1hNOgp9ezgqBByQpAHBE/jQraQ3NKZ55xm3TQDm1qFTb8SfOGL4Po2iSm0IL+VA2jWnpjmgjOmshXACusPmtfakE+55uxM3TUa16UQDyfCBfZZEtnaFLTYzJ7KmD2GPot8SCxJBqNmW7AL8pMSIxMC3cRxUbK4R3+KIisXUuB50jZH3zGHxi34e2jA6gDeFmzgHCDJRidHMsCTHTaATzlvVz9YwwNqPQaYY7OFouZXwFxVAxIg/1zVvLc3zx1gWt+UDFeI7h6Eq0h5DZPdUiR4mrhAKd70CgYEAw6WKbPgjzhJI9XVmnu0aMHHH4MK8pbIq4kddChw24yZv1e9qnNTHw3YK17X9Fqog9CU1OX3M/vddfQbc34SorBmtmGYgOfDSuXTct52Ppyl4CRwndYQc0A88Hw+klluTEPY3+NRV6YSzv8vkNMasVuOh0YI1xzbpc+Bb5LL3kwMCgYEA7R4PLYYmtzKAY2YTQOXGBh3xd6UEHgks30W+QzDxvOv75svZt6yDgiwJzXtyrQzbNaH6yca5nfjkqyhnHwpguJ6DK7+S/RnZfVib5MqRwiU7g8l3neKhIXs6xZxfORunDU9T5ntbyNaGv/TJ2cXNw+9VskhBaHfEN/kmaBNNuEMCgYARLuzlfTXH15tI07Lbqn9uWc/wUao381oI3bOyO6Amey2/YHPAqn+RD0EMiRNddjvGta3jCsWCbz9qx7uGdiRKWUcB55ZVAG3BlB3+knwXdnDwe+SLUbsmGvBw2fLesdRM3RM1a5DQHbOb2NCGQhzI1N1VhVYr1QrT/pSTlZRg+QKBgCE05nc/pEhfoC9LakLaauMManaQ+4ShUFFsWPrb7d7BRaPKxJC+biRauny2XxbxB/n410BOvkvrQUre+6ITN/xi5ofH6nPbnOO69woRfFwuDqmkG0ZXKK2hrldiUMuUnc51X5CVkgMMWA6l32bKFsjryZqQF+jjbO1RzRkiKu41AoGAHQer1NyajHEpEfempx8YTsAnOn+Hi33cXAaQoTkS41lX2YK0cBkD18yhubczZcKnMW+GRKZRYXMm0NfwiuIo5oIYWeO6K+rXF+SKptC5mnw/3FhDVnghDAmEqOcRSWnFXARk1WEbFtwG5phDeFrWXsqPzGAjoZ8bhLvKRsrG4OM=",
@@ -979,6 +1017,55 @@ func NewTestAttachments() map[string]*gtsmodel.MediaAttachment {
 			Header: FalseBool(),
 			Cached: TrueBool(),
 		},
+		"remote_account_3_header": {
+			ID:        "01PFPMWK2FF0D9WMHEJHR07C3R",
+			StatusID:  "",
+			URL:       "http://localhost:8080/fileserver/062G5WYKY35KKD12EMSM3F8PJ8/header/original/01PFPMWK2FF0D9WMHEJHR07C3R.jpeg",
+			RemoteURL: "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpeg",
+			CreatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
+			UpdatedAt: TimeMustParse("2022-06-09T13:12:00Z"),
+			Type:      gtsmodel.FileTypeImage,
+			FileMeta: gtsmodel.FileMeta{
+				Original: gtsmodel.Original{
+					Width:  472,
+					Height: 291,
+					Size:   137352,
+					Aspect: 1.6219931271477663,
+				},
+				Small: gtsmodel.Small{
+					Width:  472,
+					Height: 291,
+					Size:   137352,
+					Aspect: 1.6219931271477663,
+				},
+				Focus: gtsmodel.Focus{
+					X: 0,
+					Y: 0,
+				},
+			},
+			AccountID:         "062G5WYKY35KKD12EMSM3F8PJ8",
+			Description:       "tweet from thoughts of dog: i drank. all the water. in my bowl. earlier. but just now. i returned. to the same bowl. and it was. full again.. the bowl. is haunted",
+			ScheduledStatusID: "",
+			Blurhash:          "LARysgM_IU_3~pD%M_Rj_39FIAt6",
+			Processing:        2,
+			File: gtsmodel.File{
+				Path:        "062G5WYKY35KKD12EMSM3F8PJ8/attachment/original/01PFPMWK2FF0D9WMHEJHR07C3R.jpeg",
+				ContentType: "image/jpeg",
+				FileSize:    19310,
+				UpdatedAt:   TimeMustParse("2022-06-09T13:12:00Z"),
+			},
+			Thumbnail: gtsmodel.Thumbnail{
+				Path:        "062G5WYKY35KKD12EMSM3F8PJ8/attachment/small/01PFPMWK2FF0D9WMHEJHR07C3R.jpeg",
+				ContentType: "image/jpeg",
+				FileSize:    20395,
+				UpdatedAt:   TimeMustParse("2022-06-09T13:12:00Z"),
+				URL:         "http://localhost:8080/fileserver/062G5WYKY35KKD12EMSM3F8PJ8/header/small/01PFPMWK2FF0D9WMHEJHR07C3R.jpeg",
+				RemoteURL:   "http://fossbros-anonymous.io/attachments/small/a499f55b-2d1e-4acd-98d2-1ac2ba6d79b9.jpeg",
+			},
+			Avatar: FalseBool(),
+			Header: TrueBool(),
+			Cached: TrueBool(),
+		},
 	}
 }