2023-03-12 18:00:57 +03:00
// GoToSocial
// Copyright (C) GoToSocial Authors admin@gotosocial.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
2022-12-11 00:43:11 +03:00
package media
import (
"context"
"errors"
"fmt"
"io"
"net/url"
"github.com/superseriousbusiness/gotosocial/internal/db"
"github.com/superseriousbusiness/gotosocial/internal/gtsmodel"
"github.com/superseriousbusiness/gotosocial/internal/log"
"github.com/superseriousbusiness/gotosocial/internal/util"
)
type DereferenceMedia func ( ctx context . Context , iri * url . URL ) ( io . ReadCloser , int64 , error )
2023-05-28 15:08:35 +03:00
// RefetchEmojis iterates through remote emojis (for the given domain, or all if domain is empty string).
//
// For each emoji, the manager will check whether both the full size and static images are present in storage.
// If not, the manager will refetch and reprocess full size and static images for the emoji.
//
// The provided DereferenceMedia function will be used when it's necessary to refetch something this way.
func ( m * Manager ) RefetchEmojis ( ctx context . Context , domain string , dereferenceMedia DereferenceMedia ) ( int , error ) {
2022-12-11 00:43:11 +03:00
// normalize domain
if domain == "" {
domain = db . EmojiAllDomains
}
var (
maxShortcodeDomain string
refetchIDs [ ] string
)
// page through emojis 20 at a time, looking for those with missing images
for {
// Fetch next block of emojis from database
2023-06-22 22:46:36 +03:00
emojis , err := m . state . DB . GetEmojisBy ( ctx , domain , false , true , "" , maxShortcodeDomain , "" , 20 )
2022-12-11 00:43:11 +03:00
if err != nil {
if ! errors . Is ( err , db . ErrNoEntries ) {
2023-02-17 14:02:29 +03:00
log . Errorf ( ctx , "error fetching emojis from database: %s" , err )
2022-12-11 00:43:11 +03:00
}
break
}
for _ , emoji := range emojis {
2024-01-29 17:57:22 +03:00
if emoji . IsLocal ( ) {
2022-12-11 00:43:11 +03:00
// never try to refetch local emojis
continue
}
if refetch , err := m . emojiRequiresRefetch ( ctx , emoji ) ; err != nil {
// an error here indicates something is wrong with storage, so we should stop
return 0 , fmt . Errorf ( "error checking refetch requirement for emoji %s: %w" , util . ShortcodeDomain ( emoji ) , err )
} else if ! refetch {
continue
}
refetchIDs = append ( refetchIDs , emoji . ID )
}
// Update next maxShortcodeDomain from last emoji
maxShortcodeDomain = util . ShortcodeDomain ( emojis [ len ( emojis ) - 1 ] )
}
// bail early if we've got nothing to do
toRefetchCount := len ( refetchIDs )
if toRefetchCount == 0 {
2023-02-17 14:02:29 +03:00
log . Debug ( ctx , "no remote emojis require a refetch" )
2022-12-11 00:43:11 +03:00
return 0 , nil
}
2023-02-17 14:02:29 +03:00
log . Debugf ( ctx , "%d remote emoji(s) require a refetch, doing that now..." , toRefetchCount )
2022-12-11 00:43:11 +03:00
var totalRefetched int
for _ , emojiID := range refetchIDs {
2023-02-13 21:40:48 +03:00
emoji , err := m . state . DB . GetEmojiByID ( ctx , emojiID )
2022-12-11 00:43:11 +03:00
if err != nil {
// this shouldn't happen--since we know we have the emoji--so return if it does
return 0 , fmt . Errorf ( "error getting emoji %s: %w" , emojiID , err )
}
shortcodeDomain := util . ShortcodeDomain ( emoji )
if emoji . ImageRemoteURL == "" {
2023-02-17 14:02:29 +03:00
log . Errorf ( ctx , "remote emoji %s could not be refreshed because it has no ImageRemoteURL set" , shortcodeDomain )
2022-12-11 00:43:11 +03:00
continue
}
emojiImageIRI , err := url . Parse ( emoji . ImageRemoteURL )
if err != nil {
2023-02-17 14:02:29 +03:00
log . Errorf ( ctx , "remote emoji %s could not be refreshed because its ImageRemoteURL (%s) is not a valid uri: %s" , shortcodeDomain , emoji . ImageRemoteURL , err )
2022-12-11 00:43:11 +03:00
continue
}
dataFunc := func ( ctx context . Context ) ( reader io . ReadCloser , fileSize int64 , err error ) {
return dereferenceMedia ( ctx , emojiImageIRI )
}
2023-05-28 15:08:35 +03:00
processingEmoji , err := m . PreProcessEmoji ( ctx , dataFunc , emoji . Shortcode , emoji . ID , emoji . URI , & AdditionalEmojiInfo {
2022-12-11 00:43:11 +03:00
Domain : & emoji . Domain ,
ImageRemoteURL : & emoji . ImageRemoteURL ,
ImageStaticRemoteURL : & emoji . ImageStaticRemoteURL ,
Disabled : emoji . Disabled ,
VisibleInPicker : emoji . VisibleInPicker ,
} , true )
if err != nil {
2023-02-17 14:02:29 +03:00
log . Errorf ( ctx , "emoji %s could not be refreshed because of an error during processing: %s" , shortcodeDomain , err )
2022-12-11 00:43:11 +03:00
continue
}
if _ , err := processingEmoji . LoadEmoji ( ctx ) ; err != nil {
2023-02-17 14:02:29 +03:00
log . Errorf ( ctx , "emoji %s could not be refreshed because of an error during loading: %s" , shortcodeDomain , err )
2022-12-11 00:43:11 +03:00
continue
}
2023-02-17 14:02:29 +03:00
log . Tracef ( ctx , "refetched emoji %s successfully from remote" , shortcodeDomain )
2022-12-11 00:43:11 +03:00
totalRefetched ++
}
return totalRefetched , nil
}
2023-05-28 15:08:35 +03:00
func ( m * Manager ) emojiRequiresRefetch ( ctx context . Context , emoji * gtsmodel . Emoji ) ( bool , error ) {
2023-02-13 21:40:48 +03:00
if has , err := m . state . Storage . Has ( ctx , emoji . ImagePath ) ; err != nil {
2022-12-11 00:43:11 +03:00
return false , err
} else if ! has {
return true , nil
}
2023-02-13 21:40:48 +03:00
if has , err := m . state . Storage . Has ( ctx , emoji . ImageStaticPath ) ; err != nil {
2022-12-11 00:43:11 +03:00
return false , err
} else if ! has {
return true , nil
}
return false , nil
}