gotosocial/internal/iotools/io.go
kim 5318054808
[performance] media processing improvements (#1288)
* media processor consolidation and reformatting, reduce amount of required syscalls

Signed-off-by: kim <grufwub@gmail.com>

* update go-store library, stream jpeg/png encoding + use buffer pools, improved media processing AlreadyExists error handling

Signed-off-by: kim <grufwub@gmail.com>

* fix duration not being set, fix mp4 test expecting error

Signed-off-by: kim <grufwub@gmail.com>

* fix test expecting media files with different extension

Signed-off-by: kim <grufwub@gmail.com>

* remove unused code

Signed-off-by: kim <grufwub@gmail.com>

* fix expected storage paths in tests, update expected test thumbnails

Signed-off-by: kim <grufwub@gmail.com>

* remove dead code

Signed-off-by: kim <grufwub@gmail.com>

* fix cached presigned s3 url fetching

Signed-off-by: kim <grufwub@gmail.com>

* fix tests

Signed-off-by: kim <grufwub@gmail.com>

* fix test models

Signed-off-by: kim <grufwub@gmail.com>

* update media processing to use sync.Once{} for concurrency protection

Signed-off-by: kim <grufwub@gmail.com>

* shutup linter

Signed-off-by: kim <grufwub@gmail.com>

* fix passing in KVStore GetStream() as stream to PutStream()

Signed-off-by: kim <grufwub@gmail.com>

* fix unlocks of storage keys

Signed-off-by: kim <grufwub@gmail.com>

* whoops, return the error...

Signed-off-by: kim <grufwub@gmail.com>

* pour one out for tobi's code <3

Signed-off-by: kim <grufwub@gmail.com>

* add back the byte slurping code

Signed-off-by: kim <grufwub@gmail.com>

* check for both ErrUnexpectedEOF and EOF

Signed-off-by: kim <grufwub@gmail.com>

* add back links to file format header information

Signed-off-by: kim <grufwub@gmail.com>

Signed-off-by: kim <grufwub@gmail.com>
2023-01-11 12:13:13 +01:00

159 lines
3.3 KiB
Go

/*
GoToSocial
Copyright (C) 2021-2023 GoToSocial Authors admin@gotosocial.org
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package iotools
import (
"io"
)
// ReadFnCloser takes an io.Reader and wraps it to use the provided function to implement io.Closer.
func ReadFnCloser(r io.Reader, close func() error) io.ReadCloser {
return &readFnCloser{
Reader: r,
close: close,
}
}
type readFnCloser struct {
io.Reader
close func() error
}
func (r *readFnCloser) Close() error {
return r.close()
}
// WriteFnCloser takes an io.Writer and wraps it to use the provided function to implement io.Closer.
func WriteFnCloser(w io.Writer, close func() error) io.WriteCloser {
return &writeFnCloser{
Writer: w,
close: close,
}
}
type writeFnCloser struct {
io.Writer
close func() error
}
func (r *writeFnCloser) Close() error {
return r.close()
}
// SilentReader wraps an io.Reader to silence any
// error output during reads. Instead they are stored
// and accessible (not concurrency safe!) via .Error().
type SilentReader struct {
io.Reader
err error
}
// SilenceReader wraps an io.Reader within SilentReader{}.
func SilenceReader(r io.Reader) *SilentReader {
return &SilentReader{Reader: r}
}
func (r *SilentReader) Read(b []byte) (int, error) {
n, err := r.Reader.Read(b)
if err != nil {
// Store error for now
if r.err == nil {
r.err = err
}
// Pretend we're happy
// to continue reading.
n = len(b)
}
return n, nil
}
func (r *SilentReader) Error() error {
return r.err
}
// SilentWriter wraps an io.Writer to silence any
// error output during writes. Instead they are stored
// and accessible (not concurrency safe!) via .Error().
type SilentWriter struct {
io.Writer
err error
}
// SilenceWriter wraps an io.Writer within SilentWriter{}.
func SilenceWriter(w io.Writer) *SilentWriter {
return &SilentWriter{Writer: w}
}
func (w *SilentWriter) Write(b []byte) (int, error) {
n, err := w.Writer.Write(b)
if err != nil {
// Store error for now
if w.err == nil {
w.err = err
}
// Pretend we're happy
// to continue writing.
n = len(b)
}
return n, nil
}
func (w *SilentWriter) Error() error {
return w.err
}
func StreamReadFunc(read func(io.Reader) error) io.Writer {
// In-memory stream.
pr, pw := io.Pipe()
go func() {
var err error
defer func() {
// Always pass along error.
pr.CloseWithError(err)
}()
// Start reading.
err = read(pr)
}()
return pw
}
func StreamWriteFunc(write func(io.Writer) error) io.Reader {
// In-memory stream.
pr, pw := io.Pipe()
go func() {
var err error
defer func() {
// Always pass along error.
pw.CloseWithError(err)
}()
// Start writing.
err = write(pw)
}()
return pr
}