feat(storage): support a object storage custom path prefix

This commit is contained in:
Gabe Kangas 2023-08-02 13:35:47 -07:00
parent d5c54aacc1
commit 1a7b6b99d5
No known key found for this signature in database
GPG key ID: 4345B2060657F330
7 changed files with 44 additions and 4 deletions

View file

@ -13,7 +13,7 @@ import (
) )
// rewriteRemotePlaylist will take a local playlist and rewrite it to have absolute URLs to remote locations. // rewriteRemotePlaylist will take a local playlist and rewrite it to have absolute URLs to remote locations.
func rewriteRemotePlaylist(localFilePath, remoteServingEndpoint string) error { func rewriteRemotePlaylist(localFilePath, remoteServingEndpoint, pathPrefix string) error {
f, err := os.Open(localFilePath) // nolint f, err := os.Open(localFilePath) // nolint
if err != nil { if err != nil {
log.Fatalln(err) log.Fatalln(err)
@ -25,7 +25,14 @@ func rewriteRemotePlaylist(localFilePath, remoteServingEndpoint string) error {
} }
for _, item := range p.Variants { for _, item := range p.Variants {
item.URI = remoteServingEndpoint + filepath.Join("/hls", item.URI) // Determine the final path to this playlist.
var finalPath string
if pathPrefix != "" {
finalPath = filepath.Join(pathPrefix, "/hls")
} else {
finalPath = "/hls"
}
item.URI = remoteServingEndpoint + filepath.Join(finalPath, item.URI)
} }
publicPath := filepath.Join(config.HLSStoragePath, filepath.Base(localFilePath)) publicPath := filepath.Join(config.HLSStoragePath, filepath.Base(localFilePath))

View file

@ -37,6 +37,7 @@ type S3Storage struct {
s3AccessKey string s3AccessKey string
s3Secret string s3Secret string
s3ACL string s3ACL string
s3PathPrefix string
s3ForcePathStyle bool s3ForcePathStyle bool
// If we try to upload a playlist but it is not yet on disk // If we try to upload a playlist but it is not yet on disk
@ -73,6 +74,7 @@ func (s *S3Storage) Setup() error {
s.s3AccessKey = s3Config.AccessKey s.s3AccessKey = s3Config.AccessKey
s.s3Secret = s3Config.Secret s.s3Secret = s3Config.Secret
s.s3ACL = s3Config.ACL s.s3ACL = s3Config.ACL
s.s3PathPrefix = s3Config.PathPrefix
s.s3ForcePathStyle = s3Config.ForcePathStyle s.s3ForcePathStyle = s3Config.ForcePathStyle
s.sess = s.connectAWS() s.sess = s.connectAWS()
@ -107,6 +109,7 @@ func (s *S3Storage) SegmentWritten(localFilePath string) {
// so the segments and the HLS playlist referencing // so the segments and the HLS playlist referencing
// them are in sync. // them are in sync.
playlistPath := filepath.Join(filepath.Dir(localFilePath), "stream.m3u8") playlistPath := filepath.Join(filepath.Dir(localFilePath), "stream.m3u8")
if _, err := s.Save(playlistPath, 0); err != nil { if _, err := s.Save(playlistPath, 0); err != nil {
s.queuedPlaylistUpdates[playlistPath] = playlistPath s.queuedPlaylistUpdates[playlistPath] = playlistPath
if pErr, ok := err.(*os.PathError); ok { if pErr, ok := err.(*os.PathError); ok {
@ -133,7 +136,7 @@ func (s *S3Storage) VariantPlaylistWritten(localFilePath string) {
// MasterPlaylistWritten is called when the master hls playlist is written. // MasterPlaylistWritten is called when the master hls playlist is written.
func (s *S3Storage) MasterPlaylistWritten(localFilePath string) { func (s *S3Storage) MasterPlaylistWritten(localFilePath string) {
// Rewrite the playlist to use absolute remote S3 URLs // Rewrite the playlist to use absolute remote S3 URLs
if err := rewriteRemotePlaylist(localFilePath, s.host); err != nil { if err := rewriteRemotePlaylist(localFilePath, s.host, s.s3PathPrefix); err != nil {
log.Warnln(err) log.Warnln(err)
} }
} }
@ -151,6 +154,12 @@ func (s *S3Storage) Save(filePath string, retryCount int) (string, error) {
// Build the remote path by adding the "hls" path prefix. // Build the remote path by adding the "hls" path prefix.
remotePath := strings.Join([]string{"hls", normalizedPath}, "") remotePath := strings.Join([]string{"hls", normalizedPath}, "")
// If a custom path prefix is set prepend it.
if s.s3PathPrefix != "" {
prefix := strings.TrimPrefix(s.s3PathPrefix, "/")
remotePath = strings.Join([]string{prefix, remotePath}, "/")
}
maxAgeSeconds := utils.GetCacheDurationSecondsForPath(filePath) maxAgeSeconds := utils.GetCacheDurationSecondsForPath(filePath)
cacheControlHeader := fmt.Sprintf("max-age=%d", maxAgeSeconds) cacheControlHeader := fmt.Sprintf("max-age=%d", maxAgeSeconds)

View file

@ -11,6 +11,9 @@ type S3 struct {
ACL string `json:"acl,omitempty"` ACL string `json:"acl,omitempty"`
ForcePathStyle bool `json:"forcePathStyle"` ForcePathStyle bool `json:"forcePathStyle"`
// PathPrefix is an optional prefix for object storage.
PathPrefix string `json:"pathPrefix,omitempty"`
// This property is no longer used as of v0.1.1. See the standalone // This property is no longer used as of v0.1.1. See the standalone
// property that was pulled out of here instead. It's only left here // property that was pulled out of here instead. It's only left here
// to allow the migration to take place without data loss. // to allow the migration to take place without data loss.

View file

@ -28,7 +28,8 @@ const { Panel } = Collapse;
// we could probably add more detailed checks here // we could probably add more detailed checks here
// `currentValues` is what's currently in the global store and in the db // `currentValues` is what's currently in the global store and in the db
function checkSaveable(formValues: any, currentValues: any) { function checkSaveable(formValues: any, currentValues: any) {
const { endpoint, accessKey, secret, bucket, region, enabled, acl, forcePathStyle } = formValues; const { endpoint, accessKey, secret, bucket, region, enabled, acl, forcePathStyle, pathPrefix } =
formValues;
// if fields are filled out and different from what's in store, then return true // if fields are filled out and different from what's in store, then return true
if (enabled) { if (enabled) {
if (!!endpoint && isValidUrl(endpoint) && !!accessKey && !!secret && !!bucket && !!region) { if (!!endpoint && isValidUrl(endpoint) && !!accessKey && !!secret && !!bucket && !!region) {
@ -39,6 +40,7 @@ function checkSaveable(formValues: any, currentValues: any) {
secret !== currentValues.secret || secret !== currentValues.secret ||
bucket !== currentValues.bucket || bucket !== currentValues.bucket ||
region !== currentValues.region || region !== currentValues.region ||
pathPrefix !== currentValues.pathPrefix ||
(!currentValues.acl && acl !== '') || (!currentValues.acl && acl !== '') ||
(!!currentValues.acl && acl !== currentValues.acl) || (!!currentValues.acl && acl !== currentValues.acl) ||
forcePathStyle !== currentValues.forcePathStyle forcePathStyle !== currentValues.forcePathStyle
@ -72,6 +74,7 @@ export default function EditStorage() {
endpoint = '', endpoint = '',
region = '', region = '',
secret = '', secret = '',
pathPrefix = '',
forcePathStyle = false, forcePathStyle = false,
} = s3; } = s3;
@ -84,6 +87,7 @@ export default function EditStorage() {
endpoint, endpoint,
region, region,
secret, secret,
pathPrefix,
forcePathStyle, forcePathStyle,
}); });
setShouldDisplayForm(enabled); setShouldDisplayForm(enabled);
@ -219,6 +223,14 @@ export default function EditStorage() {
/> />
</div> </div>
<div className="field-container">
<TextField
{...S3_TEXT_FIELDS_INFO.pathPrefix}
value={formDataValues.pathPrefix}
onChange={handleFieldChange}
/>
</div>
<div className="enable-switch"> <div className="enable-switch">
<ToggleSwitch <ToggleSwitch
{...S3_TEXT_FIELDS_INFO.forcePathStyle} {...S3_TEXT_FIELDS_INFO.forcePathStyle}

View file

@ -81,6 +81,7 @@ export interface S3Field {
endpoint: string; endpoint: string;
region: string; region: string;
secret: string; secret: string;
pathPrefix: string;
forcePathStyle: boolean; forcePathStyle: boolean;
} }

View file

@ -535,6 +535,13 @@ export const S3_TEXT_FIELDS_INFO = {
placeholder: 'your secret key', placeholder: 'your secret key',
tip: '', tip: '',
}, },
pathPrefix: {
fieldName: 'pathPrefix',
label: 'Path prefix',
maxLength: 255,
placeholder: '/my/custom/path',
tip: 'Optionally prepend a custom path for the final URL',
},
forcePathStyle: { forcePathStyle: {
fieldName: 'forcePathStyle', fieldName: 'forcePathStyle',
label: 'Force path-style', label: 'Force path-style',

View file

@ -39,6 +39,7 @@ const initialServerConfigState: ConfigDetails = {
endpoint: '', endpoint: '',
region: '', region: '',
secret: '', secret: '',
pathPrefix: '',
forcePathStyle: false, forcePathStyle: false,
}, },
yp: { yp: {