Support using the custom video serving endpoint even if you don't use object storage (#2924)

* feat(video): refactor video serving endpoint

It can now be used without an object storage provider. Closes #2785

* fix: remove debug log

controllers/admin/config.go

@@ -771,6 +771,28 @@ func SetDisableSearchIndexing(w http.ResponseWriter, r *http.Request) {
 	controllers.WriteSimpleResponse(w, true, "search indexing support updated")
 }
 
+// SetVideoServingEndpoint will save the video serving endpoint.
+func SetVideoServingEndpoint(w http.ResponseWriter, r *http.Request) {
+	endpoint, success := getValueFromRequest(w, r)
+	if !success {
+		controllers.WriteSimpleResponse(w, false, "unable to update custom video serving endpoint")
+		return
+	}
+
+	value, ok := endpoint.Value.(string)
+	if !ok {
+		controllers.WriteSimpleResponse(w, false, "unable to update custom video serving endpoint")
+		return
+	}
+
+	if err := data.SetVideoServingEndpoint(value); err != nil {
+		controllers.WriteSimpleResponse(w, false, err.Error())
+		return
+	}
+
+	controllers.WriteSimpleResponse(w, true, "custom video serving endpoint updated")
+}
+
 func requirePOST(w http.ResponseWriter, r *http.Request) bool {
 	if r.Method != controllers.POST {
 		controllers.WriteSimpleResponse(w, false, r.Method+" not supported")

controllers/admin/serverConfig.go

@@ -59,6 +59,7 @@ func GetServerConfig(w http.ResponseWriter, r *http.Request) {
 		ChatDisabled:            data.GetChatDisabled(),
 		ChatJoinMessagesEnabled: data.GetChatJoinMessagesEnabled(),
 		SocketHostOverride:      data.GetWebsocketOverrideHost(),
+		VideoServingEndpoint:    data.GetVideoServingEndpoint(),
 		ChatEstablishedUserMode: data.GetChatEstbalishedUsersOnlyMode(),
 		HideViewerCount:         data.GetHideViewerCount(),
 		DisableSearchIndexing:   data.GetDisableSearchIndexing(),
@@ -107,6 +108,7 @@ type serverConfigAdminResponse struct {
 	SocketHostOverride      string                      `json:"socketHostOverride,omitempty"`
 	WebServerIP             string                      `json:"webServerIP"`
 	VideoCodec              string                      `json:"videoCodec"`
+	VideoServingEndpoint    string                      `json:"videoServingEndpoint"`
 	S3                      models.S3                   `json:"s3"`
 	Federation              federationConfigResponse    `json:"federation"`
 	SupportedCodecs         []string                    `json:"supportedCodecs"`
@@ -120,9 +122,9 @@ type serverConfigAdminResponse struct {
 	ChatDisabled            bool                        `json:"chatDisabled"`
 	ChatJoinMessagesEnabled bool                        `json:"chatJoinMessagesEnabled"`
 	ChatEstablishedUserMode bool                        `json:"chatEstablishedUserMode"`
+	DisableSearchIndexing   bool                        `json:"disableSearchIndexing"`
 	StreamKeyOverridden     bool                        `json:"streamKeyOverridden"`
 	HideViewerCount         bool                        `json:"hideViewerCount"`
-	DisableSearchIndexing   bool                        `json:"disableSearchIndexing"`
 }
 
 type videoSettings struct {

core/data/config.go

@@ -70,6 +70,7 @@ const (
 	customColorVariableValuesKey         = "custom_color_variable_values"
 	streamKeysKey                        = "stream_keys"
 	disableSearchIndexingKey             = "disable_search_indexing"
+	videoServingEndpointKey              = "video_serving_endpoint"
 )
 
 // GetExtraPageBodyContent will return the user-supplied body content.
@@ -974,3 +975,14 @@ func GetDisableSearchIndexing() bool {
 	}
 	return disableSearchIndexing
 }
+
+// GetVideoServingEndpoint returns the custom video endpont.
+func GetVideoServingEndpoint() string {
+	message, _ := _datastore.GetString(videoServingEndpointKey)
+	return message
+}
+
+// SetVideoServingEndpoint sets the custom video endpoint.
+func SetVideoServingEndpoint(message string) error {
+	return _datastore.SetString(videoServingEndpointKey, message)
+}

core/data/datastoreMigrations.go

@@ -8,7 +8,7 @@ import (
 )
 
 const (
-	datastoreValuesVersion   = 2
+	datastoreValuesVersion   = 3
 	datastoreValueVersionKey = "DATA_STORE_VERSION"
 )
 
@@ -25,6 +25,8 @@ func migrateDatastoreValues(datastore *Datastore) {
 			migrateToDatastoreValues1(datastore)
 		case 1:
 			migrateToDatastoreValues2(datastore)
+		case 2:
+			migrateToDatastoreValues3ServingEndpoint3(datastore)
 		default:
 			log.Fatalln("missing datastore values migration step")
 		}
@@ -61,3 +63,13 @@ func migrateToDatastoreValues2(datastore *Datastore) {
 		{Key: oldAdminPassword, Comment: "Default stream key"},
 	})
 }
+
+func migrateToDatastoreValues3ServingEndpoint3(_ *Datastore) {
+	s3Config := GetS3Config()
+
+	if !s3Config.Enabled {
+		return
+	}
+
+	_ = SetVideoServingEndpoint(s3Config.ServingEndpoint)
+}

core/storageproviders/local.go

@@ -6,13 +6,15 @@ import (
 	log "github.com/sirupsen/logrus"
 
 	"github.com/owncast/owncast/config"
+	"github.com/owncast/owncast/core/data"
 	"github.com/owncast/owncast/core/transcoder"
 )
 
 // LocalStorage represents an instance of the local storage provider for HLS video.
 type LocalStorage struct {
 	// Cleanup old public HLS content every N min from the webroot.
-	onlineCleanupTicker *time.Ticker
+	onlineCleanupTicker        *time.Ticker
+	customVideoServingEndpoint string
 }
 
 // NewLocalStorage returns a new LocalStorage instance.
@@ -22,6 +24,10 @@ func NewLocalStorage() *LocalStorage {
 
 // Setup configures this storage provider.
 func (s *LocalStorage) Setup() error {
+	if data.GetVideoServingEndpoint() != "" {
+		s.customVideoServingEndpoint = data.GetVideoServingEndpoint()
+	}
+
 	// NOTE: This cleanup timer will have to be disabled to support recordings in the future
 	// as all HLS segments have to be publicly available on disk to keep a recording of them.
 	s.onlineCleanupTicker = time.NewTicker(1 * time.Minute)
@@ -50,7 +56,12 @@ func (s *LocalStorage) VariantPlaylistWritten(localFilePath string) {
 
 // MasterPlaylistWritten is called when the master hls playlist is written.
 func (s *LocalStorage) MasterPlaylistWritten(localFilePath string) {
-	if _, err := s.Save(localFilePath, 0); err != nil {
+	if s.customVideoServingEndpoint != "" {
+		// Rewrite the playlist to use custom absolute remote URLs
+		if err := rewriteRemotePlaylist(localFilePath, s.customVideoServingEndpoint); err != nil {
+			log.Warnln(err)
+		}
+	} else if _, err := s.Save(localFilePath, 0); err != nil {
 		log.Warnln(err)
 	}
 }

core/storageproviders/rewriteLocalPlaylist.go

@@ -0,0 +1,36 @@
+package storageproviders
+
+import (
+	"bufio"
+	"os"
+	"path/filepath"
+
+	"github.com/grafov/m3u8"
+	"github.com/owncast/owncast/config"
+	"github.com/owncast/owncast/core/playlist"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// rewriteRemotePlaylist will take a local playlist and rewrite it to have absolute URLs to remote locations.
+func rewriteRemotePlaylist(localFilePath, remoteServingEndpoint string) error {
+	f, err := os.Open(localFilePath) // nolint
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	p := m3u8.NewMasterPlaylist()
+	if err := p.DecodeFrom(bufio.NewReader(f), false); err != nil {
+		log.Warnln(err)
+	}
+
+	for _, item := range p.Variants {
+		item.URI = remoteServingEndpoint + filepath.Join("/hls", item.URI)
+	}
+
+	publicPath := filepath.Join(config.HLSStoragePath, filepath.Base(localFilePath))
+
+	newPlaylist := p.String()
+
+	return playlist.WritePlaylist(newPlaylist, publicPath)
+}

core/storageproviders/s3Storage.go

@@ -1,7 +1,6 @@
 package storageproviders
 
 import (
-	"bufio"
 	"fmt"
 	"net/http"
 	"os"
@@ -11,7 +10,6 @@ import (
 	"time"
 
 	"github.com/owncast/owncast/core/data"
-	"github.com/owncast/owncast/core/playlist"
 	"github.com/owncast/owncast/utils"
 	log "github.com/sirupsen/logrus"
 
@@ -21,8 +19,6 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 
 	"github.com/owncast/owncast/config"
-
-	"github.com/grafov/m3u8"
 )
 
 // S3Storage is the s3 implementation of a storage provider.
@@ -58,8 +54,9 @@ func (s *S3Storage) Setup() error {
 	log.Trace("Setting up S3 for external storage of video...")
 
 	s3Config := data.GetS3Config()
-	if s3Config.ServingEndpoint != "" {
-		s.host = s3Config.ServingEndpoint
+	customVideoServingEndpoint := data.GetVideoServingEndpoint()
+	if customVideoServingEndpoint != "" {
+		s.host = customVideoServingEndpoint
 	} else {
 		s.host = fmt.Sprintf("%s/%s", s3Config.Endpoint, s3Config.Bucket)
 	}
@@ -130,7 +127,7 @@ func (s *S3Storage) VariantPlaylistWritten(localFilePath string) {
 // MasterPlaylistWritten is called when the master hls playlist is written.
 func (s *S3Storage) MasterPlaylistWritten(localFilePath string) {
 	// Rewrite the playlist to use absolute remote S3 URLs
-	if err := s.rewriteRemotePlaylist(localFilePath); err != nil {
+	if err := rewriteRemotePlaylist(localFilePath, s.host); err != nil {
 		log.Warnln(err)
 	}
 }
@@ -216,26 +213,3 @@ func (s *S3Storage) connectAWS() *session.Session {
 	}
 	return sess
 }
-
-// rewriteRemotePlaylist will take a local playlist and rewrite it to have absolute URLs to remote locations.
-func (s *S3Storage) rewriteRemotePlaylist(filePath string) error {
-	f, err := os.Open(filePath) // nolint
-	if err != nil {
-		log.Fatalln(err)
-	}
-
-	p := m3u8.NewMasterPlaylist()
-	if err := p.DecodeFrom(bufio.NewReader(f), false); err != nil {
-		log.Warnln(err)
-	}
-
-	for _, item := range p.Variants {
-		item.URI = s.host + filepath.Join("/hls", item.URI)
-	}
-
-	publicPath := filepath.Join(config.HLSStoragePath, filepath.Base(filePath))
-
-	newPlaylist := p.String()
-
-	return playlist.WritePlaylist(newPlaylist, publicPath)
-}

models/s3Storage.go

@@ -2,13 +2,17 @@ package models
 
 // S3 is the storage configuration.
 type S3 struct {
-	Endpoint        string `json:"endpoint,omitempty"`
-	ServingEndpoint string `json:"servingEndpoint,omitempty"`
-	AccessKey       string `json:"accessKey,omitempty"`
-	Secret          string `json:"secret,omitempty"`
-	Bucket          string `json:"bucket,omitempty"`
-	Region          string `json:"region,omitempty"`
-	ACL             string `json:"acl,omitempty"`
-	Enabled         bool   `json:"enabled"`
-	ForcePathStyle  bool   `json:"forcePathStyle"`
+	Enabled        bool   `json:"enabled"`
+	Endpoint       string `json:"endpoint,omitempty"`
+	AccessKey      string `json:"accessKey,omitempty"`
+	Secret         string `json:"secret,omitempty"`
+	Bucket         string `json:"bucket,omitempty"`
+	Region         string `json:"region,omitempty"`
+	ACL            string `json:"acl,omitempty"`
+	ForcePathStyle bool   `json:"forcePathStyle"`
+
+	// This property is no longer used as of v0.1.1. See the standalone
+	// property that was pulled out of here instead. It's only left here
+	// to allow the migration to take place without data loss.
+	ServingEndpoint string `json:"-"`
 }

router/router.go

@@ -291,6 +291,9 @@ func Start() error {
 	// Websocket host override
 	http.HandleFunc("/api/admin/config/sockethostoverride", middleware.RequireAdminAuth(admin.SetSocketHostOverride))
 
+	// Custom video serving endpoint
+	http.HandleFunc("/api/admin/config/videoservingendpoint", middleware.RequireAdminAuth(admin.SetVideoServingEndpoint))
+
 	// Is server marked as NSFW
 	http.HandleFunc("/api/admin/config/nsfw", middleware.RequireAdminAuth(admin.SetNSFW))
 

web/components/admin/EditInstanceDetails2.tsx

@@ -10,6 +10,7 @@ import {
   TEXTFIELD_PROPS_SOCKET_HOST_OVERRIDE,
   TEXTFIELD_PROPS_ADMIN_PASSWORD,
   TEXTFIELD_PROPS_WEB_PORT,
+  TEXTFIELD_PROPS_VIDEO_SERVING_ENDPOINT,
 } from '../../utils/config-constants';
 import { UpdateArgs } from '../../types/config-section';
 import { ResetYP } from './ResetYP';
@@ -24,8 +25,15 @@ export default function EditInstanceDetails() {
 
   const { serverConfig } = serverStatusData || {};
 
-  const { adminPassword, ffmpegPath, rtmpServerPort, webServerPort, yp, socketHostOverride } =
-    serverConfig;
+  const {
+    adminPassword,
+    ffmpegPath,
+    rtmpServerPort,
+    webServerPort,
+    yp,
+    socketHostOverride,
+    videoServingEndpoint,
+  } = serverConfig;
 
   useEffect(() => {
     setFormDataValues({
@@ -34,6 +42,7 @@ export default function EditInstanceDetails() {
       rtmpServerPort,
       webServerPort,
       socketHostOverride,
+      videoServingEndpoint,
     });
   }, [serverConfig]);
 
@@ -119,6 +128,15 @@ export default function EditInstanceDetails() {
             type={TEXTFIELD_TYPE_URL}
             onChange={handleFieldChange}
           />
+
+          <TextFieldWithSubmit
+            fieldName="videoServingEndpoint"
+            {...TEXTFIELD_PROPS_VIDEO_SERVING_ENDPOINT}
+            value={formDataValues.videoServingEndpoint}
+            initialValue={videoServingEndpoint || ''}
+            type={TEXTFIELD_TYPE_URL}
+            onChange={handleFieldChange}
+          />
           {yp.enabled && <ResetYP />}
         </Panel>
       </Collapse>

web/components/admin/config/server/EditStorage.tsx

@@ -28,17 +28,7 @@ const { Panel } = Collapse;
 // we could probably add more detailed checks here
 // `currentValues` is what's currently in the global store and in the db
 function checkSaveable(formValues: any, currentValues: any) {
-  const {
-    endpoint,
-    accessKey,
-    secret,
-    bucket,
-    region,
-    enabled,
-    servingEndpoint,
-    acl,
-    forcePathStyle,
-  } = formValues;
+  const { endpoint, accessKey, secret, bucket, region, enabled, acl, forcePathStyle } = formValues;
   // if fields are filled out and different from what's in store, then return true
   if (enabled) {
     if (!!endpoint && isValidUrl(endpoint) && !!accessKey && !!secret && !!bucket && !!region) {
@@ -49,8 +39,6 @@ function checkSaveable(formValues: any, currentValues: any) {
         secret !== currentValues.secret ||
         bucket !== currentValues.bucket ||
         region !== currentValues.region ||
-        (!currentValues.servingEndpoint && servingEndpoint !== '') ||
-        (!!currentValues.servingEndpoint && servingEndpoint !== currentValues.servingEndpoint) ||
         (!currentValues.acl && acl !== '') ||
         (!!currentValues.acl && acl !== currentValues.acl) ||
         forcePathStyle !== currentValues.forcePathStyle
@@ -84,7 +72,6 @@ export default function EditStorage() {
     endpoint = '',
     region = '',
     secret = '',
-    servingEndpoint = '',
     forcePathStyle = false,
   } = s3;
 
@@ -97,7 +84,6 @@ export default function EditStorage() {
       endpoint,
       region,
       secret,
-      servingEndpoint,
       forcePathStyle,
     });
     setShouldDisplayForm(enabled);
@@ -232,13 +218,7 @@ export default function EditStorage() {
                 onChange={handleFieldChange}
               />
             </div>
-            <div className="field-container">
-              <TextField
-                {...S3_TEXT_FIELDS_INFO.servingEndpoint}
-                value={formDataValues.servingEndpoint}
-                onChange={handleFieldChange}
-              />
-            </div>
+
             <div className="enable-switch">
               <ToggleSwitch
                 {...S3_TEXT_FIELDS_INFO.forcePathStyle}

web/types/config-section.ts

@@ -81,7 +81,6 @@ export interface S3Field {
   endpoint: string;
   region: string;
   secret: string;
-  servingEndpoint?: string;
   forcePathStyle: boolean;
 }
 
@@ -145,6 +144,7 @@ export interface ConfigDetails {
   videoSettings: VideoSettingsFields;
   webServerPort: string;
   socketHostOverride: string;
+  videoServingEndpoint: string;
   yp: ConfigDirectoryFields;
   supportedCodecs: string[];
   videoCodec: string;

web/utils/config-constants.tsx

@@ -40,6 +40,7 @@ const API_CHAT_JOIN_MESSAGES_ENABLED = '/chat/joinmessagesenabled';
 const API_CHAT_ESTABLISHED_MODE = '/chat/establishedusermode';
 const API_DISABLE_SEARCH_INDEXING = '/disablesearchindexing';
 const API_SOCKET_HOST_OVERRIDE = '/sockethostoverride';
+const API_VIDEO_SERVING_ENDPOINT = '/videoservingendpoint';
 
 // Federation
 const API_FEDERATION_ENABLED = '/federation/enable';
@@ -180,6 +181,18 @@ export const TEXTFIELD_PROPS_SOCKET_HOST_OVERRIDE = {
   useTrim: true,
 };
 
+export const TEXTFIELD_PROPS_VIDEO_SERVING_ENDPOINT = {
+  apiPath: API_VIDEO_SERVING_ENDPOINT,
+  fieldName: 'videoServingEndpoint',
+  label: 'Serving Endpoint',
+  maxLength: 255,
+  placeholder: 'http://cdn.provider.endpoint.com',
+  tip: 'Optional URL that video content should be accessed from instead of the default.  Used with CDNs and specific storage providers. Generally not required.',
+  type: TEXTFIELD_TYPE_URL,
+  pattern: DEFAULT_TEXTFIELD_URL_PATTERN,
+  useTrim: true,
+};
+
 // MISC FIELDS
 export const FIELD_PROPS_TAGS = {
   apiPath: API_TAGS,
@@ -521,16 +534,6 @@ export const S3_TEXT_FIELDS_INFO = {
     placeholder: 'your secret key',
     tip: '',
   },
-  servingEndpoint: {
-    fieldName: 'servingEndpoint',
-    label: 'Serving Endpoint',
-    maxLength: 255,
-    placeholder: 'http://cdn.ss3.provider.endpoint.com',
-    tip: 'Optional URL that content should be accessed from instead of the default.  Used with CDNs and specific storage providers. Generally not required.',
-    type: TEXTFIELD_TYPE_URL,
-    pattern: DEFAULT_TEXTFIELD_URL_PATTERN,
-    useTrim: true,
-  },
   forcePathStyle: {
     fieldName: 'forcePathStyle',
     label: 'Force path-style',

web/utils/server-status-context.tsx

@@ -30,6 +30,7 @@ const initialServerConfigState: ConfigDetails = {
   rtmpServerPort: '',
   webServerPort: '',
   socketHostOverride: null,
+  videoServingEndpoint: '',
   s3: {
     accessKey: '',
     acl: '',
@@ -38,7 +39,6 @@ const initialServerConfigState: ConfigDetails = {
     endpoint: '',
     region: '',
     secret: '',
-    servingEndpoint: '',
     forcePathStyle: false,
   },
   yp: {