First pass at YP registration/configuration (#209)

* Spike: Ping YP service with instance details

* WIP: Add to the config to support YP

* Add YP response endpoint

* Handle YP errors. Use config. Off by default

* Show message about YP support on launch

* Add animated gif preview when generating thumb

* Increase quality of preview gif and only create it if YP is enabled

* Do not allow re-registration by clearing the key

* Make large and small logos actually structured

* Change log level

* Fix default YP service URL

* Point to default hostname

* Set default value for YP to false

.gitignore

@@ -19,9 +19,11 @@ vendor/
 /stats.json
 owncast
 webroot/thumbnail.jpg
+webroot/preview.gif
 webroot/hls
 webroot/static/content.md
 hls/
 dist/
 transcoder.log
 chat.db
+.yp.key

config-example.yaml

@@ -25,3 +25,9 @@ instanceDetails:
 videoSettings:
   # Change this value and keep it secure.  Treat it like a password to your live stream.
   streamingKey: abc123
+
+yp:
+  # Enable YP to be listed in the Owncast directory and let people discover your instance.
+  enabled: false
+  # You must specify the public URL of your host that you want the directory to link to.
+  instanceURL: http://yourhost.com

config/config.go

@@ -26,18 +26,25 @@ type config struct {
 	VersionInfo          string          `yaml:"-"`
 	VideoSettings        videoSettings   `yaml:"videoSettings"`
 	WebServerPort        int             `yaml:"webServerPort"`
+	YP                   yp              `yaml:"yp"`
 }
 
 // InstanceDetails defines the user-visible information about this particular instance.
 type InstanceDetails struct {
-	Name          string            `yaml:"name" json:"name"`
-	Title         string            `yaml:"title" json:"title"`
-	Summary       string            `yaml:"summary" json:"summary"`
-	Logo          map[string]string `yaml:"logo" json:"logo"`
-	Tags          []string          `yaml:"tags" json:"tags"`
-	SocialHandles []socialHandle    `yaml:"socialHandles" json:"socialHandles"`
-	ExtraInfoFile string            `yaml:"extraUserInfoFileName" json:"extraUserInfoFileName"`
-	Version       string            `json:"version"`
+	Name          string         `yaml:"name" json:"name"`
+	Title         string         `yaml:"title" json:"title"`
+	Summary       string         `yaml:"summary" json:"summary"`
+	Logo          logo           `yaml:"logo" json:"logo"`
+	Tags          []string       `yaml:"tags" json:"tags"`
+	SocialHandles []socialHandle `yaml:"socialHandles" json:"socialHandles"`
+	ExtraInfoFile string         `yaml:"extraUserInfoFileName" json:"extraUserInfoFileName"`
+	Version       string         `json:"version"`
+	NSFW          bool           `yaml:"nsfw" json:"nsfw"`
+}
+
+type logo struct {
+	Large string `yaml:"large" json:"large"`
+	Small string `yaml:"small" json:"small"`
 }
 
 type socialHandle struct {
@@ -50,7 +57,14 @@ type videoSettings struct {
 	StreamingKey              string          `yaml:"streamingKey"`
 	StreamQualities           []StreamQuality `yaml:"streamQualities"`
 	OfflineContent            string          `yaml:"offlineContent"`
-	HighestQualityStreamIndex int             `yaml"-"`
+	HighestQualityStreamIndex int             `yaml:"-"`
+}
+
+// Registration to the central Owncast YP (Yellow pages) service operating as a directory.
+type yp struct {
+	Enabled      bool   `yaml:"enabled"`
+	InstanceURL  string `yaml:"instanceURL"`  // The public URL the directory should link to
+	YPServiceURL string `yaml:"ypServiceURL"` // The base URL to the YP API to register with (optional)
 }
 
 // StreamQuality defines the specifics of a single HLS stream variant.
@@ -129,6 +143,10 @@ func (c *config) verifySettings() error {
 		}
 	}
 
+	if c.YP.Enabled && c.YP.InstanceURL == "" {
+		return errors.New("YP is enabled but instance url is not set")
+	}
+
 	return nil
 }
 
@@ -189,6 +207,14 @@ func (c *config) GetFFMpegPath() string {
 	return _default.FFMpegPath
 }
 
+func (c *config) GetYPServiceHost() string {
+	if c.YP.YPServiceURL != "" {
+		return c.YP.YPServiceURL
+	}
+
+	return _default.YP.YPServiceURL
+}
+
 func (c *config) GetVideoStreamQualities() []StreamQuality {
 	if len(c.VideoSettings.StreamQualities) > 0 {
 		return c.VideoSettings.StreamQualities

config/defaults.go

@@ -16,6 +16,8 @@ func getDefaults() config {
 	defaults.PrivateHLSPath = "hls"
 	defaults.VideoSettings.OfflineContent = "static/offline.m4v"
 	defaults.InstanceDetails.ExtraInfoFile = "/static/content.md"
+	defaults.YP.Enabled = false
+	defaults.YP.YPServiceURL = "https://yp.owncast.online"
 
 	defaultQuality := StreamQuality{
 		IsAudioPassthrough: true,

controllers/index.go

@@ -69,7 +69,7 @@ func handleScraperMetadataPage(w http.ResponseWriter, r *http.Request) {
 	tmpl := template.Must(template.ParseFiles(path.Join("static", "metadata.html")))
 
 	fullURL, err := url.Parse(fmt.Sprintf("http://%s%s", r.Host, r.URL.Path))
-	imageURL, err := url.Parse(fmt.Sprintf("http://%s%s", r.Host, config.Config.InstanceDetails.Logo["large"]))
+	imageURL, err := url.Parse(fmt.Sprintf("http://%s%s", r.Host, config.Config.InstanceDetails.Logo.Large))
 
 	status := core.GetStatus()
 

core/chat/server.go

@@ -133,7 +133,7 @@ func (s *server) sendWelcomeMessageToClient(c *Client) {
 		time.Sleep(7 * time.Second)
 
 		initialChatMessageText := fmt.Sprintf("Welcome to %s! %s", config.Config.InstanceDetails.Title, config.Config.InstanceDetails.Summary)
-		initialMessage := models.ChatMessage{"owncast-server", config.Config.InstanceDetails.Name, initialChatMessageText, config.Config.InstanceDetails.Logo["small"], "initial-message-1", "CHAT", true, time.Now()}
+		initialMessage := models.ChatMessage{"owncast-server", config.Config.InstanceDetails.Name, initialChatMessageText, config.Config.InstanceDetails.Logo.Small, "initial-message-1", "CHAT", true, time.Now()}
 		c.Write(initialMessage)
 	}()
 

core/core.go

@@ -13,12 +13,14 @@ import (
 	"github.com/gabek/owncast/core/ffmpeg"
 	"github.com/gabek/owncast/models"
 	"github.com/gabek/owncast/utils"
+	"github.com/gabek/owncast/yp"
 )
 
 var (
 	_stats        *models.Stats
 	_storage      models.ChunkStorageProvider
 	_cleanupTimer *time.Timer
+	_yp           *yp.YP
 )
 
 //Start starts up the core processing
@@ -40,6 +42,12 @@ func Start() error {
 		return err
 	}
 
+	if config.Config.YP.Enabled {
+		_yp = yp.NewYP(GetStatus)
+	} else {
+		yp.DisplayInstructions()
+	}
+
 	chat.Setup(ChatListenerImpl{})
 
 	return nil

core/ffmpeg/thumbnailGenerator.go

@@ -40,6 +40,7 @@ func StartThumbnailGenerator(chunkPath string, variantIndex int) {
 func fireThumbnailGenerator(chunkPath string, variantIndex int) error {
 	// JPG takes less time to encode than PNG
 	outputFile := path.Join("webroot", "thumbnail.jpg")
+	previewGifFile := path.Join("webroot", "preview.gif")
 
 	framePath := path.Join(chunkPath, strconv.Itoa(variantIndex))
 	files, err := ioutil.ReadDir(framePath)
@@ -83,12 +84,32 @@ func fireThumbnailGenerator(chunkPath string, variantIndex int) error {
 	}
 
 	ffmpegCmd := strings.Join(thumbnailCmdFlags, " ")
-
-	// fmt.Println(ffmpegCmd)
-
 	if _, err := exec.Command("sh", "-c", ffmpegCmd).Output(); err != nil {
 		return err
 	}
 
+	// If YP support is enabled also create an animated GIF preview
+	if config.Config.YP.Enabled {
+		makeAnimatedGifPreview(mostRecentFile, previewGifFile)
+	}
+
 	return nil
 }
+
+func makeAnimatedGifPreview(sourceFile string, outputFile string) {
+	// Filter is pulled from https://engineering.giphy.com/how-to-make-gifs-with-ffmpeg/
+	animatedGifFlags := []string{
+		config.Config.GetFFMpegPath(),
+		"-y",             // Overwrite file
+		"-threads 1",     // Low priority processing
+		"-i", sourceFile, // Input
+		"-t 1", // Output is one second in length
+		"-filter_complex", "\"[0:v] fps=8,scale=w=480:h=-1:flags=lanczos,split [a][b];[a] palettegen=stats_mode=full [p];[b][p] paletteuse=new=1\"",
+		outputFile,
+	}
+
+	ffmpegCmd := strings.Join(animatedGifFlags, " ")
+	if _, err := exec.Command("sh", "-c", ffmpegCmd).Output(); err != nil {
+		log.Errorln(err)
+	}
+}

core/status.go

@@ -38,6 +38,10 @@ func SetStreamAsConnected() {
 		chunkPath = config.Config.GetPrivateHLSSavePath()
 	}
 
+	if _yp != nil {
+		_yp.Start()
+	}
+
 	ffmpeg.StartThumbnailGenerator(chunkPath, config.Config.VideoSettings.HighestQualityStreamIndex)
 }
 
@@ -46,6 +50,10 @@ func SetStreamAsDisconnected() {
 	_stats.StreamConnected = false
 	_stats.LastDisconnectTime = utils.NullTime{time.Now(), true}
 
+	if _yp != nil {
+		_yp.Stop()
+	}
+
 	ffmpeg.ShowStreamOfflineState()
 	startCleanupTimer()
 }

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/mssola/user_agent v0.5.2
 	github.com/nareix/joy5 v0.0.0-20200712071056-a55089207c88
 	github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
+	github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1
 	github.com/radovskyb/watcher v1.0.7
 	github.com/shirou/gopsutil v2.20.7+incompatible
 	github.com/sirupsen/logrus v1.6.0

go.sum

@@ -31,6 +31,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1 h1:CskT+S6Ay54OwxBGB0R3Rsx4Muto6UnEYTyKJbyRIAI=
+github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o=
 github.com/radovskyb/watcher v1.0.7 h1:AYePLih6dpmS32vlHfhCeli8127LzkIgwJGcwwe8tUE=
 github.com/radovskyb/watcher v1.0.7/go.mod h1:78okwvY5wPdzcb1UYnip1pvrZNIVEIh/Cm+ZuvsUYIg=
 github.com/shirou/gopsutil v2.20.7+incompatible h1:Ymv4OD12d6zm+2yONe39VSmp2XooJe8za7ngOLW/o/w=

router/router.go

@@ -13,6 +13,7 @@ import (
 	"github.com/gabek/owncast/core/chat"
 	"github.com/gabek/owncast/core/rtmp"
 	"github.com/gabek/owncast/router/middleware"
+	"github.com/gabek/owncast/yp"
 )
 
 //Start starts the router for the http, ws, and rtmp
@@ -44,6 +45,8 @@ func Start() error {
 
 		// video embed
 		http.HandleFunc("/embed/video", controllers.GetVideoEmbed)
+
+		http.HandleFunc("/api/yp", yp.GetYPResponse)
 	}
 
 	// Authenticated admin requests

yp/api.go

@@ -0,0 +1,46 @@
+package yp
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/gabek/owncast/config"
+	"github.com/gabek/owncast/utils"
+)
+
+type ypDetailsResponse struct {
+	Name                  string   `json:"name"`
+	Description           string   `json:"description"`
+	Logo                  string   `json:"logo"`
+	NSFW                  bool     `json:"nsfw"`
+	Tags                  []string `json:"tags"`
+	Online                bool     `json:"online"`
+	ViewerCount           int      `json:"viewerCount"`
+	OverallMaxViewerCount int      `json:"overallMaxViewerCount"`
+	SessionMaxViewerCount int      `json:"sessionMaxViewerCount"`
+
+	LastConnectTime utils.NullTime `json:"lastConnectTime"`
+}
+
+//GetYPResponse gets the status of the server for YP purposes
+func GetYPResponse(w http.ResponseWriter, r *http.Request) {
+	status := getStatus()
+
+	response := ypDetailsResponse{
+		Name:                  config.Config.InstanceDetails.Name,
+		Description:           config.Config.InstanceDetails.Summary,
+		Logo:                  config.Config.InstanceDetails.Logo.Large,
+		NSFW:                  config.Config.InstanceDetails.NSFW,
+		Tags:                  config.Config.InstanceDetails.Tags,
+		Online:                status.Online,
+		ViewerCount:           status.ViewerCount,
+		OverallMaxViewerCount: status.OverallMaxViewerCount,
+		SessionMaxViewerCount: status.SessionMaxViewerCount,
+		LastConnectTime:       status.LastConnectTime,
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+
+	json.NewEncoder(w).Encode(response)
+
+}

yp/yp.go

@@ -0,0 +1,139 @@
+package yp
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"time"
+
+	"encoding/json"
+
+	"github.com/gabek/owncast/config"
+	"github.com/gabek/owncast/models"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const pingInterval = 4 * time.Minute
+
+var getStatus func() models.Status
+
+//YP is a service for handling listing in the Owncast directory.
+type YP struct {
+	timer *time.Ticker
+}
+
+type ypPingResponse struct {
+	Key       string `json:"key"`
+	Success   bool   `json:"success"`
+	Error     string `json:"error"`
+	ErrorCode int    `json:"errorCode"`
+}
+
+type ypPingRequest struct {
+	Key string `json:"key"`
+	URL string `json:"url"`
+}
+
+// NewYP creates a new instance of the YP service handler
+func NewYP(getStatusFunc func() models.Status) *YP {
+	getStatus = getStatusFunc
+	return &YP{}
+}
+
+// Start is run when a live stream begins to start pinging YP
+func (yp *YP) Start() {
+	yp.timer = time.NewTicker(pingInterval)
+
+	go func() {
+		for {
+			select {
+			case <-yp.timer.C:
+				yp.ping()
+			}
+		}
+	}()
+
+	yp.ping()
+}
+
+// Stop stops the pinging of YP
+func (yp *YP) Stop() {
+	yp.timer.Stop()
+}
+
+func (yp *YP) ping() {
+	myInstanceURL := config.Config.YP.InstanceURL
+	key := yp.getSavedKey()
+
+	log.Traceln("Pinging YP as: ", config.Config.InstanceDetails.Name)
+
+	request := ypPingRequest{
+		Key: key,
+		URL: myInstanceURL,
+	}
+
+	req, err := json.Marshal(request)
+	if err != nil {
+		log.Errorln(err)
+		return
+	}
+
+	pingURL := config.Config.GetYPServiceHost() + "/ping"
+	resp, err := http.Post(pingURL, "application/json", bytes.NewBuffer(req))
+	if err != nil {
+		log.Errorln(err)
+		return
+	}
+
+	defer resp.Body.Close()
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		log.Errorln(err)
+	}
+
+	pingResponse := ypPingResponse{}
+	json.Unmarshal(body, &pingResponse)
+
+	if !pingResponse.Success {
+		log.Debugln("YP Ping error returned from service:", pingResponse.Error)
+		return
+	}
+
+	if pingResponse.Key != key {
+		yp.writeSavedKey(pingResponse.Key)
+	}
+}
+
+func (yp *YP) writeSavedKey(key string) {
+	f, err := os.Create(".yp.key")
+	defer f.Close()
+
+	if err != nil {
+		log.Errorln(err)
+		return
+	}
+
+	_, err = f.WriteString(key)
+	if err != nil {
+		log.Errorln(err)
+		return
+	}
+}
+
+func (yp *YP) getSavedKey() string {
+	fileBytes, err := ioutil.ReadFile(".yp.key")
+	if err != nil {
+		return ""
+	}
+
+	return string(fileBytes)
+}
+
+// DisplayInstructions will let the user know they are not in the directory by default and
+// how they can enable the feature.
+func DisplayInstructions() {
+	text := "Your instance can be listed on the Owncast directory at http://something.something by enabling YP in your config.  Learn more at http://something.something."
+	log.Debugln(text)
+}