feat: add support for robots.txt disabling search indexing (#2929)

* feat: add support for robots.txt

Can toggle disabling search engine indexing. Closes #2684

* fix: unexport ts const

controllers/admin/config.go

@@ -751,6 +751,26 @@ func SetHideViewerCount(w http.ResponseWriter, r *http.Request) {
 	controllers.WriteSimpleResponse(w, true, "hide viewer count setting updated")
 }
 
+// SetDisableSearchIndexing will set search indexing support.
+func SetDisableSearchIndexing(w http.ResponseWriter, r *http.Request) {
+	if !requirePOST(w, r) {
+		return
+	}
+
+	configValue, success := getValueFromRequest(w, r)
+	if !success {
+		controllers.WriteSimpleResponse(w, false, "unable to update search indexing")
+		return
+	}
+
+	if err := data.SetDisableSearchIndexing(configValue.Value.(bool)); err != nil {
+		controllers.WriteSimpleResponse(w, false, err.Error())
+		return
+	}
+
+	controllers.WriteSimpleResponse(w, true, "search indexing support updated")
+}
+
 func requirePOST(w http.ResponseWriter, r *http.Request) bool {
 	if r.Method != controllers.POST {
 		controllers.WriteSimpleResponse(w, false, r.Method+" not supported")

controllers/admin/serverConfig.go

@@ -61,6 +61,7 @@ func GetServerConfig(w http.ResponseWriter, r *http.Request) {
 		SocketHostOverride:      data.GetWebsocketOverrideHost(),
 		ChatEstablishedUserMode: data.GetChatEstbalishedUsersOnlyMode(),
 		HideViewerCount:         data.GetHideViewerCount(),
+		DisableSearchIndexing:   data.GetDisableSearchIndexing(),
 		VideoSettings: videoSettings{
 			VideoQualityVariants: videoQualityVariants,
 			LatencyLevel:         data.GetStreamLatencyLevel().Level,
@@ -121,6 +122,7 @@ type serverConfigAdminResponse struct {
 	ChatEstablishedUserMode bool                        `json:"chatEstablishedUserMode"`
 	StreamKeyOverridden     bool                        `json:"streamKeyOverridden"`
 	HideViewerCount         bool                        `json:"hideViewerCount"`
+	DisableSearchIndexing   bool                        `json:"disableSearchIndexing"`
 }
 
 type videoSettings struct {

controllers/robots.go

@@ -0,0 +1,28 @@
+package controllers
+
+import (
+	"net/http"
+	"strings"
+
+	"github.com/owncast/owncast/core/data"
+)
+
+// GetRobotsDotTxt returns the contents of our robots.txt.
+func GetRobotsDotTxt(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "text/plain")
+	contents := []string{
+		"User-agent: *",
+		"Disallow: /admin",
+		"Disallow: /api",
+	}
+
+	if data.GetDisableSearchIndexing() {
+		contents = append(contents, "Disallow: /")
+	}
+
+	txt := []byte(strings.Join(contents, "\n"))
+
+	if _, err := w.Write(txt); err != nil {
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+	}
+}

core/data/config.go

@@ -69,6 +69,7 @@ const (
 	customOfflineMessageKey              = "custom_offline_message"
 	customColorVariableValuesKey         = "custom_color_variable_values"
 	streamKeysKey                        = "stream_keys"
+	disableSearchIndexingKey             = "disable_search_indexing"
 )
 
 // GetExtraPageBodyContent will return the user-supplied body content.
@@ -959,3 +960,17 @@ func SetStreamKeys(actions []models.StreamKey) error {
 	configEntry := ConfigEntry{Key: streamKeysKey, Value: actions}
 	return _datastore.Save(configEntry)
 }
+
+// SetDisableSearchIndexing will set if the web server should be indexable.
+func SetDisableSearchIndexing(disableSearchIndexing bool) error {
+	return _datastore.SetBool(disableSearchIndexingKey, disableSearchIndexing)
+}
+
+// GetDisableSearchIndexing will return if the web server should be indexable.
+func GetDisableSearchIndexing() bool {
+	disableSearchIndexing, err := _datastore.GetBool(disableSearchIndexingKey)
+	if err != nil {
+		return false
+	}
+	return disableSearchIndexing
+}

router/router.go

@@ -50,6 +50,9 @@ func Start() error {
 	// return a logo that's compatible with external social networks
 	http.HandleFunc("/logo/external", controllers.GetCompatibleLogo)
 
+	// robots.txt
+	http.HandleFunc("/robots.txt", controllers.GetRobotsDotTxt)
+
 	// status of the system
 	http.HandleFunc("/api/status", controllers.GetStatus)
 
@@ -327,6 +330,9 @@ func Start() error {
 	// Is the viewer count hidden from viewers
 	http.HandleFunc("/api/admin/config/hideviewercount", middleware.RequireAdminAuth(admin.SetHideViewerCount))
 
+	// set disabling of search indexing
+	http.HandleFunc("/api/admin/config/disablesearchindexing", middleware.RequireAdminAuth(admin.SetDisableSearchIndexing))
+
 	// Inline chat moderation actions
 
 	// Update chat message visibility

test/automated/api/configmanagement.test.js

@@ -37,6 +37,8 @@ const defaultFederationConfig = {
 	blockedDomains: [],
 };
 const defaultHideViewerCount = false;
+const defaultDisableSearchIndexing = false;
+
 const defaultSocialHandles = [
 	{
 		icon: '/img/platformlogos/github.svg',
@@ -130,6 +132,7 @@ const newFederationConfig = {
 };
 
 const newHideViewerCount = !defaultHideViewerCount;
+const newDisableSearchIndexing = !defaultDisableSearchIndexing;
 
 const overriddenWebsocketHost = 'ws://lolcalhost.biz';
 const customCSS = randomString();
@@ -340,6 +343,14 @@ test('enable federation', async (done) => {
 	done();
 });
 
+test('disable search indexing', async (done) => {
+	await sendAdminRequest(
+		'config/disablesearchindexing',
+		newDisableSearchIndexing
+	);
+	done();
+});
+
 test('change admin password', async (done) => {
 	const res = await sendAdminRequest('config/adminpass', newAdminPassword);
 	done();
@@ -472,3 +483,18 @@ test('verify frontend status', (done) => {
 			done();
 		});
 });
+
+test('verify robots.txt is correct after disabling search indexing', (done) => {
+	const expected = `User-agent: *
+Disallow: /admin
+Disallow: /api
+Disallow: /`;
+
+	request
+		.get('/robots.txt')
+		.expect(200)
+		.then((res) => {
+			expect(res.text).toBe(expected);
+			done();
+		});
+});

web/components/admin/config/general/EditInstanceDetails.tsx

@@ -21,6 +21,7 @@ import {
   FIELD_PROPS_NSFW,
   FIELD_PROPS_HIDE_VIEWER_COUNT,
   API_SERVER_OFFLINE_MESSAGE,
+  FIELD_PROPS_DISABLE_SEARCH_INDEXING,
 } from '../../../../utils/config-constants';
 import { UpdateArgs } from '../../../../types/config-section';
 import { ToggleSwitch } from '../../ToggleSwitch';
@@ -36,7 +37,7 @@ export default function EditInstanceDetails() {
   const serverStatusData = useContext(ServerStatusContext);
   const { serverConfig } = serverStatusData || {};
 
-  const { instanceDetails, yp, hideViewerCount } = serverConfig;
+  const { instanceDetails, yp, hideViewerCount, disableSearchIndexing } = serverConfig;
   const { instanceUrl } = yp;
 
   const [offlineMessageSaveStatus, setOfflineMessageSaveStatus] = useState(null);
@@ -46,6 +47,7 @@ export default function EditInstanceDetails() {
       ...instanceDetails,
       ...yp,
       hideViewerCount,
+      disableSearchIndexing,
     });
   }, [instanceDetails, yp]);
 
@@ -87,6 +89,10 @@ export default function EditInstanceDetails() {
     handleFieldChange({ fieldName: 'hideViewerCount', value: enabled });
   }
 
+  function handleDisableSearchEngineIndexingChange(enabled: boolean) {
+    handleFieldChange({ fieldName: 'disableSearchIndexing', value: enabled });
+  }
+
   const hasInstanceUrl = instanceUrl !== '';
 
   return (
@@ -171,6 +177,14 @@ export default function EditInstanceDetails() {
         onChange={handleHideViewerCountChange}
       />
 
+      <ToggleSwitch
+        fieldName="disableSearchIndexing"
+        useSubmit
+        {...FIELD_PROPS_DISABLE_SEARCH_INDEXING}
+        checked={formDataValues.disableSearchIndexing}
+        onChange={handleDisableSearchEngineIndexingChange}
+      />
+
       <br />
       <p className="description">
         Increase your audience by appearing in the{' '}

web/types/config-section.ts

@@ -156,4 +156,5 @@ export interface ConfigDetails {
   chatJoinMessagesEnabled: boolean;
   chatEstablishedUserMode: boolean;
   hideViewerCount: boolean;
+  disableSearchIndexing: boolean;
 }

web/utils/config-constants.tsx

@@ -38,7 +38,7 @@ const API_HIDE_VIEWER_COUNT = '/hideviewercount';
 const API_CHAT_DISABLE = '/chat/disable';
 const API_CHAT_JOIN_MESSAGES_ENABLED = '/chat/joinmessagesenabled';
 const API_CHAT_ESTABLISHED_MODE = '/chat/establishedusermode';
-
+const API_DISABLE_SEARCH_INDEXING = '/disablesearchindexing';
 const API_SOCKET_HOST_OVERRIDE = '/sockethostoverride';
 
 // Federation
@@ -212,6 +212,13 @@ export const FIELD_PROPS_HIDE_VIEWER_COUNT = {
   tip: 'Turn this ON to hide the viewer count on the web page.',
 };
 
+export const FIELD_PROPS_DISABLE_SEARCH_INDEXING = {
+  apiPath: API_DISABLE_SEARCH_INDEXING,
+  configPath: '',
+  label: 'Disable search engine indexing',
+  tip: 'Turn this ON to to tell search engines not to index this site.',
+};
+
 export const DEFAULT_VARIANT_STATE: VideoVariant = {
   framerate: 24,
   videoPassthrough: false,

web/utils/server-status-context.tsx

@@ -71,6 +71,7 @@ const initialServerConfigState: ConfigDetails = {
   chatJoinMessagesEnabled: true,
   chatEstablishedUserMode: false,
   hideViewerCount: false,
+  disableSearchIndexing: false,
 };
 
 const initialServerStatusState = {