Handle pagination for the federated actions & followers responses (#1731)

* Add pagination for admin social list

* Use Paginated API for followers tab on frontend

activitypub/controllers/followers.go

@@ -98,7 +98,7 @@ func getFollowersPage(page string, r *http.Request) (vocab.ActivityStreamsOrdere
 		return nil, errors.Wrap(err, "unable to get follower count")
 	}
 
-	followers, err := persistence.GetFederationFollowers(followersPageSize, (pageInt-1)*followersPageSize)
+	followers, _, err := persistence.GetFederationFollowers(followersPageSize, (pageInt-1)*followersPageSize)
 	if err != nil {
 		return nil, errors.Wrap(err, "unable to get federation followers")
 	}

activitypub/outbox/outbox.go

@@ -171,7 +171,7 @@ func getHashtagLinkHTMLFromTagString(baseHashtag string) string {
 func SendToFollowers(payload []byte) error {
 	localActor := apmodels.MakeLocalIRIForAccount(data.GetDefaultFederationUsername())
 
-	followers, err := persistence.GetFederationFollowers(-1, 0)
+	followers, _, err := persistence.GetFederationFollowers(-1, 0)
 	if err != nil {
 		log.Errorln("unable to fetch followers to send to", err)
 		return errors.New("unable to fetch followers to send payload to")

activitypub/persistence/followers.go

@@ -6,6 +6,7 @@ import (
 	"github.com/owncast/owncast/db"
 	"github.com/owncast/owncast/models"
 	"github.com/owncast/owncast/utils"
+	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -44,14 +45,19 @@ func GetFollowerCount() (int64, error) {
 }
 
 // GetFederationFollowers will return a slice of the followers we keep track of locally.
-func GetFederationFollowers(limit int, offset int) ([]models.Follower, error) {
+func GetFederationFollowers(limit int, offset int) ([]models.Follower, int, error) {
 	ctx := context.Background()
+	total, err := _datastore.GetQueries().GetFollowerCount(ctx)
+	if err != nil {
+		return nil, 0, errors.Wrap(err, "unable to fetch total number of followers")
+	}
+
 	followersResult, err := _datastore.GetQueries().GetFederationFollowersWithOffset(ctx, db.GetFederationFollowersWithOffsetParams{
 		Limit:  int32(limit),
 		Offset: int32(offset),
 	})
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
 	followers := make([]models.Follower, 0)
@@ -69,7 +75,7 @@ func GetFederationFollowers(limit int, offset int) ([]models.Follower, error) {
 		followers = append(followers, singleFollower)
 	}
 
-	return followers, nil
+	return followers, int(total), nil
 }
 
 // GetPendingFollowRequests will return pending follow requests.

activitypub/persistence/persistence.go

@@ -319,18 +319,23 @@ func SaveInboundFediverseActivity(objectIRI string, actorIRI string, eventType s
 
 // GetInboundActivities will return a collection of saved, federated activities
 // limited and offset by the values provided to support pagination.
-func GetInboundActivities(limit int, offset int) ([]models.FederatedActivity, error) {
+func GetInboundActivities(limit int, offset int) ([]models.FederatedActivity, int, error) {
 	ctx := context.Background()
 	rows, err := _datastore.GetQueries().GetInboundActivitiesWithOffset(ctx, db.GetInboundActivitiesWithOffsetParams{
 		Limit:  int32(limit),
 		Offset: int32(offset),
 	})
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
 	activities := make([]models.FederatedActivity, 0)
 
+	total, err := _datastore.GetQueries().GetInboundActivityCount(context.Background())
+	if err != nil {
+		return nil, 0, errors.Wrap(err, "unable to fetch total activity count")
+	}
+
 	for _, row := range rows {
 		singleActivity := models.FederatedActivity{
 			IRI:       row.Iri,
@@ -341,7 +346,7 @@ func GetInboundActivities(limit int, offset int) ([]models.FederatedActivity, er
 		activities = append(activities, singleActivity)
 	}
 
-	return activities, nil
+	return activities, int(total), nil
 }
 
 // HasPreviouslyHandledInboundActivity will return if we have previously handled

controllers/admin/federation.go

@@ -160,12 +160,19 @@ func SetFederationBlockDomains(w http.ResponseWriter, r *http.Request) {
 
 // GetFederatedActions will return the saved list of accepted inbound
 // federated activities.
-func GetFederatedActions(w http.ResponseWriter, r *http.Request) {
+func GetFederatedActions(page int, pageSize int, w http.ResponseWriter, r *http.Request) {
-	activities, err := persistence.GetInboundActivities(100, 0)
+	offset := pageSize * page
+
+	activities, total, err := persistence.GetInboundActivities(pageSize, offset)
 	if err != nil {
 		controllers.WriteSimpleResponse(w, false, err.Error())
 		return
 	}
 
-	controllers.WriteResponse(w, activities)
+	response := controllers.PaginatedResponse{
+		Total:   total,
+		Results: activities,
+	}
+
+	controllers.WriteResponse(w, response)
 }

controllers/followers.go

@@ -7,12 +7,16 @@ import (
 )
 
 // GetFollowers will handle an API request to fetch the list of followers (non-activitypub response).
-func GetFollowers(w http.ResponseWriter, r *http.Request) {
+func GetFollowers(offset int, limit int, w http.ResponseWriter, r *http.Request) {
-	followers, err := persistence.GetFederationFollowers(-1, 0)
+	followers, total, err := persistence.GetFederationFollowers(limit, offset)
 	if err != nil {
 		WriteSimpleResponse(w, false, "unable to fetch followers")
 		return
 	}
 
-	WriteResponse(w, followers)
+	response := PaginatedResponse{
+		Total:   total,
+		Results: followers,
+	}
+	WriteResponse(w, response)
 }

controllers/pagination.go

@@ -0,0 +1,7 @@
+package controllers
+
+// PaginatedResponse is a structure for returning a total count with results.
+type PaginatedResponse struct {
+	Total   int         `json:"total"`
+	Results interface{} `json:"results"`
+}

db/query.sql

@@ -47,6 +47,9 @@ INSERT INTO ap_outbox(iri, value, type, live_notification) values($1, $2, $3, $4
 -- name: AddToAcceptedActivities :exec
 INSERT INTO ap_accepted_activities(iri, actor, type, timestamp) values($1, $2, $3, $4);
 
+-- name: GetInboundActivityCount :one
+SELECT count(*) FROM ap_accepted_activities;
+
 -- name: GetInboundActivitiesWithOffset :many
 SELECT iri, actor, type, timestamp FROM ap_accepted_activities ORDER BY timestamp DESC LIMIT $1 OFFSET $2;
 

db/query.sql.go

@@ -280,6 +280,17 @@ func (q *Queries) GetInboundActivitiesWithOffset(ctx context.Context, arg GetInb
 	return items, nil
 }
 
+const getInboundActivityCount = `-- name: GetInboundActivityCount :one
+SELECT count(*) FROM ap_accepted_activities
+`
+
+func (q *Queries) GetInboundActivityCount(ctx context.Context) (int64, error) {
+	row := q.db.QueryRowContext(ctx, getInboundActivityCount)
+	var count int64
+	err := row.Scan(&count)
+	return count, err
+}
+
 const getLocalPostCount = `-- name: GetLocalPostCount :one
 SElECT count(*) FROM ap_outbox
 `

router/middleware/pagination.go

@@ -0,0 +1,39 @@
+package middleware
+
+import (
+	"net/http"
+	"strconv"
+)
+
+// PaginatedHandlerFunc is a handler for endpoints that require pagination.
+type PaginatedHandlerFunc func(int, int, http.ResponseWriter, *http.Request)
+
+// HandlePagination is a middleware handler that pulls pagination values
+// and passes them along.
+func HandlePagination(handler PaginatedHandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		// Default 50 items per page
+		limitString := r.URL.Query().Get("limit")
+		if limitString == "" {
+			limitString = "50"
+		}
+		limit, err := strconv.Atoi(limitString)
+		if err != nil {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		// Default first page 0
+		offsetString := r.URL.Query().Get("offset")
+		if offsetString == "" {
+			offsetString = "0"
+		}
+		offset, err := strconv.Atoi(offsetString)
+		if err != nil {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+
+		handler(offset, limit, w, r)
+	}
+}

router/router.go

@@ -77,7 +77,7 @@ func Start() error {
 	http.HandleFunc("/api/remotefollow", controllers.RemoteFollow)
 
 	// return followers
-	http.HandleFunc("/api/followers", controllers.GetFollowers)
+	http.HandleFunc("/api/followers", middleware.HandlePagination(controllers.GetFollowers))
 
 	// Authenticated admin requests
 
@@ -127,7 +127,7 @@ func Start() error {
 	http.HandleFunc("/api/admin/chat/users/moderators", middleware.RequireAdminAuth(admin.GetModerators))
 
 	// return followers
-	http.HandleFunc("/api/admin/followers", middleware.RequireAdminAuth(controllers.GetFollowers))
+	http.HandleFunc("/api/admin/followers", middleware.RequireAdminAuth(middleware.HandlePagination(controllers.GetFollowers)))
 
 	// Get a list of pending follow requests
 	http.HandleFunc("/api/admin/followers/pending", middleware.RequireAdminAuth(admin.GetPendingFollowRequests))
@@ -310,7 +310,7 @@ func Start() error {
 	http.HandleFunc("/api/admin/federation/send", middleware.RequireAdminAuth(admin.SendFederatedMessage))
 
 	// Return federated activities
-	http.HandleFunc("/api/admin/federation/actions", middleware.RequireAdminAuth(admin.GetFederatedActions))
+	http.HandleFunc("/api/admin/federation/actions", middleware.RequireAdminAuth(middleware.HandlePagination(admin.GetFederatedActions)))
 
 	// ActivityPub has its own router
 	activitypub.Start(data.GetDatastore())

webroot/js/components/federation/followers.js

@@ -2,7 +2,6 @@ import { h, Component } from '/js/web_modules/preact.js';
 import htm from '/js/web_modules/htm.js';
 import { URL_FOLLOWERS } from '/js/utils/constants.js';
 const html = htm.bind(h);
-import { paginateArray } from '../../utils/helpers.js';
 export default class FollowerList extends Component {
   constructor(props) {
     super(props);
@@ -10,6 +9,8 @@ export default class FollowerList extends Component {
     this.state = {
       followers: [],
       followersPage: 0,
+      currentPage: 0,
+      total: 0,
     };
   }
 
@@ -22,23 +23,26 @@ export default class FollowerList extends Component {
   }
 
   async getFollowers() {
-    const response = await fetch(URL_FOLLOWERS);
+    const { currentPage } = this.state;
+    const limit = 16;
+    const offset = currentPage * limit;
+    const u = `${URL_FOLLOWERS}?offset=${offset}&limit=${limit}`;
+    const response = await fetch(u);
     const followers = await response.json();
 
     this.setState({
-      followers: followers,
+      followers: followers.results,
+      total: response.total,
     });
   }
 
   changeFollowersPage(page) {
-    this.setState({ followersPage: page });
+    this.setState({ currentPage: page });
+    this.getFollowers();
   }
 
   render() {
-    const FOLLOWER_PAGE_SIZE = 16;
+    const { followers, total, currentPage } = this.state;
-    const { followersPage } = this.state;
-
-    const { followers } = this.state;
     if (!followers) {
       return null;
     }
@@ -57,21 +61,15 @@ export default class FollowerList extends Component {
       </p>
     </div>`;
 
-    const paginatedFollowers = paginateArray(
-      followers,
-      followersPage + 1,
-      FOLLOWER_PAGE_SIZE
-    );
-
     const paginationControls =
-      paginatedFollowers.totalPages > 1 &&
+      total > 1 &&
-      Array(paginatedFollowers.totalPages)
+      Array(total)
         .fill()
         .map((x, n) => {
           const activePageClass =
-            n === followersPage &&
+            n === currentPage &&
             'bg-indigo-600 rounded-full shadow-md focus:shadow-md text-white';
-          return html` <li class="page-item active">
+          return html` <li class="page-item active w-10">
             <a
               class="page-link relative block cursor-pointer hover:no-underline py-1.5 px-3 border-0 rounded-full hover:text-gray-800 hover:bg-gray-200 outline-none transition-all duration-300 ${activePageClass}"
               onClick=${() => this.changeFollowersPage(n)}
@@ -85,13 +83,13 @@ export default class FollowerList extends Component {
       <div>
         <div class="flex flex-wrap">
           ${followers.length === 0 && noFollowersInfo}
-          ${paginatedFollowers.items.map((follower) => {
+          ${followers.map((follower) => {
             return html` <${SingleFollower} user=${follower} /> `;
           })}
         </div>
         <div class="flex">
-          <nav aria-label="Page navigation example">
+          <nav aria-label="Tab pages">
-            <ul class="flex list-style-none">
+            <ul class="flex list-style-none flex-wrap">
               ${paginationControls}
             </ul>
           </nav>