Messages table fixes to improve query performance (#2026)

* Move to yaml sqlc config

* Add util for ungraceful sql execs

* Fix messages schema + add indexes

* Add migration to drop+recreate messages table

* Create index only if does not exist

* Fix typo

* Unexport function

core/data/data.go

@@ -17,7 +17,7 @@ import (
 )
 
 const (
-	schemaVersion = 5
+	schemaVersion = 6
 )
 
 var (

core/data/messages.go

@@ -13,30 +13,26 @@ import (
 func CreateMessagesTable(db *sql.DB) {
 	createTableSQL := `CREATE TABLE IF NOT EXISTS messages (
 		"id" string NOT NULL,
-		"user_id" INTEGER,
+		"user_id" TEXT,
 		"body" TEXT,
 		"eventType" TEXT,
 		"hidden_at" DATETIME,
 		"timestamp" DATETIME,
-    "title" TEXT,
-    "subtitle" TEXT,
-    "image" TEXT,
-    "link" TEXT,
+		"title" TEXT,
+		"subtitle" TEXT,
+		"image" TEXT,
+		"link" TEXT,
 		PRIMARY KEY (id)
-	);CREATE INDEX index ON messages (id, user_id, hidden_at, timestamp);
-	CREATE INDEX id ON messages (id);
-	CREATE INDEX user_id ON messages (user_id);
-	CREATE INDEX hidden_at ON messages (hidden_at);
-	CREATE INDEX timestamp ON messages (timestamp);`
+	);`
+	mustExec(createTableSQL, db)
 
-	stmt, err := db.Prepare(createTableSQL)
-	if err != nil {
-		log.Fatal("error creating chat messages table", err)
-	}
-	defer stmt.Close()
-	if _, err := stmt.Exec(); err != nil {
-		log.Fatal("error creating chat messages table", err)
-	}
+	// Create indexes
+	mustExec(`CREATE INDEX IF NOT EXISTS user_id_hidden_at_timestamp ON messages (id, user_id, hidden_at, timestamp);`, db)
+	mustExec(`CREATE INDEX IF NOT EXISTS idx_id ON messages (id);`, db)
+	mustExec(`CREATE INDEX IF NOT EXISTS idx_user_id ON messages (user_id);`, db)
+	mustExec(`CREATE INDEX IF NOT EXISTS idx_hidden_at ON messages (hidden_at);`, db)
+	mustExec(`CREATE INDEX IF NOT EXISTS idx_timestamp ON messages (timestamp);`, db)
+	mustExec(`CREATE INDEX IF NOT EXISTS idx_messages_hidden_at_timestamp on messages(hidden_at, timestamp);`, db)
 }
 
 // GetMessagesCount will return the number of messages in the database.

core/data/migrations.go

@@ -29,6 +29,8 @@ func migrateDatabaseSchema(db *sql.DB, from, to int) error {
 			migrateToSchema4(db)
 		case 4:
 			migrateToSchema5(db)
+		case 5:
+			migrateToSchema6(db)
 		default:
 			log.Fatalln("missing database migration step")
 		}
@@ -42,6 +44,16 @@ func migrateDatabaseSchema(db *sql.DB, from, to int) error {
 	return nil
 }
 
+func migrateToSchema6(db *sql.DB) {
+	// Fix chat messages table schema. Since chat is ephemeral we can drop
+	// the table and recreate it.
+	// Drop the old messages table
+	mustExec(`DROP TABLE messages`, db)
+
+	// Recreate it
+	CreateMessagesTable(db)
+}
+
 // nolint:cyclop
 func migrateToSchema5(db *sql.DB) {
 	// Create the access tokens table.

core/data/utils.go

@@ -0,0 +1,20 @@
+package data
+
+import (
+	"database/sql"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// mustExec will execute a SQL statement on a provided database instance.
+func mustExec(s string, db *sql.DB) {
+	stmt, err := db.Prepare(s)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer stmt.Close()
+	_, err = stmt.Exec()
+	if err != nil {
+		log.Warnln(err)
+	}
+}

sqlc.json

@@ -1,9 +0,0 @@
-{
-    "version": "1",
-    "packages": [{
-      "schema": "db/schema.sql",
-      "queries": "db/query.sql",
-      "name": "db",
-      "path": "db"
-    }]
-  } 

sqlc.yaml

@@ -0,0 +1,6 @@
+version: 1
+packages:
+  - path: db
+    name: db
+    schema: 'db/schema.sql'
+    queries: 'db/query.sql'