FEAT: Video Search In Discovery

discovery/docs/docs.go

@@ -145,7 +145,7 @@ var SwaggerInfo = &swag.Spec{
 	BasePath:         "/",
 	Schemes:          []string{},
 	Title:            "Thamanyah Discovery API",
-	Description:      "JSON API for browsing the Thamanyah catalogue. Read-side counterpart to the CMS, which is what ingests videos.",
+	Description:      "JSON API for browsing the Thamanyah catalogue. Read-side counterpart to the CMS, which is what ingests videos.\n\nNot listed below: `QUERY /api/videos`, the catalogue search. It takes a JSON body `{title, categories, limit, cursor}` and answers `{videos, nextCursor}` — title is matched lexically with the last word as a prefix, categories narrow to videos filed under any one of them, limit defaults to 20 and is capped at 100, and cursor is the opaque `nextCursor` of a previous search. It is absent here because OpenAPI has no slot for the QUERY method, not because it is unsupported; see handlers.SearchVideos.",
 	InfoInstanceName: "swagger",
 	SwaggerTemplate:  docTemplate,
 	LeftDelim:        "{{",

discovery/docs/swagger.json

@@ -1,7 +1,7 @@
 {
     "swagger": "2.0",
     "info": {
-        "description": "JSON API for browsing the Thamanyah catalogue. Read-side counterpart to the CMS, which is what ingests videos.",
+        "description": "JSON API for browsing the Thamanyah catalogue. Read-side counterpart to the CMS, which is what ingests videos.\n\nNot listed below: `QUERY /api/videos`, the catalogue search. It takes a JSON body `{title, categories, limit, cursor}` and answers `{videos, nextCursor}` — title is matched lexically with the last word as a prefix, categories narrow to videos filed under any one of them, limit defaults to 20 and is capped at 100, and cursor is the opaque `nextCursor` of a previous search. It is absent here because OpenAPI has no slot for the QUERY method, not because it is unsupported; see handlers.SearchVideos.",
         "title": "Thamanyah Discovery API",
         "contact": {},
         "version": "1.0"

discovery/docs/swagger.yaml

@@ -43,8 +43,10 @@ definitions:
     type: object
 info:
   contact: {}
-  description: JSON API for browsing the Thamanyah catalogue. Read-side counterpart
-    to the CMS, which is what ingests videos.
+  description: |-
+    JSON API for browsing the Thamanyah catalogue. Read-side counterpart to the CMS, which is what ingests videos.
+
+    Not listed below: `QUERY /api/videos`, the catalogue search. It takes a JSON body `{title, categories, limit, cursor}` and answers `{videos, nextCursor}` — title is matched lexically with the last word as a prefix, categories narrow to videos filed under any one of them, limit defaults to 20 and is capped at 100, and cursor is the opaque `nextCursor` of a previous search. It is absent here because OpenAPI has no slot for the QUERY method, not because it is unsupported; see handlers.SearchVideos.
   title: Thamanyah Discovery API
   version: "1.0"
 paths:

discovery/internal/api/videos.go

@@ -11,3 +11,34 @@ type Video struct {
 	PlaybackURL string   `json:"playbackUrl" example:"https://d111111abcdef8.cloudfront.net/videos/abc123/index.m3u8"`
 	Categories  []string `json:"categories" example:"documentary,news"`
 }
+
+// SearchRequest is the body of QUERY /api/videos: what a reader is looking
+// for. Every member is optional — an empty body browses the whole catalogue.
+type SearchRequest struct {
+	// Title is matched lexically against video titles: the words are all
+	// required, and the last is treated as a prefix so a part-typed word still
+	// finds something.
+	Title string `json:"title" example:"desert fal"`
+	// Categories narrows the search to videos filed under any one of these
+	// names — not all of them, so it widens the choice of category while
+	// narrowing the results. Empty means every category.
+	Categories []string `json:"categories" example:"documentary,news"`
+	// Limit is how many videos to return. Absent or zero takes the default of
+	// 20; anything above the maximum of 100 is capped to it rather than
+	// refused, and the cursor still reaches the rest.
+	Limit int `json:"limit" example:"20"`
+	// Cursor asks for the page after the one a previous search ended at. Send
+	// back the nextCursor from that search, unchanged; it is opaque, and the
+	// only thing to do with it is return it.
+	Cursor string `json:"cursor"`
+}
+
+// SearchResults is one page of search results.
+type SearchResults struct {
+	// Videos are the matches, most relevant first. Never null: a search that
+	// matches nothing is an empty list.
+	Videos []Video `json:"videos"`
+	// NextCursor reaches the page after this one. Empty on the last page,
+	// which is how a reader knows there is no more to ask for.
+	NextCursor string `json:"nextCursor"`
+}

discovery/internal/db/migrations/0003_add_video_search.down.sql

@@ -0,0 +1,2 @@
+DROP INDEX IF EXISTS videos_search_vector_idx;
+ALTER TABLE videos DROP COLUMN IF EXISTS search_vector;

discovery/internal/db/migrations/0003_add_video_search.up.sql

@@ -0,0 +1,25 @@
+-- Lexical search over video titles.
+--
+-- The vector is a generated column rather than something the consumer writes:
+-- the catalogue's only writer is SaveVideo, and a title that arrives in an
+-- announcement should be searchable because it is stored, not because a second
+-- statement remembered to index it. STORED because a GIN index needs the value
+-- on disk to index it at all.
+--
+-- 'arabic' is the text search configuration, not 'english' or 'simple': it
+-- stems Arabic (الوثائقي -> وثايق) while leaving Latin-script tokens as written,
+-- which is the right trade for a mixed catalogue. The cost is that English
+-- words are not stemmed, so "documentaries" does not find "documentary".
+--
+-- The configuration is named explicitly rather than left to default_text_search_config,
+-- which is a per-session GUC — an expression that reads it is only STABLE, and
+-- a generated column requires IMMUTABLE. Naming it is also what keeps the
+-- column and every query in step: a query built with a different
+-- configuration would stem its terms differently and silently match nothing.
+ALTER TABLE videos
+    ADD COLUMN search_vector tsvector
+    GENERATED ALWAYS AS (to_tsvector('arabic', title)) STORED;
+
+-- GIN, not GiST: the catalogue is read far more than it is written, and GIN
+-- answers @@ faster at the cost of a slower update — the right way round here.
+CREATE INDEX videos_search_vector_idx ON videos USING GIN (search_vector);

discovery/internal/db/migrations/0004_add_video_categories_index.down.sql

@@ -0,0 +1 @@
+DROP INDEX IF EXISTS videos_categories_idx;

discovery/internal/db/migrations/0004_add_video_categories_index.up.sql

@@ -0,0 +1,7 @@
+-- Narrowing a search by category.
+--
+-- categories is a TEXT[] of names, and the search overlaps it against the
+-- names a reader asked for (&&). GIN is the index type that operator can use;
+-- without it the overlap is a filter applied after the rows are read, which is
+-- the whole table once the title term is broad.
+CREATE INDEX videos_categories_idx ON videos USING GIN (categories);

discovery/internal/db/migrations/0005_add_video_recency_index.down.sql

@@ -0,0 +1 @@
+DROP INDEX IF EXISTS videos_recent_idx;

discovery/internal/db/migrations/0005_add_video_recency_index.up.sql

@@ -0,0 +1,17 @@
+-- Browsing the catalogue, and paging through it.
+--
+-- Every search orders by (rank, created_at, id). When there is no title term
+-- the rank is a constant, and Postgres folds a constant sort key away — so the
+-- order that remains is exactly this index, and a browse becomes an index-only
+-- scan that stops as soon as the page is full instead of sorting the table.
+--
+-- Measured at 200k rows: a first page of a browse goes from a 15 ms parallel
+-- sequential scan with a top-N sort to 0.03 ms. It matters more for the cursor
+-- than for the first page — the keyset comparison (created_at, id) < (…)
+-- becomes an index condition rather than a filter, so a deep page is a seek
+-- to the right place in the index rather than a walk through everything
+-- before it.
+--
+-- DESC on both columns to match the ORDER BY: a btree can be read backwards,
+-- but only as a whole, so a mixed-direction ordering could not use it.
+CREATE INDEX videos_recent_idx ON videos (created_at DESC, id DESC);

discovery/internal/db/repositories/errors.go

@@ -7,3 +7,8 @@ import "errors"
 
 // ErrVideoNotFound reports that the catalogue holds no video with that id.
 var ErrVideoNotFound = errors.New("video not found")
+
+// ErrInvalidCursor reports that a paging cursor could not be read. It is the
+// caller's mistake, not a fault in the catalogue, so it must not reach the
+// handler as an indistinguishable query failure.
+var ErrInvalidCursor = errors.New("invalid cursor")

discovery/internal/db/repositories/videos.go

@@ -3,8 +3,13 @@ package repositories
 import (
 	"context"
 	"database/sql"
+	"encoding/base64"
+	"encoding/json"
 	"errors"
+	"strings"
 	"thamanyah/discovery/internal/models"
+	"time"
+	"unicode"
 
 	"github.com/lib/pq"
 )
@@ -57,3 +62,191 @@ func (svc VideoRepository) GetVideoByID(ctx context.Context, id string) (models.
 
 	return v, nil
 }
+
+// searchConfig is the text search configuration the search_vector column is
+// generated with. Every query has to name the same one: a tsquery built under
+// a different configuration stems its terms differently and matches nothing.
+const searchConfig = "arabic"
+
+// tsQueryFor turns what a reader typed into a tsquery.
+//
+// The words are ANDed, and the last one is given a :* prefix match — someone
+// typing into a search box is usually part-way through their last word, so
+// "desert fal" should still find "Desert Falcons". Earlier words are matched
+// whole, because they have been finished.
+//
+// The input is split on everything that is not a letter or a digit, which
+// drops every character tsquery gives a meaning to (&, |, !, :, *, parens) and
+// leaves nothing that could change the shape of the query. That is what makes
+// it safe to interpolate the result into to_tsquery: the alternative,
+// websearch_to_tsquery, parses user syntax but cannot express a prefix match.
+//
+// It returns "" when there is nothing to search for, which callers read as
+// "no title filter" rather than "match nothing".
+func tsQueryFor(term string) string {
+	words := strings.FieldsFunc(term, func(r rune) bool {
+		return !unicode.IsLetter(r) && !unicode.IsDigit(r)
+	})
+	if len(words) == 0 {
+		return ""
+	}
+
+	for i, word := range words {
+		words[i] = strings.ToLower(word)
+	}
+	return strings.Join(words, " & ") + ":*"
+}
+
+// searchCursor is where a page of results stopped: the sort key of its last
+// row. Paging resumes by asking for the rows that sort after it.
+//
+// It carries the rank as well as the row's identity because the results are
+// ordered by relevance first, and "the row after this one" is only a
+// well-defined place if every term of the ordering is pinned.
+type searchCursor struct {
+	Rank      float64   `json:"r"`
+	CreatedAt time.Time `json:"t"`
+	ID        string    `json:"i"`
+}
+
+// encode renders a cursor as an opaque string. Opaque on purpose: it is a
+// place in a result set, not a number a client should do arithmetic on, and
+// making it look like one invites callers to guess a page.
+func (c searchCursor) encode() (string, error) {
+	raw, err := json.Marshal(c)
+	if err != nil {
+		return "", err
+	}
+	return base64.RawURLEncoding.EncodeToString(raw), nil
+}
+
+func decodeCursor(encoded string) (searchCursor, error) {
+	raw, err := base64.RawURLEncoding.DecodeString(encoded)
+	if err != nil {
+		return searchCursor{}, ErrInvalidCursor
+	}
+
+	var c searchCursor
+	if err := json.Unmarshal(raw, &c); err != nil {
+		return searchCursor{}, ErrInvalidCursor
+	}
+	if strings.TrimSpace(c.ID) == "" {
+		return searchCursor{}, ErrInvalidCursor
+	}
+
+	return c, nil
+}
+
+// SearchVideos finds catalogued videos whose title matches the given term and
+// which are filed under any of the given categories, most relevant first.
+//
+// It returns the page and the cursor that reaches the page after it; that
+// cursor is empty on the last page, which is how a caller knows to stop.
+//
+// The title predicate is skipped entirely when the term yields no words, and
+// the category predicate when no categories are named, so a search with
+// neither browses the whole catalogue newest-first.
+//
+// Paging is by key rather than by offset. OFFSET makes the database walk and
+// discard every row it skips, so deep pages get steadily more expensive, and
+// it addresses a page by its position — which moves whenever a video is
+// announced, repeating or skipping rows for a reader mid-browse. Comparing
+// against the last row's sort key has neither problem: the work is the same at
+// any depth, and the page after a given row is that same page however much has
+// been added since.
+func (svc VideoRepository) SearchVideos(ctx context.Context, term string, categories []string, limit int, cursor string) ([]models.Video, string, error) {
+	query := tsQueryFor(term)
+
+	// Never nil: a nil slice reaches Postgres as NULL, and cardinality(NULL)
+	// is NULL rather than 0, so the "no category filter" branch would not fire
+	// and every search would come back empty.
+	if categories == nil {
+		categories = []string{}
+	}
+
+	// A zero cursor is passed as NULL, which the query reads as "start at the
+	// beginning" rather than as a row to seek past.
+	var after *searchCursor
+	if strings.TrimSpace(cursor) != "" {
+		decoded, err := decodeCursor(cursor)
+		if err != nil {
+			return nil, "", err
+		}
+		after = &decoded
+	}
+
+	var (
+		afterRank *float64
+		afterTime *time.Time
+		afterID   *string
+	)
+	if after != nil {
+		afterRank, afterTime, afterID = &after.Rank, &after.CreatedAt, &after.ID
+	}
+
+	// One row more than asked for: if it comes back, there is another page and
+	// the extra row is discarded. That is what lets the last page say so,
+	// rather than handing out a cursor to an empty page.
+	//
+	// The rank is computed in the subquery and referenced by name in both the
+	// cursor comparison and the ORDER BY, so the two cannot drift apart — if
+	// they did, "the row after this one" would mean something different from
+	// the order the rows are actually in, and pages would overlap.
+	rows, err := svc.SQLDB.QueryContext(ctx, `
+		SELECT id, title, playback_url, categories, created_at, rank
+		FROM (
+			SELECT id, title, playback_url, categories, created_at,
+			       CASE WHEN $1 = '' THEN 0::float8
+			            ELSE ts_rank(search_vector, to_tsquery('`+searchConfig+`', $1))::float8
+			       END AS rank
+			FROM videos
+			WHERE ($1 = '' OR search_vector @@ to_tsquery('`+searchConfig+`', $1))
+			  AND (cardinality($2::text[]) = 0 OR categories && $2)
+		) ranked
+		WHERE $4::float8 IS NULL
+		   OR (rank, created_at, id) < ($4::float8, $5::timestamptz, $6::uuid)
+		ORDER BY rank DESC, created_at DESC, id DESC
+		LIMIT $3
+	`, query, pq.Array(categories), limit+1, afterRank, afterTime, afterID)
+	if err != nil {
+		return nil, "", err
+	}
+	defer rows.Close()
+
+	// Never nil: a search that matches nothing is an empty list on the wire,
+	// not null.
+	videos := []models.Video{}
+	ranks := []float64{}
+	for rows.Next() {
+		var (
+			v    models.Video
+			rank float64
+		)
+		if err := rows.Scan(&v.ID, &v.Title, &v.PlaybackURL, pq.Array(&v.Categories), &v.CreatedAt, &rank); err != nil {
+			return nil, "", err
+		}
+		videos = append(videos, v)
+		ranks = append(ranks, rank)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, "", err
+	}
+
+	if len(videos) <= limit {
+		return videos, "", nil
+	}
+
+	videos, ranks = videos[:limit], ranks[:limit]
+	last := videos[len(videos)-1]
+
+	next, err := searchCursor{
+		Rank:      ranks[len(ranks)-1],
+		CreatedAt: last.CreatedAt,
+		ID:        last.ID,
+	}.encode()
+	if err != nil {
+		return nil, "", err
+	}
+
+	return videos, next, nil
+}

discovery/internal/handlers/videos.go

@@ -1,6 +1,7 @@
 package handlers
 
 import (
+	"encoding/json"
 	"errors"
 	"log"
 	"net/http"
@@ -47,3 +48,76 @@ func GetVideo(w http.ResponseWriter, r *http.Request) {
 		Categories: append([]string{}, video.Categories...),
 	})
 }
+
+const (
+	// searchPageSize is how many videos a search returns when the caller does
+	// not say. Small enough to keep the common request cheap.
+	searchPageSize = 20
+
+	// maxSearchPageSize is the largest page the catalogue will build, however
+	// big a page the caller asks for. The endpoint is read constantly, and a
+	// page size is a request for work: without a ceiling, one client asking
+	// for the whole catalogue in a single response makes every other reader
+	// wait behind it.
+	//
+	// Asking for more is not an error — the page is simply capped, and the
+	// cursor still reaches the rest — because a client that wants everything
+	// is doing something legitimate, just not in one request.
+	maxSearchPageSize = 100
+)
+
+// SearchVideos serves the catalogue's lexical search.
+//
+// It is served on QUERY rather than GET or POST: a search is safe and
+// idempotent, which POST is not, but its parameters are a structured document
+// that does not belong in a query string. QUERY says both. It is specified in
+// draft-ietf-httpbis-safe-method-w-body, not in RFC 9110 — still a draft, so
+// expect intermediaries that have never heard of it. Note it also cannot be
+// expressed in the generated OpenAPI spec — Swagger 2.0 and OpenAPI 3.x define a fixed set of
+// per-method fields with no slot for QUERY, and swag rejects the annotation
+// outright — so this endpoint is deliberately absent from /swagger and
+// documented here instead.
+func SearchVideos(w http.ResponseWriter, r *http.Request) {
+	var request api.SearchRequest
+
+	if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
+		writeProblem(w, http.StatusBadRequest, "Malformed Search",
+			"The request body could not be read as JSON. Send a JSON object describing what to search for; an empty object browses the whole catalogue.")
+		return
+	}
+
+	limit := request.Limit
+	if limit <= 0 {
+		limit = searchPageSize
+	}
+	if limit > maxSearchPageSize {
+		limit = maxSearchPageSize
+	}
+
+	found, next, err := repositories.VideoRepo.SearchVideos(r.Context(), request.Title, request.Categories, limit, request.Cursor)
+	if errors.Is(err, repositories.ErrInvalidCursor) {
+		writeProblem(w, http.StatusBadRequest, "Malformed Search Cursor",
+			"The cursor could not be read. Send back the nextCursor from a previous search unchanged, or omit it to start from the first page.")
+		return
+	}
+	if err != nil {
+		log.Printf("Something Went Wrong Searching The Catalogue: title=%q: %s", request.Title, err)
+		writeProblem(w, http.StatusInternalServerError, "Something Went Wrong Searching The Catalogue",
+			"The catalogue could not be searched. This is a server-side fault and the request was not processed; retrying in a few moments may succeed.")
+		return
+	}
+
+	// Translated field by field, for the same reason GetVideo does it: the row
+	// and the published schema are different things.
+	videos := make([]api.Video, 0, len(found))
+	for _, v := range found {
+		videos = append(videos, api.Video{
+			ID:          v.ID,
+			Title:       v.Title,
+			PlaybackURL: v.PlaybackURL,
+			Categories:  append([]string{}, v.Categories...),
+		})
+	}
+
+	writeJSON(w, http.StatusOK, api.SearchResults{Videos: videos, NextCursor: next})
+}

discovery/main.go

@@ -23,6 +23,8 @@ import (
 //	@title			Thamanyah Discovery API
 //	@version		1.0
 //	@description	JSON API for browsing the Thamanyah catalogue. Read-side counterpart to the CMS, which is what ingests videos.
+//	@description
+//	@description	Not listed below: `QUERY /api/videos`, the catalogue search. It takes a JSON body `{title, categories, limit, cursor}` and answers `{videos, nextCursor}` — title is matched lexically with the last word as a prefix, categories narrow to videos filed under any one of them, limit defaults to 20 and is capped at 100, and cursor is the opaque `nextCursor` of a previous search. It is absent here because OpenAPI has no slot for the QUERY method, not because it is unsupported; see handlers.SearchVideos.
 //	@BasePath		/
 
 func main() {
@@ -81,6 +83,9 @@ func runServer() {
 
 	mux.HandleFunc("GET /health", handlers.Health)
 	mux.HandleFunc("GET /api/videos/{id}", handlers.GetVideo)
+	// QUERY, not GET: a search is safe and idempotent but carries a structured
+	// body. See handlers.SearchVideos for why it is absent from the spec.
+	mux.HandleFunc("QUERY /api/videos", handlers.SearchVideos)
 
 	// Swagger UI and the generated spec. The UI assets are embedded in the
 	// binary by swaggo/files, so this needs no static directory on disk.