FEAT: Video Search In Discovery

tests/README.md

@@ -77,6 +77,19 @@ Both steps of the upload flow, end to end:
   not there at all, and a video announced a second time is updated rather than
   colliding with its own row
 
+- searching that catalogue over `QUERY /api/videos`: finding a video by a word
+  in its title and by a part-typed one, narrowing to a category and being left
+  out of another, several categories meaning "any of", paging by cursor so the
+  pages tile the results exactly once, refusing a cursor no search issued, and
+  capping a page size that asks for the whole catalogue
+
+Search scenarios give every video they catalogue a nonce in its title, and
+assert relative to the video they created ("returns that video") rather than on
+absolute counts. Nothing cleans up between runs, so a fixed title accumulates a
+copy per run and a count would stop meaning anything. The paging scenario shares
+one nonce across its videos so that searching for it matches that run's cohort
+and nothing else.
+
 The publication scenarios read `tests-catalogue-events`, the suite's **own**
 queue on the catalogue topic — not discovery's. A consumer destroys what it
 reads, so sharing discovery's queue would have the suite and the running

tests/client_test.go

@@ -139,6 +139,55 @@ type catalogueVideoBody struct {
 	Categories  []string `json:"categories"`
 }
 
+// searchRequest is the body of QUERY /api/videos on discovery: what a reader
+// is looking for, plus where in the results to carry on from.
+//
+// Every field is optional. An empty body is the whole catalogue, newest first.
+type searchRequest struct {
+	Title      string   `json:"title,omitempty"`
+	Categories []string `json:"categories,omitempty"`
+	Limit      int      `json:"limit,omitempty"`
+	Cursor     string   `json:"cursor,omitempty"`
+}
+
+// searchResultsBody is what that query answers with: the page of videos, and
+// the cursor that reaches the page after it. NextCursor is empty on the last
+// page, which is how a client knows to stop.
+type searchResultsBody struct {
+	Videos     []catalogueVideoBody `json:"videos"`
+	NextCursor string               `json:"nextCursor"`
+}
+
+// ids lists the video ids in the page, for asserting on which videos came back
+// without caring about the rest of their fields.
+func (b searchResultsBody) ids() []string {
+	found := make([]string, 0, len(b.Videos))
+	for _, v := range b.Videos {
+		found = append(found, v.ID)
+	}
+	return found
+}
+
+func (b searchResultsBody) holds(id string) bool {
+	for _, v := range b.Videos {
+		if v.ID == id {
+			return true
+		}
+	}
+	return false
+}
+
+// queryJSON sends a QUERY request — the method the catalogue search is served
+// on, chosen because a search is safe and idempotent like a GET but carries a
+// structured body no query string would hold comfortably.
+func (c *client) queryJSON(path string, payload any) (response, error) {
+	encoded, err := json.Marshal(payload)
+	if err != nil {
+		return response{}, err
+	}
+	return c.send("QUERY", c.base+path, "application/json", encoded)
+}
+
 // postJSON marshals payload and posts it to a path on the CMS.
 func (c *client) postJSON(path string, payload any) (response, error) {
 	encoded, err := json.Marshal(payload)

tests/features/catalogue_search.feature

@@ -0,0 +1,78 @@
+Feature: Searching the catalogue
+  Discovery holds a copy of every video cms has announced as ready. Reaching
+  one by id is no use to a reader who does not have an id — finding something
+  to watch means searching for it by what it is called and what it is filed
+  under.
+
+  The search is lexical: it matches the words of a title, not its meaning. The
+  last word a reader types is treated as a prefix, because someone typing into
+  a search box is usually part-way through a word.
+
+  Background:
+    Given the CMS API is available
+    And the Discovery API is available
+
+  Scenario: Finding a catalogued video by a word in its title
+    Given the catalogue holds a video titled "Desert Falcons"
+    When I search the catalogue for that video's title
+    Then the search returns that video
+
+  # The last word is matched as a prefix, so the results keep up with someone
+  # still typing. Earlier words are matched whole — they have been finished.
+  Scenario: Finding a video from a part-typed word
+    Given the catalogue holds a video titled "Desert Falcons"
+    When I search the catalogue for "Desert Fal"
+    Then the search returns that video
+
+  # Categories narrow a search rather than widening it: a video filed elsewhere
+  # is not an answer to a reader who asked for one category in particular.
+  Scenario: A search narrowed to another category leaves the video out
+    Given the catalogue holds a video titled "Desert Falcons" filed under "documentary"
+    When I search the catalogue for that video's title in category "news"
+    Then the search does not return that video
+
+  Scenario: A search narrowed to the video's own category still finds it
+    Given the catalogue holds a video titled "Desert Falcons" filed under "documentary"
+    When I search the catalogue for that video's title in category "documentary"
+    Then the search returns that video
+
+  # Several categories mean "any of these", not "all of these": naming more of
+  # them offers the reader more, rather than demanding the video be filed under
+  # every one at once.
+  Scenario: Naming several categories matches a video filed under any of them
+    Given the catalogue holds a video titled "Desert Falcons" filed under "documentary"
+    When I search the catalogue for that video's title in categories "news, documentary"
+    Then the search returns that video
+
+  # Paging is by cursor rather than by offset: the catalogue is added to while
+  # people are reading it, and an offset silently repeats or skips a video when
+  # something is announced between one page and the next. A cursor names where
+  # the last page stopped, so the page after it is the same page whenever it is
+  # asked for.
+  Scenario: Paging through the results a page at a time
+    Given the catalogue holds 3 videos titled "Mountain Wolves"
+    When I search the catalogue for that title 2 at a time
+    Then the search returns 2 videos
+    And the search offers a cursor to the next page
+    When I follow the cursor
+    Then the search returns 1 video
+    And the search offers no further cursor
+    And the pages together hold every one of those videos exactly once
+
+  # A cursor is opaque, so a client that makes one up has made a mistake rather
+  # than found a fault. Saying 400 is what tells the two apart.
+  Scenario: Refusing a cursor no search issued
+    Given the catalogue holds a video titled "Desert Falcons"
+    When I search the catalogue for that video's title from the cursor "not-a-cursor"
+    Then the request is rejected with status 400
+    And the problem title is "Malformed Search Cursor"
+
+  # A page size is a request for work, and an uncapped one is a request for all
+  # of it — the catalogue is read constantly, and one client asking for every
+  # row at once should not be able to make everyone else wait. The cap applies
+  # silently rather than as a rejection: the reader still gets a page, and the
+  # cursor still reaches the rest.
+  Scenario: Capping a page size that asks for the whole catalogue
+    When I search the whole catalogue 5000 at a time
+    Then the search returns at most 100 videos
+    And the search offers a cursor to the next page

tests/steps_test.go

@@ -3,8 +3,10 @@ package tests
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"net/url"
 	"slices"
+	"strconv"
 	"strings"
 	"time"
 
@@ -902,3 +904,316 @@ func theCatalogueEventuallyHoldsThatVideoWithNoPlaybackURL(t gobdd.StepTest, ctx
 	t.Errorf("the catalogue still points a player at %q for video %q after %s; the later "+
 		"announcement carried no playlist and should have replaced it", last, w.video.ID, catalogueSettleTimeout)
 }
+
+// --- Catalogue search ------------------------------------------------------
+
+// searchLimit is the page size the search scenarios ask for unless they are
+// about paging itself.
+const searchLimit = 20
+
+// uniqueTitle appends a nonce to the title a scenario names.
+//
+// Scenarios write real rows and nothing cleans up after them, so a fixed title
+// accumulates a copy per run and "the search returns exactly this video" stops
+// meaning anything. The nonce keeps every run's video findable on its own
+// while the feature file still reads in plain words.
+func uniqueTitle(title string) string {
+	return fmt.Sprintf("%s %s", title, strings.ToUpper(strconv.FormatInt(time.Now().UnixNano(), 36)))
+}
+
+// theCatalogueHoldsAVideoTitled puts one video into the catalogue under a
+// title the scenario chose: register it, report its transcode finished, and
+// wait for the announcement to land on the read side.
+func theCatalogueHoldsAVideoTitled(t gobdd.StepTest, ctx gobdd.Context, title string) {
+	catalogueVideoTitledUnder(t, ctx, title, "other")
+}
+
+// catalogueVideoTitledUnder is the whole write-to-read round trip for one
+// video, which is what "the catalogue holds …" costs: cms issues the id, the
+// job completes, cms announces it, discovery ingests it.
+func catalogueVideoTitledUnder(t gobdd.StepTest, ctx gobdd.Context, title, categories string) {
+	w := worldOf(t, ctx)
+
+	full := uniqueTitle(title)
+
+	requestUploadSlot(t, ctx, "searchable.mp4", "video/mp4")
+	uploadTheFile(t, ctx)
+	registerUploadedVideo(t, ctx, full, categories)
+
+	if w.last.status != 201 {
+		t.Fatalf("could not register a video to search for: %s", w.last.summary())
+		return
+	}
+
+	w.searchTitle = full
+
+	mediaConvertReportsState(t, ctx, "COMPLETE")
+	theCatalogueEventuallyHoldsThatVideo(t, ctx)
+}
+
+// searchTheCatalogueForThatTitle searches for the exact title the scenario's
+// video was catalogued under, nonce included — the scenario says "that video's
+// title" precisely so it does not have to know about the nonce.
+func searchTheCatalogueForThatTitle(t gobdd.StepTest, ctx gobdd.Context) {
+	w := worldOf(t, ctx)
+
+	if strings.TrimSpace(w.searchTitle) == "" {
+		t.Fatalf("no video has been catalogued, so there is no title to search for")
+		return
+	}
+
+	searchTheCatalogue(t, w, searchRequest{Title: w.searchTitle, Limit: searchLimit})
+}
+
+// searchTheCatalogue performs the query and decodes the page, leaving both the
+// raw exchange and the decoded results on the world so a "Then" step can
+// assert on either.
+func searchTheCatalogue(t gobdd.StepTest, w *world, request searchRequest) {
+	if w.discovery == nil {
+		w.discovery = newDiscoveryClient()
+	}
+
+	result, err := w.discovery.queryJSON("/api/videos", request)
+	if err != nil {
+		t.Fatalf("could not search the catalogue: %s", err)
+		return
+	}
+
+	w.last = result
+	w.results = searchResultsBody{}
+
+	// A rejected search has no page to decode; the scenario asserting the
+	// rejection reads w.last instead.
+	if result.status == 200 {
+		if err := result.json(&w.results); err != nil {
+			t.Fatalf("could not decode the search results: %s", err)
+		}
+	}
+}
+
+func theSearchReturnsThatVideo(t gobdd.StepTest, ctx gobdd.Context) {
+	w := worldOf(t, ctx)
+
+	if w.last.status != 200 {
+		t.Fatalf("the search did not succeed: %s", w.last.summary())
+		return
+	}
+	if !w.results.holds(w.video.ID) {
+		t.Errorf("the search for %q did not return video %q; it returned %v",
+			w.searchTitle, w.video.ID, w.results.ids())
+	}
+}
+
+// searchTheCatalogueFor searches for a term the scenario spells out, rather
+// than for the title of the video it catalogued.
+func searchTheCatalogueFor(t gobdd.StepTest, ctx gobdd.Context, term string) {
+	searchTheCatalogue(t, worldOf(t, ctx), searchRequest{Title: term, Limit: searchLimit})
+}
+
+// theCatalogueHoldsAVideoTitledUnder is theCatalogueHoldsAVideoTitled for a
+// scenario that cares which categories the video is filed under.
+func theCatalogueHoldsAVideoTitledUnder(t gobdd.StepTest, ctx gobdd.Context, title, categories string) {
+	catalogueVideoTitledUnder(t, ctx, title, categories)
+}
+
+func searchTheCatalogueForThatTitleInCategory(t gobdd.StepTest, ctx gobdd.Context, category string) {
+	w := worldOf(t, ctx)
+
+	if strings.TrimSpace(w.searchTitle) == "" {
+		t.Fatalf("no video has been catalogued, so there is no title to search for")
+		return
+	}
+
+	searchTheCatalogue(t, w, searchRequest{
+		Title:      w.searchTitle,
+		Categories: []string{category},
+		Limit:      searchLimit,
+	})
+}
+
+func theSearchDoesNotReturnThatVideo(t gobdd.StepTest, ctx gobdd.Context) {
+	w := worldOf(t, ctx)
+
+	if w.last.status != 200 {
+		t.Fatalf("the search did not succeed: %s", w.last.summary())
+		return
+	}
+	if w.results.holds(w.video.ID) {
+		t.Errorf("the search returned video %q, which it should have left out; it returned %v",
+			w.video.ID, w.results.ids())
+	}
+}
+
+// searchTheCatalogueForThatTitleInCategories is the several-categories form,
+// for pinning that they mean "any of" rather than "all of".
+func searchTheCatalogueForThatTitleInCategories(t gobdd.StepTest, ctx gobdd.Context, categories string) {
+	w := worldOf(t, ctx)
+
+	if strings.TrimSpace(w.searchTitle) == "" {
+		t.Fatalf("no video has been catalogued, so there is no title to search for")
+		return
+	}
+
+	names := []string{}
+	for _, name := range strings.Split(categories, ",") {
+		if name = strings.TrimSpace(name); name != "" {
+			names = append(names, name)
+		}
+	}
+
+	searchTheCatalogue(t, w, searchRequest{
+		Title:      w.searchTitle,
+		Categories: names,
+		Limit:      searchLimit,
+	})
+}
+
+// theCatalogueHoldsVideosTitled catalogues several videos under one shared
+// title, which is what a paging scenario needs: enough matches to fill more
+// than one page.
+//
+// The nonce is generated once and shared, rather than per video, so that
+// searching for it matches this scenario's videos and no others. Runs leave
+// their videos behind, so a term that also matched a previous run's would make
+// "the search returns 2 videos" mean nothing.
+func theCatalogueHoldsVideosTitled(t gobdd.StepTest, ctx gobdd.Context, count int, title string) {
+	w := worldOf(t, ctx)
+
+	w.searchTitle = uniqueTitle(title)
+	w.cohort = nil
+	w.pageSeen = nil
+
+	for i := 0; i < count; i++ {
+		requestUploadSlot(t, ctx, "searchable.mp4", "video/mp4")
+		uploadTheFile(t, ctx)
+		registerUploadedVideo(t, ctx, w.searchTitle, "other")
+
+		if w.last.status != 201 {
+			t.Fatalf("could not register video %d of %d to page through: %s", i+1, count, w.last.summary())
+			return
+		}
+
+		mediaConvertReportsState(t, ctx, "COMPLETE")
+		theCatalogueEventuallyHoldsThatVideo(t, ctx)
+
+		w.cohort = append(w.cohort, w.video.ID)
+	}
+}
+
+func searchTheCatalogueForThatTitleAPageAtATime(t gobdd.StepTest, ctx gobdd.Context, size int) {
+	w := worldOf(t, ctx)
+
+	w.pageSeen = nil
+	searchTheCatalogue(t, w, searchRequest{Title: w.searchTitle, Limit: size})
+	recordPage(w)
+}
+
+// followTheCursor asks for the page after the one just returned, with the same
+// term and page size — a cursor says where to carry on from, not what to look
+// for.
+func followTheCursor(t gobdd.StepTest, ctx gobdd.Context) {
+	w := worldOf(t, ctx)
+
+	if w.results.NextCursor == "" {
+		t.Fatalf("the last page offered no cursor, so there is no next page to follow")
+		return
+	}
+
+	searchTheCatalogue(t, w, searchRequest{
+		Title:  w.searchTitle,
+		Limit:  len(w.results.Videos),
+		Cursor: w.results.NextCursor,
+	})
+	recordPage(w)
+}
+
+// recordPage remembers what a page held, so a later step can check the pages
+// tile the results.
+func recordPage(w *world) {
+	w.pageSeen = append(w.pageSeen, w.results.ids()...)
+}
+
+func theSearchReturnsNVideos(t gobdd.StepTest, ctx gobdd.Context, count int) {
+	w := worldOf(t, ctx)
+
+	if w.last.status != 200 {
+		t.Fatalf("the search did not succeed: %s", w.last.summary())
+		return
+	}
+	if len(w.results.Videos) != count {
+		t.Errorf("expected %d videos in the page, got %d: %v", count, len(w.results.Videos), w.results.ids())
+	}
+}
+
+func theSearchOffersACursor(t gobdd.StepTest, ctx gobdd.Context) {
+	w := worldOf(t, ctx)
+
+	if strings.TrimSpace(w.results.NextCursor) == "" {
+		t.Errorf("the page offered no cursor, so there is no way to ask for the next one")
+	}
+}
+
+func theSearchOffersNoFurtherCursor(t gobdd.StepTest, ctx gobdd.Context) {
+	w := worldOf(t, ctx)
+
+	if w.results.NextCursor != "" {
+		t.Errorf("the last page still offered cursor %q, so a reader cannot tell they have reached the end",
+			w.results.NextCursor)
+	}
+}
+
+// thePagesTileTheCohort checks the pages between them held each of the
+// scenario's videos exactly once — the property a cursor exists to give, and
+// the one an offset loses as soon as the catalogue is written to.
+func thePagesTileTheCohort(t gobdd.StepTest, ctx gobdd.Context) {
+	w := worldOf(t, ctx)
+
+	seen := map[string]int{}
+	for _, id := range w.pageSeen {
+		seen[id]++
+	}
+
+	for _, id := range w.cohort {
+		switch seen[id] {
+		case 1:
+		case 0:
+			t.Errorf("video %q was catalogued but appeared in none of the pages; the pages held %v",
+				id, w.pageSeen)
+		default:
+			t.Errorf("video %q appeared in %d pages; a video should be on exactly one", id, seen[id])
+		}
+	}
+
+	if len(w.pageSeen) != len(w.cohort) {
+		t.Errorf("the pages held %d videos between them, but %d were catalogued: %v",
+			len(w.pageSeen), len(w.cohort), w.pageSeen)
+	}
+}
+
+func searchTheCatalogueFromCursor(t gobdd.StepTest, ctx gobdd.Context, cursor string) {
+	w := worldOf(t, ctx)
+	searchTheCatalogue(t, w, searchRequest{
+		Title:  w.searchTitle,
+		Limit:  searchLimit,
+		Cursor: cursor,
+	})
+}
+
+// searchTheWholeCatalogue searches with no term and no categories, which is
+// how a reader browses rather than searches.
+func searchTheWholeCatalogue(t gobdd.StepTest, ctx gobdd.Context, size int) {
+	searchTheCatalogue(t, worldOf(t, ctx), searchRequest{Limit: size})
+}
+
+func theSearchReturnsAtMostNVideos(t gobdd.StepTest, ctx gobdd.Context, most int) {
+	w := worldOf(t, ctx)
+
+	if w.last.status != 200 {
+		t.Fatalf("the search did not succeed: %s", w.last.summary())
+		return
+	}
+	if len(w.results.Videos) > most {
+		t.Errorf("the search returned %d videos, more than the %d it should cap at",
+			len(w.results.Videos), most)
+	}
+}

tests/suite_test.go

@@ -48,6 +48,24 @@ type world struct {
 	// catalogued is the read side's own copy of the video, once a step has
 	// waited for it to arrive.
 	catalogued *catalogueVideoBody
+
+	// searchTitle is the exact title the scenario's video was catalogued
+	// under. Scenarios name a plain title like "Desert Falcons"; the step
+	// registering it appends a nonce, because nothing cleans up between runs
+	// and a title reused across runs would make a result count meaningless.
+	searchTitle string
+
+	// results is the last page the catalogue search answered with.
+	results searchResultsBody
+
+	// pageSeen accumulates the ids from every page a paging scenario has
+	// walked, so it can assert the pages tile the results rather than
+	// repeating or dropping one.
+	pageSeen []string
+
+	// cohort is the ids of the videos a scenario catalogued together under one
+	// shared title, in the order they were created.
+	cohort []string
 }
 
 type worldKey struct{}
@@ -186,6 +204,24 @@ func TestVideoUpload(t *testing.T) {
 	suite.AddStep(`^the catalogue does not hold that video$`, theCatalogueDoesNotHoldThatVideo)
 	suite.AddStep(`^the catalogue eventually holds that video with no playback URL$`, theCatalogueEventuallyHoldsThatVideoWithNoPlaybackURL)
 	suite.AddStep(`^the catalogue's copy carries its title, playback URL and categories$`, theCataloguesCopyCarriesTheDetails)
+	suite.AddStep(`^the catalogue holds a video titled "([^"]*)"$`, theCatalogueHoldsAVideoTitled)
+	suite.AddStep(`^I search the catalogue for that video's title$`, searchTheCatalogueForThatTitle)
+	suite.AddStep(`^I search the catalogue for "([^"]*)"$`, searchTheCatalogueFor)
+	suite.AddStep(`^the catalogue holds a video titled "([^"]*)" filed under "([^"]*)"$`, theCatalogueHoldsAVideoTitledUnder)
+	suite.AddStep(`^I search the catalogue for that video's title in category "([^"]*)"$`, searchTheCatalogueForThatTitleInCategory)
+	suite.AddStep(`^I search the catalogue for that video's title in categories "([^"]*)"$`, searchTheCatalogueForThatTitleInCategories)
+	suite.AddStep(`^the catalogue holds (\d+) videos titled "([^"]*)"$`, theCatalogueHoldsVideosTitled)
+	suite.AddStep(`^I search the catalogue for that title (\d+) at a time$`, searchTheCatalogueForThatTitleAPageAtATime)
+	suite.AddStep(`^I search the whole catalogue (\d+) at a time$`, searchTheWholeCatalogue)
+	suite.AddStep(`^I follow the cursor$`, followTheCursor)
+	suite.AddStep(`^I search the catalogue for that video's title from the cursor "([^"]*)"$`, searchTheCatalogueFromCursor)
+	suite.AddStep(`^the search returns (\d+) videos?$`, theSearchReturnsNVideos)
+	suite.AddStep(`^the search returns at most (\d+) videos$`, theSearchReturnsAtMostNVideos)
+	suite.AddStep(`^the search offers a cursor to the next page$`, theSearchOffersACursor)
+	suite.AddStep(`^the search offers no further cursor$`, theSearchOffersNoFurtherCursor)
+	suite.AddStep(`^the pages together hold every one of those videos exactly once$`, thePagesTileTheCohort)
+	suite.AddStep(`^the search does not return that video$`, theSearchDoesNotReturnThatVideo)
+	suite.AddStep(`^the search returns that video$`, theSearchReturnsThatVideo)
 	suite.AddStep(`^I register the uploaded video titled "(.*)" under categories "(.*)"$`, registerUploadedVideo)
 	suite.AddStep(`^I register the uploaded video titled "(.*)" under category id (\d+)$`, registerUnderCategoryID)
 	suite.AddStep(`^I register a video titled "(.*)" with the key "(.*)" under categories "(.*)"$`, registerWithKey)