feat: Add pagination to /devices/list (#236)

* Add pagination to /devices/list

* Fix schema

* Run codegen

* Allow page_cursor nullable

* Reuse query from previous page_cursor

* Validate params when using next_page_cursor

* Validate query for page_cursor

* Fix type error

* Mark failing test

Author: razor-x

src/lib/server.ts

@@ -42,7 +42,7 @@ export async function startServer(
     middlewares: [
       (next) => (req: ApiRequest, res) => {
         req.db = database
-        req.baseUrl = baseUrl
+        req.baseUrl = baseUrl.replace(/\/+$/, "")
         return next(req, res)
       },
     ],

src/pages/api/devices/list.ts

@@ -1,3 +1,8 @@
+import { createHash } from "node:crypto"
+
+import { serializeUrlSearchParams } from "@seamapi/url-search-params-serializer"
+import { sortBy } from "lodash"
+import { BadRequestException } from "nextlove"
 import { z } from "zod"
 
 import { device, device_type } from "lib/zod/index.ts"
@@ -12,6 +17,23 @@ export const common_params = z.object({
   device_type: device_type.optional(),
   device_types: z.array(device_type).optional(),
   manufacturer: z.string().optional(),
+  limit: z.coerce.number().int().positive().default(500),
+  page_cursor: z
+    .string()
+    .optional()
+    .nullable()
+    .transform((page_cursor) => {
+      if (page_cursor == null) return page_cursor
+      return page_cursor_schema.parse(
+        JSON.parse(Buffer.from(page_cursor, "base64").toString("utf8")),
+      )
+    }),
+})
+
+const page_cursor_schema = z.object({
+  created_at: z.coerce.date(),
+  device_id: z.string(),
+  query_hash: z.string(),
 })
 
 export default withRouteSpec({
@@ -20,16 +42,24 @@ export default withRouteSpec({
   commonParams: common_params,
   jsonResponse: z.object({
     devices: z.array(device),
+    pagination: z.object({
+      has_next_page: z.boolean(),
+      next_page_cursor: z.string().nullable(),
+      next_page_url: z.string().url().nullable(),
+    }),
   }),
 } as const)(async (req, res) => {
+  const { page_cursor, ...params } = req.commonParams
+
   const {
     device_ids,
     connected_account_id,
     connected_account_ids,
     device_type,
     device_types,
     manufacturer,
-  } = req.commonParams
+    limit,
+  } = params
 
   const { workspace_id } = req.auth
 
@@ -52,7 +82,74 @@ export default withRouteSpec({
     )
   }
 
+  devices = sortBy(devices, ["created_at", "device_id"])
+
+  const device_id = page_cursor?.device_id
+  const startIdx =
+    device_id == null
+      ? 0
+      : devices.findIndex((device) => device.device_id === device_id)
+
+  const endIdx = Math.min(startIdx + limit, devices.length)
+  const page = devices.slice(startIdx, endIdx)
+  const next_device = devices[endIdx]
+  const has_next_page = next_device != null
+
+  const query_hash = getPageCursorQueryHash(params)
+  if (
+    page_cursor?.query_hash != null &&
+    page_cursor.query_hash !== query_hash
+  ) {
+    throw new BadRequestException({
+      type: "mismatched_page_parameters",
+      message:
+        "When using next_page_cursor, the request send parameters identical to the initial request.",
+    })
+  }
+
+  const next_page_cursor = has_next_page
+    ? Buffer.from(
+        JSON.stringify({
+          device_id: next_device.device_id,
+          created_at: next_device.created_at,
+          query_hash,
+        }),
+        "utf8",
+      ).toString("base64")
+    : null
+
+  const next_page_url = getNextPageUrl(next_page_cursor, { req })
+
   res.status(200).json({
-    devices,
+    devices: page,
+    pagination: { has_next_page, next_page_cursor, next_page_url },
   })
 })
+
+const getNextPageUrl = (
+  next_page_cursor: string | null,
+  {
+    req,
+  }: {
+    req: {
+      url?: string
+      commonParams: Record<string, unknown>
+      baseUrl: string | undefined
+    }
+  },
+): string | null => {
+  if (req.url == null || req.baseUrl == null) return null
+  if (next_page_cursor == null) return null
+  const { page_cursor, ...params } = req.commonParams
+  const query = serializeUrlSearchParams(params)
+  const url = new URL([req.baseUrl, req.url].join(""))
+  url.search = query
+  url.searchParams.set("next_page_cursor", next_page_cursor)
+  url.searchParams.sort()
+  return url.toString()
+}
+
+const getPageCursorQueryHash = (params: Record<string, unknown>): string => {
+  const query = serializeUrlSearchParams(params)
+  return createHash("sha256").update(query).digest("hex")
+}

src/route-types.ts

@@ -2283,6 +2283,8 @@ export type Routes = {
           )[]
         | undefined
       manufacturer?: string | undefined
+      limit?: number
+      page_cursor?: (string | undefined) | null
     }
     formData: {}
     jsonResponse: {
@@ -2512,6 +2514,11 @@ export type Routes = {
         can_remotely_unlock?: boolean | undefined
         can_program_online_access_codes?: boolean | undefined
       }[]
+      pagination: {
+        has_next_page: boolean
+        next_page_cursor: string | null
+        next_page_url: string | null
+      }
       ok: boolean
     }
   }
@@ -2612,6 +2619,8 @@ export type Routes = {
           )[]
         | undefined
       manufacturer?: string | undefined
+      limit?: number
+      page_cursor?: (string | undefined) | null
     }
     formData: {}
     jsonResponse: {
@@ -3691,6 +3700,8 @@ export type Routes = {
           )[]
         | undefined
       manufacturer?: string | undefined
+      limit?: number
+      page_cursor?: (string | undefined) | null
     }
     formData: {}
     jsonResponse: {

test/api/devices/list.test.ts

@@ -51,6 +51,179 @@ test("GET /devices/list with api key", async (t: ExecutionContext) => {
   }
 })
 
+test("GET /devices/list with limit", async (t: ExecutionContext) => {
+  const { axios, db } = await getTestServer(t, { seed: false })
+  const seed_result = seedDatabase(db)
+
+  axios.defaults.headers.common.Authorization = `Bearer ${seed_result.seam_apikey1_token}`
+
+  const {
+    data: { devices },
+  } = await axios.get("/devices/list", { params: { limit: 2 } })
+
+  t.is(devices.length, 2)
+})
+
+test("GET /devices/list with pages", async (t: ExecutionContext) => {
+  const { axios, db } = await getTestServer(t, { seed: false })
+  const seed_result = seedDatabase(db)
+
+  const params = { limit: 2 }
+
+  axios.defaults.headers.common.Authorization = `Bearer ${seed_result.seam_apikey1_token}`
+
+  const {
+    data: {
+      devices,
+      pagination: { has_next_page, next_page_cursor, next_page_url },
+    },
+  } = await axios.get("/devices/list")
+  t.false(has_next_page)
+  t.is(next_page_cursor, null)
+  t.is(next_page_url, null)
+  t.is(devices.length, 5)
+
+  const {
+    data: {
+      devices: page1,
+      pagination: {
+        has_next_page: has_page_2,
+        next_page_cursor: page2_cursor,
+        next_page_url: page2_url,
+      },
+    },
+  } = await axios.get("/devices/list", { params })
+
+  t.is(page1.length, 2)
+  t.true(has_page_2)
+  t.truthy(page2_cursor)
+
+  if (page2_url == null) {
+    t.fail("Null next_page_url")
+    return
+  }
+
+  const url = new URL(page2_url)
+  t.is(url.pathname, "/devices/list")
+  t.deepEqual(url.searchParams.getAll("limit"), ["2"])
+
+  t.deepEqual(page1, [devices[0], devices[1]])
+
+  const {
+    data: {
+      devices: page2,
+      pagination: { has_next_page: has_page_3, next_page_cursor: page3_cursor },
+    },
+  } = await axios.get("/devices/list", {
+    params: { ...params, page_cursor: page2_cursor },
+  })
+
+  t.is(page2.length, 2)
+  t.true(has_page_3)
+  t.truthy(page3_cursor)
+
+  t.deepEqual(page2, [devices[2], devices[3]])
+
+  const {
+    data: {
+      devices: page3,
+      pagination: { has_next_page: has_page_4, next_page_cursor: page4_cursor },
+    },
+  } = await axios.get("/devices/list", {
+    params: { ...params, page_cursor: page3_cursor },
+  })
+
+  t.is(page3.length, 1)
+  t.false(has_page_4)
+  t.is(page4_cursor, null)
+
+  t.deepEqual(page3, [devices[4]])
+})
+
+test("GET /devices/list validates query hash", async (t: ExecutionContext) => {
+  const { axios, db } = await getTestServer(t, { seed: false })
+  const seed_result = seedDatabase(db)
+
+  axios.defaults.headers.common.Authorization = `Bearer ${seed_result.seam_apikey1_token}`
+
+  const {
+    data: {
+      pagination: { has_next_page, next_page_cursor },
+    },
+  } = await axios.get("/devices/list", { params: { limit: 2 } })
+
+  t.true(has_next_page)
+
+  const err = await t.throwsAsync<SimpleAxiosError>(
+    async () =>
+      await axios.get("/devices/list", {
+        params: { limit: 3, page_cursor: next_page_cursor },
+      }),
+  )
+  t.is(err?.status, 400)
+  t.regex(
+    (err?.response?.error?.message as string) ?? "",
+    /parameters identical/,
+  )
+
+  const err_empty = await t.throwsAsync<SimpleAxiosError>(
+    async () =>
+      await axios.get("/devices/list", {
+        params: { page_cursor: next_page_cursor },
+      }),
+  )
+  t.is(err_empty?.status, 400)
+  t.regex(
+    (err_empty?.response?.error?.message as string) ?? "",
+    /parameters identical/,
+  )
+
+  const err_post = await t.throwsAsync<SimpleAxiosError>(
+    async () =>
+      await axios.post("/devices/list", {
+        limit: 3,
+        device_types: ["august_lock"],
+        page_cursor: next_page_cursor,
+      }),
+  )
+  t.is(err_post?.status, 400)
+  t.regex(
+    (err_post?.response?.error?.message as string) ?? "",
+    /parameters identical/,
+  )
+})
+
+test("GET /devices/list handles array params", async (t: ExecutionContext) => {
+  const { axios, db } = await getTestServer(t, { seed: false })
+  const seed_result = seedDatabase(db)
+
+  axios.defaults.headers.common.Authorization = `Bearer ${seed_result.seam_apikey1_token}`
+
+  const {
+    data: {
+      pagination: { has_next_page, next_page_cursor, next_page_url },
+    },
+  } = await axios.get("/devices/list", {
+    params: { limit: 1, device_types: ["august_lock", "schlage_lock"] },
+  })
+
+  t.true(has_next_page)
+  t.truthy(next_page_cursor)
+
+  if (next_page_url == null) {
+    t.fail("Null next_page_url")
+    return
+  }
+
+  const url = new URL(next_page_url)
+  t.is(url.pathname, "/devices/list")
+  t.deepEqual(url.searchParams.getAll("limit"), ["1"])
+  t.deepEqual(url.searchParams.getAll("device_types"), [
+    "august_lock",
+    "schlage_lock",
+  ])
+})
+
 test("GET /devices/list with filters", async (t: ExecutionContext) => {
   const { axios, db } = await getTestServer(t, { seed: false })
   const seed_result = seedDatabase(db)

test/array-parsing.test.ts

@@ -59,7 +59,8 @@ test.failing("index array param", async (t: ExecutionContext) => {
   t.is(devices.length, 2)
 })
 
-test("empty array param", async (t: ExecutionContext) => {
+// UPSTREAM: nextlove will parse device_ids= to [''] but it should parse to []
+test.failing("empty array param", async (t: ExecutionContext) => {
   const { axios, db } = await getTestServer(t, { seed: false })
   const seed_result = seedDatabase(db)