Atom.

Full text search, again - Sat Nov 1111:51 AM

Full text search, again

I set up in-memory full text search (for the second time) on all my site’s mdx documents.

This leverages the Unified.js ecosystem for MDX parsing and Orama (formerly Lyra) for indexing and searching.

The indexing happens at build time, and the index is persisted as a JSON file. At runtime, the JSON file is loaded into memory and used inside an API route. CloudFront caching helps to reduce the compute load as well, and speed up response times of repeat requests.

Some code dumps

ETL code snippet.

    const result = await unified()
      .use(remarkParse)
      .use(remarkFrontmatter, ["yaml"])
      .use(remarkGfm)
      .use(remarkMdx)
      .use(function plugin() {
        return (tree, file) => {
          visit(tree, "yaml", (node) => {
            file.data.frontmatter = YAML.parse(node.value);
          });
          remove(tree, "mdxJsxFlowElement");
          remove(tree, "mdxFlowExpression");
          remove(tree, "yaml");
          remove(tree, "blockquote");
          remove(tree, "code");
        };
      })
      .use(remarkRehype)
      .use(retextStringify)
      .process(source);

Next.js GET /api/search route handler

import { NextResponse, NextRequest } from "next/server";

import { search, AnyOrama } from "@orama/orama";
import { restore } from "@orama/plugin-data-persistence";

import JSONIndex from "./index.json";

const JSONstring = JSON.stringify(JSONIndex);

let newInstance: AnyOrama;

export async function GET(request: NextRequest) {
  const searchParams = request.nextUrl.searchParams;
  const query = searchParams.get("q") || searchParams.get("query");

  if (!query) {
    return NextResponse.json({ error: "no query" }, { status: 400 });
  }

  newInstance ??= await restore("json", JSONstring);
  const res = await search(newInstance, {
    term: query,
  });

  res.hits.forEach((hit) => {
    hit.document.content = hit.document.content.slice(0, 100);
  });

  return NextResponse.json(res, {
    status: 200,

    // browser cache for 1 minute
    // CDN cache for 1 day
    // allow serving stale content for 30 days
    headers: {
      "Cache-Control":
        "public, s-maxage=86400, max-age=60, stale-while-revalidate=2592000",
    },
  });
}