Search Server Indexing

Copyexport interface SharedIndex {
  id: string;
  title: string;
  description?: string;
  breadcrumbs?: string[];
  tag?: string | string[];
  structuredData: StructuredData;
  url: string;
}
 
export interface SharedDocument {
  id: string;
  content: string;
  page_id: string;
  type: 'page' | 'heading' | 'text';
  breadcrumbs?: string[];
  tags: string[];
  url: string;
}

Copyexport async function buildIndexDefault(page: Page): Promise<SharedIndex> {
  let structuredData: StructuredData | undefined;
 
  if (page.data.structuredData) {
    structuredData =
      typeof page.data.structuredData === 'function'
        ? await page.data.structuredData()
        : page.data.structuredData;
  } else if ('load' in page.data && typeof page.data.load === 'function') {
    structuredData = (await page.data.load()).structuredData;
  }
 
  if (!structuredData)
    throw new Error(
      'Cannot find structured data from page, please define the page to index function.',
    );
 
  return {
    title: page.data.title ?? basename(page.path, extname(page.path)),
    description: page.data.description,
    url: page.url,
    id: page.url,
    structuredData,
  };
}

Copyfunction isBreadcrumbItem(item: unknown): item is string {
  return typeof item === 'string' && item.length > 0;
}
 
export function buildBreadcrumbs<C extends LoaderConfig>(
  source: LoaderOutput<C>,
  page: C['page'],
): string[] | undefined {
  const pageTree = source.getPageTree(page.locale);
  const path = findPath(pageTree.children, (node) => node.type === 'page' && node.url === page.url);
 
  if (path) {
    const breadcrumbs: string[] = [];
    path.pop();
 
    if (isBreadcrumbItem(pageTree.name)) {
      breadcrumbs.push(pageTree.name);
    }
 
    for (const segment of path) {
      if (!isBreadcrumbItem(segment.name)) continue;
      breadcrumbs.push(segment.name);
    }
 
    return breadcrumbs;
  }
}

Copyexport type SimpleDocument = TypedDocument<Orama<typeof simpleSchema>>;
export const simpleSchema = {
  url: 'string',
  title: 'string',
  breadcrumbs: 'string[]',
  description: 'string',
  content: 'string',
  keywords: 'string',
} as const;
 
export type AdvancedDocument = TypedDocument<Orama<typeof advancedSchema>>;
export const advancedSchema = {
  content: 'string',
  page_id: 'string',
  type: 'string',
  breadcrumbs: 'string[]',
  tags: 'enum[]',
  url: 'string',
  embeddings: 'vector[512]',
} as const;

Copyexport async function createDB({
  indexes,
  tokenizer,
  search: _,
  ...rest
}: AdvancedOptions): Promise<Orama<typeof advancedSchema>> {
  const items = typeof indexes === 'function' ? await indexes() : indexes;
 
  const db = create({
    schema: advancedSchema,
    ...rest,
    components: {
      ...rest.components,
      tokenizer: tokenizer ?? rest.components?.tokenizer,
    },
  }) as Orama<typeof advancedSchema>;
 
  const mapTo: PartialSchemaDeep<AdvancedDocument>[] = buildDocuments(items);
  await insertMultiple(db, mapTo);
  return db;
}

Copyexport async function createDBSimple({
  indexes,
  tokenizer,
  ...rest
}: SimpleOptions): Promise<Orama<typeof simpleSchema>> {
  const items = typeof indexes === 'function' ? await indexes() : indexes;
  const db = create({
    schema: simpleSchema,
    ...rest,
    components: {
      ...rest.components,
      tokenizer: tokenizer ?? rest.components?.tokenizer,
    },
  }) as Orama<typeof simpleSchema>;
 
  await insertMultiple(
    db,
    items.map((page) => ({
      title: page.title,
      description: page.description,
      breadcrumbs: page.breadcrumbs,
      url: page.url,
      content: page.content,
      keywords: page.keywords,
    })),
  );
 
  return db;
}

Copyfunction getTokenizer(locale: string): { language: string } | { tokenizer: Tokenizer } {
  return {
    language: Object.keys(STEMMERS).find((lang) => STEMMERS[lang] === locale) ?? locale,
  };
}

Copyexport function createDocument(options?: DocumentOptions<Doc>) {
  return new Search.Document<Doc>({
    tokenize: 'full',
    ...options,
    document: {
      id: 'id',
      index: ['content'],
      tag: ['tags'],
      store: true,
      ...options?.document,
    },
  });

Copyexport async function sync(client: Algoliasearch, options: SyncOptions): Promise<void> {
  const { indexName = 'document', documents } = options;
  await setIndexSettings(client, indexName);
  await updateDocuments(client, indexName, documents);
}

Copyexport async function setIndexSettings(client: Algoliasearch, indexName: string): Promise<void> {
  await client.setSettings({
    indexName,
    indexSettings: {
      attributeForDistinct: 'page_id',
      attributesToRetrieve: ['title', 'section', 'content', 'url', 'section_id', 'breadcrumbs'],
      searchableAttributes: ['title', 'section', 'content'],
      attributesToSnippet: [],
      attributesForFaceting: ['tag'],
    },
  });
}

Copyfunction toIndex(page: DocumentRecord): BaseIndex[] {
  let id = 0;
  const indexes: BaseIndex[] = [];
  const scannedHeadings = new Set<string>();
 
  function createIndex(
    section: string | undefined,
    sectionId: string | undefined,
    content: string,
  ): BaseIndex {
    return {
      objectID: `${page._id}-${(id++).toString()}`,
      breadcrumbs: page.breadcrumbs,
      title: page.title,
      url: page.url,
      page_id: page._id,
      tag: page.tag,
      section,
      section_id: sectionId,
      content,
      ...page.extra_data,
    };
  }
 
  if (page.description) indexes.push(createIndex(undefined, undefined, page.description));

Copyconst { headings, contents } = page.structured;
 
for (const p of contents) {
  const heading = p.heading ? headings.find((h) => p.heading === h.id) : null;
 
  const index = createIndex(heading?.content, heading?.id, p.content);
 
  if (heading && !scannedHeadings.has(heading.id)) {
    scannedHeadings.add(heading.id);
 
    indexes.push(createIndex(heading.content, heading.id, heading.content));
  }
 
  indexes.push(index);
}

Copy 
  return indexes;
}

Copyexport async function exportSearchIndexes() {
  return source.getPages().map((page) => {
    return {
      id: page.url,
      structured: page.data.structuredData,
      url: page.url,
      title: page.data.title,
      description: page.data.description,
    } satisfies OramaDocument;
  });
}

Copyfunction toIndex(page: OramaDocument): OramaIndex[] {
  let id = 0;
  const indexes: OramaIndex[] = [];
  const scannedHeadings = new Set<string>();
 
  function createIndex(
    section: string | undefined,
    sectionId: string | undefined,
    content: string,
  ): OramaIndex {
    return {
      id: `${page.id}-${(id++).toString()}`,
      title: page.title,
      url: page.url,
      page_id: page.id,
      tag: page.tag,
      section,
      section_id: sectionId,
      content,
      breadcrumbs: page.breadcrumbs,
      ...page.extra_data,
    };
  }
 
  if (page.description) indexes.push(createIndex(undefined, undefined, page.description));
 

Copypage.structured.contents.forEach((p) => {
  const heading = p.heading ? page.structured.headings.find((h) => p.heading === h.id) : null;
 
  const index = createIndex(heading?.content, heading?.id, p.content);
 
  if (heading && !scannedHeadings.has(heading.id)) {
    scannedHeadings.add(heading.id);
 
    indexes.push(createIndex(heading.content, heading.id, heading.content));
  }
 
  indexes.push(index);
});
 
return indexes;

Copyexport async function sync(orama: OramaCloud, options: SyncOptions): Promise<void> {
  const { autoDeploy = true } = options;
  const index = orama.index.set(options.index);
  await index.transaction.open();
 
  await index.transaction.insertDocuments(
    options.documents.flatMap(toIndex) as unknown as AnyObject[],
  );
 
  if (autoDeploy) await index.transaction.commit();
}

Copyexport async function syncI18n(orama: OramaCloud, options: I18nSyncOptions): Promise<void> {
  const { autoDeploy = true, indexes } = options;
 
  const tasks = options.documents.map(async (document) => {
    const index = orama.index.set(indexes[document.locale]);
    await index.transaction.open();
 
    await index.transaction.insertDocuments(
      document.items.flatMap(toIndex) as unknown as AnyObject[],
    );
 
    if (autoDeploy) await index.transaction.commit();
  });
 
  await Promise.all(tasks);
}

Copyexport interface MixedbreadSearchOptions {
  /**
   * The Mixedbread SDK client instance
   */
  client: Mixedbread;
 
  /**
   * The identifier of the store to search in
   */
  storeIdentifier: string;
 
  /**
   * Maximum number of results to return
   *
   * @defaultValue 10
   */
  topK?: number;
 
  /**
   * Re-rank search results for improved relevance (this adds latency to the search)
   */
  rerank?: boolean;
 
  /**
   * Rewrite the query for better search results (this adds latency to the search)
   */
  rewriteQuery?: boolean;
 
  /**
   * Minimum score threshold for results
   */
  scoreThreshold?: number;
 
  /**
   * Custom transform function for search results
   */
  transform?: (results: StoreSearchResult[], query: string) => SortedResult[];
}

Copyexport function createMixedbreadSearchAPI(options: MixedbreadSearchOptions): SearchAPI {
  const {
    client,
    storeIdentifier,
    topK = 10,
    rerank,
    rewriteQuery,
    scoreThreshold,
    transform,
  } = options;
 

Copyfunction defaultTransform(results: StoreSearchResult[]): SortedResult[] {
  return results.flatMap((item) => {
    const metadata = item.generated_metadata;
 
    const url = metadata.url || '#';
    const title = metadata.title || 'Untitled';
 
    const chunkResults: SortedResult[] = [
      {
        id: `${item.file_id}-${item.chunk_index}-page`,
        type: 'page',
        content: title,
        url,
      },
    ];
 
    const headingTitle = item.type === 'text' && item.text ? extractHeadingTitle(item.text) : '';
 
    if (headingTitle) {
      slugger.reset();
 
      chunkResults.push({
        id: `${item.file_id}-${item.chunk_index}-heading`,
        type: 'heading',
        content: headingTitle,
        url: `${url}#${slugger.slug(headingTitle)}`,
      });
    }
 
    return chunkResults;
  });
}

Copyconst { tag, limit } = searchOptions;
let filters: StoreSearchParams['filters'] | undefined;
if (Array.isArray(tag) && tag.length > 0) {
  filters = {
    key: 'generated_metadata.tag',
    operator: 'in',
    value: tag,
  };
} else if (typeof tag === 'string') {
  filters = {
    key: 'generated_metadata.tag',
    operator: 'eq',
    value: tag,
  };
}