Migrate to CelerSearch

// Copies every index (settings and documents) from one Meilisearch to another.
// SOURCE_URL=… SOURCE_KEY=… TARGET_URL=… TARGET_KEY=… node migrate.mjs
const { SOURCE_URL, SOURCE_KEY, TARGET_URL, TARGET_KEY, EMBEDDER_API_KEY } = process.env
const BATCH_SIZE = Number(process.env.BATCH_SIZE || 1000)

if (!SOURCE_URL || !SOURCE_KEY || !TARGET_URL || !TARGET_KEY) {
  console.error('Set SOURCE_URL, SOURCE_KEY, TARGET_URL and TARGET_KEY.')
  process.exit(1)
}

async function api(base, key, method, path, body) {
  const res = await fetch(base.replace(/\/+$/, '') + path, {
    method,
    headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' },
    body: body === undefined ? undefined : JSON.stringify(body),
  })
  const text = await res.text()
  let data = null
  try { data = text ? JSON.parse(text) : null } catch { data = null }
  if (!res.ok) {
    const error = new Error(`${method} ${path} failed (${res.status}): ${data?.message ?? text}`)
    error.status = res.status
    error.code = data?.code ?? data?.error
    throw error
  }
  return data
}

const source = (method, path, body) => api(SOURCE_URL, SOURCE_KEY, method, path, body)
const target = (method, path, body) => api(TARGET_URL, TARGET_KEY, method, path, body)

async function waitForTask({ taskUid }) {
  for (;;) {
    const task = await target('GET', `/tasks/${taskUid}`)
    if (task.status === 'succeeded') return
    if (task.status === 'failed' || task.status === 'canceled') {
      throw new Error(`Task ${taskUid} ${task.status}: ${task.error?.message ?? ''}`)
    }
    await new Promise((resolve) => setTimeout(resolve, 500))
  }
}

async function listIndexes() {
  const indexes = []
  for (let offset = 0; ; offset += 100) {
    const page = await source('GET', `/indexes?offset=${offset}&limit=100`)
    indexes.push(...page.results)
    if (page.results.length === 0 || indexes.length >= page.total) return indexes
  }
}

// Copies settings and returns the embedders whose vectors live in the documents.
async function copySettings(uid) {
  const settings = await source('GET', `/indexes/${uid}/settings`)
  const userProvided = []

  for (const [name, embedder] of Object.entries(settings.embedders ?? {})) {
    if (embedder.source === 'userProvided') userProvided.push(name)
    // API keys come back partially hidden, so they can't be copied as-is.
    if (embedder.apiKey == null && embedder.source !== 'openAi') continue
    if (EMBEDDER_API_KEY) {
      embedder.apiKey = EMBEDDER_API_KEY
    } else {
      delete settings.embedders[name]
      console.warn(`  ! Skipped embedder "${name}": set EMBEDDER_API_KEY and re-run, or add it later.`)
    }
  }

  await waitForTask(await target('PATCH', `/indexes/${uid}/settings`, settings))
  return userProvided
}

async function copyDocuments(uid, userProvided) {
  for (let offset = 0; ; offset += BATCH_SIZE) {
    const page = await source('POST', `/indexes/${uid}/documents/fetch`, {
      offset,
      limit: BATCH_SIZE,
      ...(userProvided.length > 0 ? { retrieveVectors: true } : {}),
    })
    if (page.results.length === 0) return

    // Keep only user-provided vectors; other embedders re-embed on the target.
    const documents = page.results.map(({ _vectors, ...doc }) => {
      const vectors = Object.fromEntries(
        Object.entries(_vectors ?? {}).filter(([name]) => userProvided.includes(name)),
      )
      return Object.keys(vectors).length > 0 ? { ...doc, _vectors: vectors } : doc
    })

    await waitForTask(await target('POST', `/indexes/${uid}/documents`, documents))
    const copied = offset + page.results.length
    console.log(`  ${copied} / ${page.total} documents`)
    if (copied >= page.total) return
  }
}

async function main() {
  const stats = await source('GET', '/stats')
  const indexes = await listIndexes()
  const total = indexes.reduce((sum, { uid }) => sum + (stats.indexes[uid]?.numberOfDocuments ?? 0), 0)
  console.log(`${indexes.length} indexes, ${total} documents to copy`)

  for (const { uid, primaryKey } of indexes) {
    console.log(`→ ${uid}`)
    try {
      await target('GET', `/indexes/${uid}`)
    } catch (error) {
      if (error.status !== 404) throw error
      await waitForTask(await target('POST', '/indexes', { uid, ...(primaryKey ? { primaryKey } : {}) }))
    }
    const userProvided = await copySettings(uid)
    await copyDocuments(uid, userProvided)
  }

  console.log('Done.')
}

main().catch((error) => {
  console.error(error.message)
  if (error.code === 'document_quota_exceeded') {
    console.error('Your plan\'s document limit is reached. Move to a larger plan and re-run.')
  }
  process.exit(1)
})

SOURCE_URL=https://your-old-meilisearch \
SOURCE_KEY=your-source-key \
TARGET_URL=https://your-celersearch-endpoint \
TARGET_KEY=your-celersearch-api-key \
node migrate.mjs