knowledge-base-management

The Knowledge Base system allows users to store and manage files, documents, and other knowledge items that agents can reference during conversations.

Core Concepts

KnowledgeItem Interface

interface KnowledgeItem {
  id: string
  name: string
  type: 'file' | 'folder'
  fileType?: 'document' | 'image' | 'text'
  content?: string // Text content or base64 for files
  contentHash?: string // SHA-256 hash for deduplication
  mimeType?: string
  size?: number
  path: string // Virtual path in knowledge base
  parentId?: string // Parent folder ID
  lastModified: Date
  createdAt: Date
  tags?: string[]
  description?: string
  syncSource?: 'manual' | 'filesystem_api' | 'connector'
  fileSystemHandle?: string // For File System API sync
  watchId?: string
  lastSyncCheck?: Date
}

File Type Detection

Supported file types in src/lib/knowledge-utils.ts:

const TEXT_EXTENSIONS = [
  '.txt',
  '.md',
  '.js',
  '.ts',
  '.jsx',
  '.tsx',
  '.css',
  '.html',
  '.xml',
  '.csv',
  '.yaml',
  '.yml',
  '.log',
]

const IMAGE_EXTENSIONS = [
  '.jpg',
  '.jpeg',
  '.png',
  '.gif',
  '.bmp',
  '.webp',
  '.svg',
  '.ico',
  '.tiff',
  '.tif',
]

const DOCUMENT_EXTENSIONS = [
  '.pdf',
  '.doc',
  '.docx',
  '.xls',
  '.xlsx',
  '.ppt',
  '.pptx',
  '.rtf',
  '.epub',
]

function getFileType(
  filename: string,
): 'text' | 'image' | 'document' | 'unknown' {
  const ext = filename.toLowerCase().split('.').pop()
  if (TEXT_EXTENSIONS.includes(`.${ext}`)) return 'text'
  if (IMAGE_EXTENSIONS.includes(`.${ext}`)) return 'image'
  if (DOCUMENT_EXTENSIONS.includes(`.${ext}`)) return 'document'
  return 'unknown'
}

Database Operations

import { db } from '@/lib/db'

// Create knowledge item
async function createKnowledgeItem(
  item: Omit<KnowledgeItem, 'id'>,
): Promise<string> {
  const id = crypto.randomUUID()
  await db.knowledge.add({ ...item, id })
  return id
}

// Get items by path
async function getItemsByPath(path: string): Promise<KnowledgeItem[]> {
  return db.knowledge.where('path').startsWith(path).toArray()
}

// Get folder contents
async function getFolderContents(folderId?: string): Promise<KnowledgeItem[]> {
  if (folderId) {
    return db.knowledge.where('parentId').equals(folderId).toArray()
  }
  return db.knowledge.filter((item) => !item.parentId).toArray()
}

// Search by content hash (deduplication)
async function findDuplicate(
  contentHash: string,
): Promise<KnowledgeItem | undefined> {
  return db.knowledge.where('contentHash').equals(contentHash).first()
}

Content Hashing for Deduplication

async function generateContentHash(
  content: string | ArrayBuffer,
): Promise<string> {
  const data =
    typeof content === 'string'
      ? new TextEncoder().encode(content)
      : new Uint8Array(content)

  const hashBuffer = await crypto.subtle.digest('SHA-256', data)
  const hashArray = Array.from(new Uint8Array(hashBuffer))
  return hashArray.map((b) => b.toString(16).padStart(2, '0')).join('')
}

File Upload Handling

async function handleFileUpload(
  file: File,
  parentId?: string,
): Promise<KnowledgeItem> {
  const content = await readFileContent(file)
  const contentHash = await generateContentHash(content)

  // Check for duplicates
  const existing = await findDuplicate(contentHash)
  if (existing) {
    toast.info('File already exists in knowledge base')
    return existing
  }

  const item: KnowledgeItem = {
    id: crypto.randomUUID(),
    name: file.name,
    type: 'file',
    fileType: getFileType(file.name),
    content: typeof content === 'string' ? content : btoa(content),
    contentHash,
    mimeType: file.type,
    size: file.size,
    path: parentId ? `${getPath(parentId)}/${file.name}` : `/${file.name}`,
    parentId,
    lastModified: new Date(file.lastModified),
    createdAt: new Date(),
    syncSource: 'manual',
  }

  await db.knowledge.add(item)
  return item
}

async function readFileContent(file: File): Promise<string | ArrayBuffer> {
  return new Promise((resolve, reject) => {
    const reader = new FileReader()

    if (file.type.startsWith('text/') || getFileType(file.name) === 'text') {
      reader.onload = () => resolve(reader.result as string)
      reader.onerror = reject
      reader.readAsText(file)
    } else {
      reader.onload = () => resolve(reader.result as ArrayBuffer)
      reader.onerror = reject
      reader.readAsArrayBuffer(file)
    }
  })
}

Folder Watching (File System API)

import { db } from '@/lib/db'

async function watchFolder(handle: FileSystemDirectoryHandle): Promise<void> {
  // Request permission
  const permission = await handle.requestPermission({ mode: 'read' })
  if (permission !== 'granted') {
    throw new Error('Permission denied')
  }

  // Scan and sync folder
  await syncFolder(handle, undefined)

  // Store handle for persistence
  await db.knowledge.add({
    id: crypto.randomUUID(),
    name: handle.name,
    type: 'folder',
    path: `/${handle.name}`,
    createdAt: new Date(),
    lastModified: new Date(),
    syncSource: 'filesystem_api',
    fileSystemHandle: handle.name,
  })
}

async function syncFolder(
  handle: FileSystemDirectoryHandle,
  parentId?: string,
): Promise<void> {
  for await (const entry of handle.values()) {
    if (entry.kind === 'file') {
      const file = await entry.getFile()
      await handleFileUpload(file, parentId)
    } else if (entry.kind === 'directory') {
      // Create folder and recurse
      const folderId = await createFolder(entry.name, parentId)
      await syncFolder(entry, folderId)
    }
  }
}

Context Injection for Agents

When agents need knowledge context:

import { db } from '@/lib/db'

async function getRelevantKnowledge(
  query: string,
  agentId: string,
): Promise<KnowledgeItem[]> {
  // Get agent's assigned knowledge items
  const agentKnowledge = await getAgentKnowledge(agentId)

  // Search for relevant items
  const searchTerms = query.toLowerCase().split(' ')

  return agentKnowledge.filter((item) => {
    const searchText =
      `${item.name} ${item.description || ''} ${item.tags?.join(' ') || ''}`.toLowerCase()
    return searchTerms.some((term) => searchText.includes(term))
  })
}

function formatKnowledgeForContext(items: KnowledgeItem[]): string {
  return items
    .map((item) => {
      return `## ${item.name}
${item.description || ''}
${item.content || '[Binary content]'}
---`
    })
    .join('\n\n')
}

Document Processing

For processing different document types:

// src/lib/document-processor.ts
export async function processDocument(item: KnowledgeItem): Promise<string> {
  switch (item.fileType) {
    case 'text':
      return item.content || ''

    case 'document':
      return await extractTextFromDocument(item)

    case 'image':
      // Could use OCR or image description
      return `[Image: ${item.name}]`

    default:
      return `[Unsupported file type: ${item.mimeType}]`
  }
}

Component Integration

import { useCallback, useState } from 'react'
import { Button, Card, Progress } from '@heroui/react'
import { Icon } from '@/components/Icon'
import { toast } from '@/lib/toast'

function KnowledgeUploader() {
  const [isUploading, setIsUploading] = useState(false)
  const [progress, setProgress] = useState(0)

  const handleDrop = useCallback(async (e: React.DragEvent) => {
    e.preventDefault()
    const files = Array.from(e.dataTransfer.files)

    setIsUploading(true)
    setProgress(0)

    for (let i = 0; i < files.length; i++) {
      await handleFileUpload(files[i])
      setProgress(((i + 1) / files.length) * 100)
    }

    setIsUploading(false)
    toast.success(`Uploaded ${files.length} files`)
  }, [])

  return (
    <Card
      onDrop={handleDrop}
      onDragOver={(e) => e.preventDefault()}
      className="border-dashed border-2 p-8 text-center"
    >
      <Icon name="Upload" size={48} className="mx-auto mb-4" />
      <p>Drop files here to upload</p>
      {isUploading && <Progress value={progress} className="mt-4" />}
    </Card>
  )
}

Testing

import { describe, it, expect, beforeEach, vi } from 'vitest'
import { db } from '@/lib/db'

vi.mock('@/lib/db')

describe('Knowledge Base', () => {
  beforeEach(() => {
    vi.clearAllMocks()
  })

  it('should detect file types correctly', () => {
    expect(getFileType('document.md')).toBe('text')
    expect(getFileType('image.png')).toBe('image')
    expect(getFileType('file.pdf')).toBe('document')
    expect(getFileType('data.bin')).toBe('unknown')
  })

  it('should generate consistent content hashes', async () => {
    const content = 'test content'
    const hash1 = await generateContentHash(content)
    const hash2 = await generateContentHash(content)
    expect(hash1).toBe(hash2)
  })

  it('should detect duplicates', async () => {
    const hash = 'abc123'
    vi.mocked(db.knowledge.where).mockReturnValue({
      equals: vi.fn().mockReturnValue({
        first: vi.fn().mockResolvedValue({ id: 'existing' }),
      }),
    } as any)

    const result = await findDuplicate(hash)
    expect(result).toEqual({ id: 'existing' })
  })
})