PDF 解析

pnpm add pdfjs-dist

import * as pdfjsLib from 'pdfjs-dist'
 
// 禁用 worker，让解析在主线程执行
pdfjsLib.GlobalWorkerOptions.workerSrc = ''

import * as pdfjsLib from 'pdfjs-dist'
 
pdfjsLib.GlobalWorkerOptions.workerSrc = ''
 
/**
 * 从 PDF 二进制数据中按页提取文本
 */
async function extractTextByPage(
  pdfData: ArrayBuffer
): Promise<{ page: number; text: string }[]> {
  const loadingTask = pdfjsLib.getDocument({ data: new Uint8Array(pdfData) })
  const pdf = await loadingTask.promise
 
  const results: { page: number; text: string }[] = []
 
  for (let i = 1; i <= pdf.numPages; i++) {
    const page = await pdf.getPage(i)
    const textContent = await page.getTextContent()
 
    // textContent.items 是每个文字片段的数组
    // 需要把它们的 str 字段拼起来
    const pageText = textContent.items
      .filter((item): item is { str: string; dir: string; width: number; height: number; transform: number[]; fontName: string; hasEOL: boolean } => 'str' in item)
      .map((item) => item.str)
      .join('')
 
    results.push({ page: i, text: pageText })
  }
 
  return results
}

async function extractFullText(pdfData: ArrayBuffer): Promise<string> {
  const pages = await extractTextByPage(pdfData)
  return pages.map((p) => p.text).join('\n\n')
}

async function extractMetadata(
  pdfData: ArrayBuffer
): Promise<{
  numPages: number
  title: string | undefined
  author: string | undefined
  subject: string | undefined
  creator: string | undefined
  producer: string | undefined
  creationDate: Date | undefined
  modificationDate: Date | undefined
}> {
  const loadingTask = pdfjsLib.getDocument({ data: new Uint8Array(pdfData) })
  const pdf = await loadingTask.promise
  const metadata = await pdf.getMetadata()
 
  const info = metadata.info as Record<string, unknown>
 
  return {
    numPages: pdf.numPages,
    title: info.Title as string | undefined,
    author: info.Author as string | undefined,
    subject: info.Subject as string | undefined,
    creator: info.Creator as string | undefined,
    producer: info.Producer as string | undefined,
    creationDate: info.CreationDate
      ? new Date(info.CreationDate as string)
      : undefined,
    modificationDate: info.ModDate
      ? new Date(info.ModDate as string)
      : undefined,
  }
}

function isLikelyScanned(
  pages: { page: number; text: string }[],
  threshold = 50
): boolean {
  // 超过一半的页面文本少于 threshold 个字符，大概率是扫描版
  const emptyPages = pages.filter((p) => p.text.trim().length < threshold)
  return emptyPages.length > pages.length / 2
}

function cleanText(raw: string): string {
  let text = raw
 
  // 1. 去除零宽字符和不可见控制字符（保留换行和制表符）
  text = text.replace(/[​-‍]/g, '')
  text = text.replace(/[^\S\n\t]+/g, (match) => {
    // 把连续空白压缩成一个空格，但保留换行
    return match.includes('\n') ? '\n' : ' '
  })
 
  // 2. 把只有 1-2 个字的行（可能是页码）去掉
  text = text
    .split('\n')
    .filter((line) => {
      const trimmed = line.trim()
      if (trimmed.length === 0) return false
      // 过滤纯数字行（页码）
      if (/^\d{1,4}$/.test(trimmed)) return false
      return true
    })
    .join('\n')
 
  // 3. 合并被截断的段落
  // 规则：如果一行不以句号、问号、叹号、冒号结尾，
  // 且下一行以小写字母或中文开头，则合并
  text = text.replace(/([^\n。？！：\?\!])\n([a-z一-鿿])/g, '$1$2')
 
  // 4. 压缩连续空行
  text = text.replace(/\n{3,}/g, '\n\n')
 
  return text.trim()
}

interface Chunk {
  text: string
  index: number
  startChar: number
  endChar: number
}
 
function splitIntoChunks(
  text: string,
  chunkSize = 500,
  overlap = 50
): Chunk[] {
  const chunks: Chunk[] = []
  let start = 0
 
  while (start < text.length) {
    let end = start + chunkSize
 
    // 尝试在句号或换行处断开，避免切断句子
    if (end < text.length) {
      const segment = text.slice(start, end)
      const lastBreak = Math.max(
        segment.lastIndexOf('。'),
        segment.lastIndexOf('\n'),
        segment.lastIndexOf('. ')
      )
      // 如果在后 30% 的位置找到了断点，就在那里切
      if (lastBreak > chunkSize * 0.7) {
        end = start + lastBreak + 1
      }
    }
 
    chunks.push({
      text: text.slice(start, end).trim(),
      index: chunks.length,
      startChar: start,
      endChar: end,
    })
 
    start = end - overlap
  }
 
  return chunks
}

interface DocumentChunk {
  text: string
  metadata: {
    source: string       // 文件名
    page?: number        // 来自第几页
    chunkIndex: number   // 在文档内的块序号
    totalChunks: number  // 总块数
    author?: string      // PDF 元数据中的作者
    createdAt?: string   // PDF 元数据中的创建时间
  }
}

import { Hono } from 'hono'
import * as pdfjsLib from 'pdfjs-dist'
 
pdfjsLib.GlobalWorkerOptions.workerSrc = ''
 
type Bindings = {
  BUCKET: R2Bucket
}
 
const app = new Hono<{ Bindings: Bindings }>()
 
app.post('/api/documents/parse', async (c) => {
  // 1. 读取上传的文件
  const body = await c.req.parseBody()
  const file = body['file']
 
  if (!file || !(file instanceof File)) {
    return c.json({ error: '请上传一个 PDF 文件' }, 400)
  }
 
  // 2. 校验文件类型
  if (file.type !== 'application/pdf') {
    return c.json({ error: '仅支持 PDF 格式' }, 400)
  }
 
  // 3. 读取二进制数据
  const pdfData = await file.arrayBuffer()
 
  // 4. 检查文件大小（Workers 内存限制）
  if (pdfData.byteLength > 50 * 1024 * 1024) {
    return c.json({ error: '文件大小不能超过 50MB' }, 413)
  }
 
  // 5. 提取文本（逻辑同第 4 节）
  const loadingTask = pdfjsLib.getDocument({
    data: new Uint8Array(pdfData),
  })
  const pdf = await loadingTask.promise
 
  const pages: { page: number; text: string }[] = []
  for (let i = 1; i <= pdf.numPages; i++) {
    const page = await pdf.getPage(i)
    const textContent = await page.getTextContent()
    const pageText = textContent.items
      .filter(
        (item): item is { str: string; dir: string; width: number; height: number; transform: number[]; fontName: string; hasEOL: boolean } =>
          'str' in item
      )
      .map((item) => item.str)
      .join('')
    pages.push({ page: i, text: pageText })
  }
 
  // 6. 检测是否为扫描版（逻辑同第 6 节）
  const emptyPages = pages.filter((p) => p.text.trim().length < 50)
  if (emptyPages.length > pages.length / 2) {
    return c.json({
      isScanned: true,
      message: '该 PDF 疑似为扫描版，需要 OCR 处理',
      numPages: pdf.numPages,
    })
  }
 
  // 7. 提取元数据
  const metadata = await pdf.getMetadata()
  const info = metadata.info as Record<string, unknown>
 
  // 8. 合并全文 → 清洗 → 分块（函数定义见第 7、8 节）
  const rawText = pages.map((p) => p.text).join('\n\n')
  const cleanedText = cleanText(rawText)
  const chunks = splitIntoChunks(cleanedText, 500, 50).map(
    (chunk, index) => ({ text: chunk.text, index })
  )
 
  // 9. 返回结构化结果
  return c.json({
    metadata: {
      numPages: pdf.numPages,
      title: info.Title as string | undefined,
      author: info.Author as string | undefined,
      creationDate: info.CreationDate
        ? new Date(info.CreationDate as string).toISOString()
        : undefined,
    },
    isScanned: false,
    chunks,
    fullText: cleanedText,
  })
})
 
export default app