User Guide · 用户指南

# 安装并启动（使用 uvx 从 GitHub 安装）
uvx --with git+https://github.com/ThreeFish-AI/data-negentropy.perceives.git@v0.2.0a1 negentropy-perceives

from negentropy.perceives.sdk import NegentropyPerceivesClient

async with NegentropyPerceivesClient("http://localhost:2992/mcp") as client:
    # 网页 → Markdown
    result = await client.parse_webpage_to_markdown(
        url="https://example.com",
        method="auto",
    )
    print(result)

async with NegentropyPerceivesClient("http://localhost:2992/mcp") as client:
    # PDF → Markdown（自动选择最佳引擎）
    result = await client.call_tool(
        "parse_pdf_to_markdown",
        {"pdf_source": "https://example.com/paper.pdf", "method": "auto"},
    )
    print(result)

# 启动 MCP 服务器（主要启动命令）
uv run negentropy-perceives

# 以 Python 模块方式运行
uv run python -m negentropy.perceives

# 启用调试级别日志
uv run --env NEGENTROPY_PERCEIVES_LOG_LEVEL=DEBUG negentropy-perceives

# 启用完整功能特性的开发配置
uv run --env NEGENTROPY_PERCEIVES_ENABLE_JAVASCRIPT=true \
          --env NEGENTROPY_PERCEIVES_USE_RANDOM_USER_AGENT=true \
          negentropy-perceives

# 查看 Ruff 所有可用的检查规则
uv run ruff rule --all

# MyPy 显示详细的错误代码信息
uv run mypy src/negentropy/perceives/ --show-error-codes

# MyPy 生成 HTML 格式的类型检查报告
uv run mypy src/negentropy/perceives/ --html-report mypy-report

# 添加生产/开发依赖
uv add <package-name>
uv add --dev <package-name>

# 移除/更新依赖
uv remove <package-name>
uv lock --upgrade

# 查看依赖信息
uv tree              # 完整依赖树
uv tree --outdated   # 过时依赖
uv list              # 已安装包列表

# 版本号查询（从 pyproject.toml 动态读取）
uv run python -c "from negentropy.perceives import __version__; print(__version__)"

# 缓存清理
uv cache clean

# Python 编译缓存清理
find . -type d -name "__pycache__" -exec rm -rf {} + 2>/dev/null || true

# 检查环境变量
printenv | grep NEGENTROPY_PERCEIVES

# 验证配置正确性
uv run python -c "from negentropy.perceives.config import settings; print(settings.model_dump())"

# 检查模块导入
uv run python -c "import negentropy.perceives; print('Import successful')"

# 查看可用工具列表
uv run python -c "import asyncio; from negentropy.perceives.tools import app; print([t.name for t in asyncio.run(app.list_tools())])"

# 检查配置来源
uv run python -c "from negentropy.perceives.config import describe_config_sources; print(describe_config_sources())"

# 生成默认用户配置
uv run negentropy-perceives --init-config

{
  "mcpServers": {
    "negentropy-perceives": {
      "command": "uv",
      "args": [
        "run",
        "--with",
        "git+https://github.com/ThreeFish-AI/data-negentropy.perceives.git@v0.2.0a1",
        "negentropy-perceives"
      ]
    }
  }
}

NEGENTROPY_PERCEIVES_TRANSPORT_MODE=http \
NEGENTROPY_PERCEIVES_HTTP_HOST=0.0.0.0 \
NEGENTROPY_PERCEIVES_HTTP_PORT=2992 \
negentropy-perceives

# ~/.negentropy/perceives.config.yaml（生产环境）
transport:
  mode: http
http:
  host: "0.0.0.0"
  port: 2992
log:
  level: INFO
enable_javascript: true
browser:
  headless: true
concurrent_requests: 32
use_random_user_agent: true

{
  "mcpServers": {
    "negentropy-perceives": {
      "url": "http://localhost:2992/mcp",
      "transport": "http"
    }
  }
}

from negentropy.perceives.sdk import NegentropyPerceivesClient

async with NegentropyPerceivesClient("http://localhost:2992/mcp") as client:
    result = await client.parse_webpage_to_markdown(
        url="https://example.com",
        method="auto",
    )

NEGENTROPY_PERCEIVES_TRANSPORT_MODE=sse \
NEGENTROPY_PERCEIVES_HTTP_PORT=2992 \
negentropy-perceives

{
  "mcpServers": {
    "negentropy-perceives": {
      "url": "http://localhost:2992/mcp",
      "transport": "sse"
    }
  }
}

{
  "mcpServers": {
    "negentropy-perceives": {
      "command": "uv",
      "args": [
        "run",
        "--with",
        "git+https://github.com/ThreeFish-AI/data-negentropy.perceives.git@v0.2.0a1",
        "negentropy-perceives"
      ],
      "env": {
        "NEGENTROPY_PERCEIVES_ENABLE_JAVASCRIPT": "true",
        "NEGENTROPY_PERCEIVES_CONCURRENT_REQUESTS": "8"
      }
    }
  }
}

NEGENTROPY_PERCEIVES_HTTP_CORS_ORIGINS=https://app.example.com,https://admin.example.com

from negentropy.perceives.sdk import NegentropyPerceivesClient

async with NegentropyPerceivesClient(
    "https://api.example.com/mcp",
    headers={
        "Authorization": "Bearer your-jwt-token",
        "X-API-Key": "your-api-key"
    },
) as client:
    tools = await client.list_tools()

{
  "url": "https://example.com",
  "filter_domains": ["example.com", "blog.example.com"],
  "exclude_domains": ["ads.example.com"],
  "internal_only": false
}

{
  "success": true,
  "url": "https://example.com",
  "total_links": 45,
  "internal_links_count": 32,
  "external_links_count": 13,
  "links": [
    {"url": "https://example.com/about", "text": "关于我们", "is_internal": true},
    {"url": "https://partner.com", "text": "合作伙伴", "is_internal": false}
  ]
}

{
  "success": true,
  "url": "https://example.com",
  "title": "Example Website",
  "description": "This is an example website",
  "status_code": 200,
  "content_type": "text/html",
  "content_length": 15420
}

{
  "url": "https://example.com/article",
  "method": "auto",
  "extract_main_content": true,
  "formatting_options": {
    "table_alignment": true,
    "code_detection": true
  }
}

{
  "success": true,
  "url": "https://example.com/article",
  "method": "pipeline_auto",
  "markdown_content": "# 文章标题\n\n这是文章的主要内容...",
  "metadata": {
    "title": "文章标题",
    "description": "文章描述",
    "word_count": 1250
  },
  "word_count": 1250,
  "images_embedded": 0,
  "conversion_time": 1.23
}

{
  "urls": [
    "https://blog.example.com/post1",
    "https://blog.example.com/post2",
    "https://blog.example.com/post3"
  ],
  "method": "auto",
  "extract_main_content": true
}

{
  "success": true,
  "total_urls": 3,
  "successful_count": 3,
  "failed_count": 0,
  "results": [
    {"success": true, "url": "https://blog.example.com/post1", "markdown_content": "...", "word_count": 800},
    {"success": true, "url": "https://blog.example.com/post2", "markdown_content": "...", "word_count": 1200},
    {"success": true, "url": "https://blog.example.com/post3", "markdown_content": "...", "word_count": 950}
  ],
  "total_word_count": 2950,
  "total_conversion_time": 3.45
}

{
  "pdf_source": "https://example.com/document.pdf",
  "method": "auto",
  "page_range": [0, 10]
}

{
  "pdf_source": "https://example.com/document.pdf",
  "method": "docling",
  "extract_images": true,
  "extract_tables": true,
  "extract_formulas": true,
  "embed_images": false,
  "enhanced_options": {
    "output_dir": "./extracted_assets",
    "image_size": [800, 600]
  }
}

{
  "success": true,
  "pdf_source": "https://example.com/document.pdf",
  "method": "pipeline_auto",
  "output_format": "markdown",
  "content": "# 文档标题\n\n转换后的 Markdown 内容...",
  "metadata": {
    "title": "文档标题",
    "author": "作者姓名",
    "total_pages": 50,
    "pages_processed": 10
  },
  "page_count": 10,
  "word_count": 2500,
  "conversion_time": 5.67,
  "enhanced_assets": {
    "images_extracted": 3,
    "tables_extracted": 2,
    "formulas_extracted": 5,
    "code_blocks_detected": 1,
    "engines_used": ["docling"]
  }
}

[PDF engines] docling=ok(docling), mineru=missing, marker=ok(marker_pdf), pymupdf=ok(fitz)
[PDF engines] 部分引擎未安装：mineru。参考 docs/development.md 的 all-engines extras 指引补齐依赖。

pipeline:
  pdf:
    stages:
      - name: layout_analysis
        competition_mode: true # 启用 4 引擎竞争 + LLM 评审择优
      - name: table_extraction
        competition_mode: true

{
  "pdf_sources": [
    "https://example.com/doc1.pdf",
    "/local/path/doc2.pdf",
    "https://example.com/doc3.pdf"
  ],
  "method": "auto",
  "extract_images": true,
  "extract_tables": true,
  "extract_formulas": true
}

{
  "success": true,
  "total_pdfs": 3,
  "successful_count": 2,
  "failed_count": 1,
  "results": [
    {"success": true, "pdf_source": "https://example.com/doc1.pdf", "content": "...", "page_count": 10, "word_count": 1500},
    {"success": true, "pdf_source": "/local/path/doc2.pdf", "content": "...", "page_count": 25, "word_count": 5000}
  ],
  "total_pages": 35,
  "total_word_count": 6500,
  "total_conversion_time": 12.34
}

from negentropy.perceives.sdk import NegentropyPerceivesClient

# 基础连接（默认 http://localhost:2992/mcp）
async with NegentropyPerceivesClient() as client:
    tools = await client.list_tools()
    print([tool.name for tool in tools])

# 自定义端点 + 认证头
async with NegentropyPerceivesClient(
    "http://api.example.com:8081/mcp",
    headers={"X-API-Key": "your-key"},
) as client:
    result = await client.call_tool("inspect_page", {"url": "https://example.com"})

# call_tool() 可以调用任意 MCP 工具
result = await client.call_tool(
    "discover_links",
    {"url": "https://example.com", "internal_only": True},
)

# 带超时控制
result = await client.call_tool(
    "parse_pdf_to_markdown",
    {"pdf_source": "paper.pdf", "method": "auto"},
    timeout=60.0,
)

# 网页转 Markdown（便捷封装）
result = await client.parse_webpage_to_markdown(
    url="https://example.com/docs",
    method="auto",
    extract_main_content=True,
)

from negentropy.perceives.scraping import WebScraper
from negentropy.perceives.markdown.converter import MarkdownConverter

# 网页抓取
scraper = WebScraper()
scrape_result = await scraper.scrape_url("https://example.com", method="simple")

# Markdown 转换
converter = MarkdownConverter()
md_result = converter.parse_webpage_to_markdown(
    scrape_result=scrape_result,
    extract_main_content=True,
)

from negentropy.perceives.sdk import NegentropyPerceivesClient

async def archive_website(base_url: str, page_urls: list[str]):
    async with NegentropyPerceivesClient() as client:
        # 批量转换
        result = await client.call_tool(
            "parse_webpages_to_markdown",
            {
                "urls": page_urls,
                "method": "auto",
                "extract_main_content": True,
                "include_metadata": True,
            },
        )
        return result

async def process_papers(pdf_paths: list[str]):
    async with NegentropyPerceivesClient() as client:
        result = await client.call_tool(
            "parse_pdfs_to_markdown",
            {
                "pdf_sources": pdf_paths,
                "method": "auto",
                "extract_images": True,
                "extract_tables": True,
                "extract_formulas": True,
                "output_format": "markdown",
            },
        )
        return result

async def build_link_map(target_url: str):
    async with NegentropyPerceivesClient() as client:
        # 获取所有内部链接
        result = await client.call_tool(
            "discover_links",
            {"url": target_url, "internal_only": True},
        )
        return result

async def smart_pdf_processing():
    async with NegentropyPerceivesClient() as client:
        result = await client.call_tool(
            "parse_pdf_to_markdown",
            {
                "pdf_source": "complex_paper.pdf",
                "method": "smart",
                "extract_formulas": True,
                "extract_images": True,
                "extract_tables": True,
            },
        )
        return result

async def convert_technical_manual():
    async with NegentropyPerceivesClient() as client:
        result = await client.call_tool(
            "parse_pdf_to_markdown",
            {
                "pdf_source": "technical_manual.pdf",
                "method": "docling",
                "extract_images": True,
                "extract_tables": True,
                "embed_images": True,
                "enhanced_options": {
                    "output_dir": "./extracted_assets",
                    "image_size": [1200, 900],
                },
            },
        )
        return result

# Apple Silicon (MPS) — 自动检测，无需配置
NEGENTROPY_PERCEIVES_DOCLING_ENABLED=true negentropy-perceives

# NVIDIA GPU (CUDA) — 自动检测，无需配置
NEGENTROPY_PERCEIVES_DOCLING_ENABLED=true negentropy-perceives

# 显式指定设备
NEGENTROPY_PERCEIVES_ACCELERATOR_DEVICE=cuda NEGENTROPY_PERCEIVES_DOCLING_ENABLED=true negentropy-perceives

# 增加超时时间
NEGENTROPY_PERCEIVES_REQUEST_TIMEOUT=60 negentropy-perceives

# 客户端带超时调用
result = await client.call_tool("parse_webpage_to_markdown", {"url": "..."}, timeout=30.0)

# 启用 JavaScript 支持
NEGENTROPY_PERCEIVES_ENABLE_JAVASCRIPT=true negentropy-perceives

{
  "url": "https://example.com",
  "method": "selenium",
  "wait_for_element": ".dynamic-content"
}

# 检查端口占用
netstat -tlnp | grep 2992
# 更换端口
NEGENTROPY_PERCEIVES_HTTP_PORT=8082 negentropy-perceives

# 检查当前入口
which negentropy-perceives
# 检查最终配置
uv run python -c "from negentropy.perceives.config import settings; print(settings.model_dump())"
# 检查用户配置文件
cat ~/.negentropy/perceives.config.yaml

NEGENTROPY_PERCEIVES_HTTP_CORS_ORIGINS="http://localhost:3000,https://yourdomain.com" negentropy-perceives

# 必须同时启用代理开关和配置代理 URL
NEGENTROPY_PERCEIVES_USE_PROXY=true \
NEGENTROPY_PERCEIVES_PROXY_URL=http://proxy.example.com:8080 \
negentropy-perceives

# 禁用缓存或缩短 TTL
NEGENTROPY_PERCEIVES_ENABLE_CACHING=false negentropy-perceives
# 或调整缓存生存时间
NEGENTROPY_PERCEIVES_CACHE_TTL_HOURS=1 negentropy-perceives

# 显式指定设备
NEGENTROPY_PERCEIVES_ACCELERATOR_DEVICE=cuda negentropy-perceives  # NVIDIA GPU
NEGENTROPY_PERCEIVES_ACCELERATOR_DEVICE=mps negentropy-perceives   # Apple Silicon