在昇腾310p服务器上运行PP-OCRv4，拉起后发生请求报错

[Jack941-gu]

我们在华为 Atlas 300I Duo（310P，4卡8芯）服务器上部署独立的 PP-Structure 合同结构化解析服务（FastAPI + uvicorn）。
使用官方提供的 NPU 每日构建 Docker 镜像，通过 NPU 硬件加速运行离线 V4 Server 结构化模型，在推理含有表格和版面分析的合同图片时，触发了 ACL error: 500001 底层编译报错。

报错日志：
{
"status": "error",
"message": "In user code:\n\n File "tools/export_model.py", line 111, in \n main()\n File "tools/export_model.py", line 107, in main\n run(FLAGS, cfg)\n File "tools/export_model.py", line 73, in run\n trainer.export(FLAGS.output_dir)\n File "/anPaddle/det/PaddleDetection/ppdet/engine/trainer.py", line 776, in export\n save_dir)\n File "/anPaddle/det/PaddleDetection/ppdet/engine/trainer.py", line 753, in _get_infer_cfg_and_input_spec\n input_spec, static_model.forward.main_program,\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/dygraph_to_static/program_translator.py", line 563, in main_program\n concrete_program = self.concrete_program\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/dygraph_to_static/program_translator.py", line 479, in concrete_program\n return self.concrete_program_specify_input_spec(input_spec=None)\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/dygraph_to_static/program_translator.py", line 517, in concrete_program_specify_input_spec\n *desired_input_spec)\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/dygraph_to_static/program_translator.py", line 427, in get_concrete_program\n concrete_program, partial_program_layer = self._program_cache[cache_key]\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/dygraph_to_static/program_translator.py", line 723, in getitem\n self._caches[item] = self._build_once(item)\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/dygraph_to_static/program_translator.py", line 714, in _build_once\n **cache_key.kwargs)\n File "", line 2, in from_func_spec\n \n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/wrapped_decorator.py", line 25, in impl\n return wrapped_func(*args, **kwargs)\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/base.py", line 51, in impl\n return func(*args, **kwargs)\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/dygraph_to_static/program_translator.py", line 662, in from_func_spec\n outputs = static_func(*inputs)\n File "/tmp/tmppiay90cr.py", line 100, in forward\n false_fn_5, (), (inputs, self), (out,))\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/dygraph_to_static/convert_operators.py", line 211, in convert_ifelse\n out = _run_py_ifelse(pred, true_fn, false_fn, true_args, false_args)\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/dygraph_to_static/convert_operators.py", line 257, in _run_py_ifelse\n return true_fn(*true_args) if pred else false_fn(*false_args)\n File "/tmp/tmppiay90cr.py", line 83, in false_fn_5\n for_loop_body_0, [inputs_list, __for_loop_var_index_0])\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/dygraph_to_static/convert_operators.py", line 45, in convert_while_loop\n loop_vars = _run_py_while(cond, body, loop_vars)\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/dygraph_to_static/convert_operators.py", line 59, in _run_py_while\n loop_vars = body(*loop_vars)\n File "/tmp/tmppiay90cr.py", line 78, in for_loop_body_0\n dy2static.convert_call(self.get_pred)())\n File "/tmp/tmp321m2tiu.py", line 45, in get_pred\n __return_value_0,))\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/dygraph_to_static/convert_operators.py", line 211, in convert_ifelse\n out = _run_py_ifelse(pred, true_fn, false_fn, true_args, false_args)\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/dygraph_to_static/convert_operators.py", line 257, in _run_py_ifelse\n return true_fn(*true_args) if pred else false_fn(*false_args)\n File "/anPaddle/det/PaddleDetection/ppdet/modeling/architectures/picodet.py", line 87, in get_pred\n return {'picodet': self._forward()[0]}\n File "/anPaddle/det/PaddleDetection/ppdet/modeling/architectures/picodet.py", line 64, in _forward\n body_feats = self.backbone(self.inputs)\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/layers.py", line 917, in call\n return self._dygraph_call_func(*inputs, **kwargs)\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/layers.py", line 907, in _dygraph_call_func\n outputs = self.forward(*inputs, **kwargs)\n File "/anPaddle/det/PaddleDetection/ppdet/modeling/backbones/lcnet.py", line 251, in forward\n x = self.blocks6(x)\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/layers.py", line 917, in call\n return self._dygraph_call_func(*inputs, **kwargs)\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/layers.py", line 907, in _dygraph_call_func\n outputs = self.forward(*inputs, **kwargs)\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/container.py", line 98, in forward\n input = layer(input)\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/layers.py", line 917, in call\n return self._dygraph_call_func(*inputs, **kwargs)\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/layers.py", line 907, in _dygraph_call_func\n outputs = self.forward(*inputs, **kwargs)\n File "/tmp/tmp9wlji__q.py", line 15, in forward\n false_fn_9, (x,), (x,), (x,))\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/dygraph_to_static/convert_operators.py", line 211, in convert_ifelse\n out = _run_py_ifelse(pred, true_fn, false_fn, true_args, false_args)\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/dygraph_to_static/convert_operators.py", line 257, in _run_py_ifelse\n return true_fn(*true_args) if pred else false_fn(*false_args)\n File "/anPaddle/det/PaddleDetection/ppdet/modeling/backbones/lcnet.py", line 123, in forward\n x = self.se(x)\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/layers.py", line 917, in call\n return self._dygraph_call_func(*inputs, **kwargs)\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/layers.py", line 907, in _dygraph_call_func\n outputs = self.forward(*inputs, **kwargs)\n File "/anPaddle/det/PaddleDetection/ppdet/modeling/backbones/lcnet.py", line 150, in forward\n x = self.conv1(x)\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/layers.py", line 917, in call\n return self._dygraph_call_func(*inputs, **kwargs)\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/dygraph/layers.py", line 907, in _dygraph_call_func\n outputs = self.forward(*inputs, **kwargs)\n File "/usr/local/lib/python3.7/site-packages/paddle/nn/layer/conv.py", line 677, in forward\n use_cudnn=self._use_cudnn)\n File "/usr/local/lib/python3.7/site-packages/paddle/nn/functional/conv.py", line 157, in _conv_nd\n 'use_mkldnn': use_mkldnn})\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/layer_helper.py", line 43, in append_op\n return self.main_program.current_block().append_op(*args, **kwargs)\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/framework.py", line 3184, in append_op\n attrs=kwargs.get("attrs", None))\n File "/usr/local/lib/python3.7/site-packages/paddle/fluid/framework.py", line 2224, in init\n for frame in traceback.extract_stack():\n\n ExternalError: ACL error, the error code is : 500001. (at /paddle/backends/npu/kernels/funcs/npu_op_runner.cc:456)\n [operator < elementwise_add > error]"
}

尝试过的规避方案：
尝试配置环境变量 export CUSTOM_DEVICE_BLACK_LIST="elementwise_add,set_value,set_value_with_tensor" 以期让报错算子下沉到 CPU，但依然会报相同的算子执行错误。

启动脚本：
#!/usr/bin/bash

LOG_DIR="/workspace/logs"
CONTAINER_ID=$HOSTNAME
PREFIX_NAME=${CONTAINER_NAME:-"paddleocr_service"}
EXISTING_LOG=$(ls ${LOG_DIR}/${PREFIX_NAME}*${CONTAINER_ID}.log 2>/dev/null | head -1)

if [ -n "$EXISTING_LOG" ]; then
LOG_FILE=$EXISTING_LOG
echo -e "\n\n=======================================================" >> "$LOG_FILE"
echo "[$PREFIX_NAME] Container Restarted at $(date +'%Y-%m-%d %H:%M:%S')" >> "$LOG_FILE"
echo "=======================================================" >> "$LOG_FILE"
else
START_TIME=$(date +"%Y%m%d_%H%M%S")
LOG_FILE="${LOG_DIR}/${PREFIX_NAME}${START_TIME}${CONTAINER_ID}.log"
echo "[$PREFIX_NAME] New Container Created at $(date +'%Y-%m-%d %H:%M:%S')" > "$LOG_FILE"
fi

echo "All outputs will be redirected to: $LOG_FILE"

--- 注入昇腾生态环境变量 ---

容器内自带有 CANN 依赖包

source /usr/local/Ascend/ascend-toolkit/set_env.sh
export LD_PRELOAD=/usr/local/lib/python3.10/dist-packages/simsimd.libs/libgomp-a49a47f9.so.1.0.0:$LD_PRELOAD

2. 【🔥 NPU算子联合黑名单】仅将报错的元素相加、切片赋值算子剥离给 CPU 运行！

export CUSTOM_DEVICE_BLACK_LIST="elementwise_add,set_value,set_value_with_tensor"

export FLAGS_use_stride_kernel=0 # 关闭 stride 机制，防止维度/步长冲突引起的 Segmentation fault
export FLAGS_npu_storage_format=0 # 关闭私有格式，防止多页推理时内存对齐错误
export FLAGS_npu_blocking_run="true" # 开启同步下发，彻底杜绝 NPU 异步调度累积引发的 500001 崩溃

系统参数调优

sysctl -w vm.swappiness=0 >> "$LOG_FILE" 2>&1

--- 核心：自动依赖检测与安装（仅在第一次创建容器时运行，保证持久化） ---

python -c "import paddleocr" >/dev/null 2>&1
if [ $? -ne 0 ]; then
echo "First-time setup: Installing PaddlePaddle NPU dependencies..." >> "$LOG_FILE"
python -m pip install --pre paddlepaddle -i https://www.paddlepaddle.org.cn/packages/nightly/cpu/ >> "$LOG_FILE" 2>&1
python -m pip install --pre paddle-custom-npu -i https://www.paddlepaddle.org.cn/packages/nightly/npu/ >> "$LOG_FILE" 2>&1
pip install "paddleocr<3.0.0" fastapi uvicorn pydantic python-multipart -i https://pypi.tuna.tsinghua.edu.cn/simple >> "$LOG_FILE" 2>&1
echo "First-time setup: Installation completed." >> "$LOG_FILE"
fi

启动独立的 FastAPI + PP-Structure 接口服务，占用 8005 端口

cd /workspace/scripts
exec uvicorn app:app --host 0.0.0.0 --port 8005 >> "$LOG_FILE" 2>&1

cat python app.py

import base64
import io
import uvicorn
from fastapi import FastAPI
from pydantic import BaseModel
from paddleocr import PPStructure
from PIL import Image
import numpy as np
import logging

关闭繁杂的日志

logging.getLogger("ppocr").setLevel(logging.ERROR)

app = FastAPI(title="合同结构化 OCR API (310P NPU 加速版)")

==========================================

初始化 PP-Structure 引擎，严格映射模型路径

==========================================

print("正在加载 NPU 模型，请稍候...")
table_engine = PPStructure(
use_gpu=False, # 必须为 False（这是给英伟达CUDA用的）
use_npu=True, # 激活华为昇腾 NPU 加速
layout=True, # 开启版面分析
table=True, # 开启表格还原
show_log=False,
# 下面的路径对应我们刚刚映射进容器的 /models 目录
det_model_dir="/models/ch_PP-OCRv4_det_server_infer",
rec_model_dir="/models/ch_PP-OCRv4_rec_server_infer",
cls_model_dir="/models/ch_ppocr_mobile_v2.0_cls_infer",
layout_model_dir="/models/picodet_lcnet_x1_0_fgd_layout_cdla_infer",
table_model_dir="/models/ch_ppstructure_mobile_v2.0_SLANet_infer"
)
print("模型加载完成，NPU 就绪！")

定义接口接收的数据格式

class ImageRequest(BaseModel):
image_base64: str

@app.post("/extract_contract")
async def extract_contract(req: ImageRequest):
try:
# 1. 解码 Base64
img_data = base64.b64decode(req.image_base64)
image = Image.open(io.BytesIO(img_data)).convert('RGB')
img_ndarray = np.array(image)

    # 2. 送入 NPU 处理，获取带版面的结构化数据
    result = table_engine(img_ndarray)
    
    # 3. 整理输出格式，剔除无用坐标，保留干货
    structured_data = []
    for region in result:
        region_type = region['type']
        
        # 如果是表格，提取其 HTML 代码
        if region_type == 'table' and 'res' in region:
            structured_data.append({
                "type": "table",
                "content": region['res'].get('html', '')
            })
        
        # 如果是文本（正文/标题等），将这段话拼起来
        elif region_type in ['text', 'title', 'figure'] and 'res' in region:
            text_content = "".join([line.get('text', '') for line in region['res']])
            if text_content:
                structured_data.append({
                    "type": "text",
                    "content": text_content
                })
            
    return {"status": "success", "data": structured_data}

except Exception as e:
    return {"status": "error", "message": str(e)}

if name == "main":
uvicorn.run(app, host="0.0.0.0", port=8005)

运行环境参数：

1. 宿主机 NPU 型号：Atlas 300I Duo (昇腾 310P)
2. 宿主机 CANN 版本：CANN 9.1.0-beta1
3. 容器基础镜像：ccr-2vdh3abv-pub.cnc.bj.baidubce.com/device/paddle-npu:cann80RC2-ubuntu20-npu-base-aarch64-gcc84
4. PaddlePaddle版本：paddlepaddle-3.5.0.dev20260604
5. paddle-custom-npu版本：paddle-custom-npu-3.0.0.dev20260429
6. PaddleOCR版本：paddleocr < 3.0.0 (2.x 稳定版)