From 55d1f17ae6008ff0454efcdadc87711d9df38822 Mon Sep 17 00:00:00 2001 From: leiqing10 Date: Thu, 26 Feb 2026 17:31:56 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E9=80=82=E9=85=8D=E5=8A=A8=E6=80=81shape?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../runtime/backends/om/om_backend.cc | 82 ++++++++++++------- .../runtime/backends/om/om_backend.h | 3 + 2 files changed, 55 insertions(+), 30 deletions(-) diff --git a/deploy/ultra-infer/ultra_infer/runtime/backends/om/om_backend.cc b/deploy/ultra-infer/ultra_infer/runtime/backends/om/om_backend.cc index f8a87e2a2b..db50b348a4 100644 --- a/deploy/ultra-infer/ultra_infer/runtime/backends/om/om_backend.cc +++ b/deploy/ultra-infer/ultra_infer/runtime/backends/om/om_backend.cc @@ -25,6 +25,7 @@ bool OmBackend::aclInitFlag = false; OmBackend::~OmBackend() { FreeInputBuffer(); FreeOutputBuffer(); + Unload(); DestroyInput(); DestroyOutput(); DestroyResource(); @@ -124,6 +125,15 @@ bool OmBackend::Infer(std::vector &inputs, << ", errorCode is " << static_cast(aclRet); return false; } + aclDataType dtype = aclmdlGetInputDataType(modelDesc_, i); + aclTensorDesc *inputDesc = + aclCreateTensorDesc(dtype, inputs[i].Shape().size(), inputs[i].Shape(), ACL_FORMAT_NCHW); + aclRet = aclmdlSetDatasetTensorDesc(input_, inputDesc, i); + if (aclRet != ACL_SUCCESS) { + FDERROR << "SetDatasetTensorDesc failed." + << ", errorCode is " << static_cast(aclRet); + return false; + } } bool ret = Execute(); @@ -138,10 +148,15 @@ bool OmBackend::Infer(std::vector &inputs, // cp outputbuffer to outputs outputs->resize(outputs_desc_.size()); std::vector temp_shape(4); + aclTensorDesc *outputDesc; for (size_t i = 0; i < outputs_desc_.size(); ++i) { temp_shape.resize(outputs_desc_[i].shape.size()); + outputDesc = aclmdlGetDatasetTensorDesc(output_, i); for (int j = 0; j < outputs_desc_[i].shape.size(); ++j) { temp_shape[j] = outputs_desc_[i].shape[j]; + if (temp_shape[j] <= 0) { + aclGetTensorDescDimV2(outputDesc, j, &temp_shape[j]); + } } (*outputs)[i].Resize(temp_shape, outputs_desc_[i].dtype, outputs_desc_[i].name); @@ -219,35 +234,8 @@ bool OmBackend::LoadModel(const char *modelPath) { FDERROR << "model has already been loaded"; return false; } - aclError ret = aclmdlQuerySize(modelPath, &modelWorkSize_, &modelWeightSize_); - if (ret != ACL_SUCCESS) { - FDERROR << "query model false, model file is" << modelPath - << ", errorCode is " << static_cast(ret); - return false; - } - // using ACL_MEM_MALLOC_HUGE_FIRST to malloc memory, huge memory is preferred - // to use and huge memory can improve performance. - ret = aclrtMalloc(&modelWorkPtr_, modelWorkSize_, ACL_MEM_MALLOC_HUGE_FIRST); - if (ret != ACL_SUCCESS) { - FDERROR << "malloc buffer for work failed, require size is " - << modelWorkSize_ << ", errorCode is " << static_cast(ret); - return false; - } - // using ACL_MEM_MALLOC_HUGE_FIRST to malloc memory, huge memory is preferred - // to use and huge memory can improve performance. - ret = aclrtMalloc(&modelWeightPtr_, modelWeightSize_, - ACL_MEM_MALLOC_HUGE_FIRST); - if (ret != ACL_SUCCESS) { - FDERROR << "malloc buffer for weight failed, require size is " - << modelWeightSize_ << ", errorCode is " - << static_cast(ret); - return false; - } - - ret = aclmdlLoadFromFileWithMem(modelPath, &modelId_, modelWorkPtr_, - modelWorkSize_, modelWeightPtr_, - modelWeightSize_); + aclError ret = aclmdlLoadFromFile(modelPath, &modelId_); if (ret != ACL_SUCCESS) { FDERROR << "load model from file failed, model file is " << modelPath << ", errorCode is " << static_cast(ret); @@ -264,6 +252,13 @@ bool OmBackend::Execute() { if (ret != ACL_SUCCESS) { FDERROR << "execute model failed, modelId is " << modelId_ << ", errorCode is " << static_cast(ret); + if(outputSize_ != 0) { + FDERROR << "Infer failed in dynamic shape, Possible reasons:\n" + << "1.Insufficient memory allocated for output. Current value:" + << outputSize_ << "Please estimate the required output memory size" + << "and configure it via the environment variable ASCEND_OM_OUTPUTSIZE\n" + << "2.Some other reasons. Please analyze the error logs at ~/ascend/log\n"; + } return false; } return true; @@ -390,6 +385,21 @@ bool OmBackend::CreateInput() { return true; } +size_t OmBackend::GetOutputSizeFromENV() { + const char *outputSize = std::getenv("ASCEND_OM_OUTPUTSIZE"); + size defaultOutputSize = 64; // 64MB + try { + size_t size = static_cast(std::stoul(std::string(outputSize))) + if (size <= 0 || size >= 32000) { + FDWARNING << "ASCEND_OM_OUTPUTSIZE is invalid, use default outputSize"; + size = defaultOutputSize + } + } catch (const std::exception &e) { + return defaultOutputSize; + } + return defaultOutputSize; +} + bool OmBackend::CreateOutput() { if (modelDesc_ == nullptr) { FDERROR << "no model description, create output failed"; @@ -407,6 +417,10 @@ bool OmBackend::CreateOutput() { outputBuffer.resize(outputSize, nullptr); for (size_t i = 0; i < outputSize; ++i) { size_t modelOutputSize = aclmdlGetOutputSizeByIndex(modelDesc_, i); + if (modelOutputSize == 0) { + outputSize_ = GetOutputSizeFromENV(); + modelOutputSize = outputSize_ * 1024 * 1024; + } aclError ret = aclrtMalloc(&outputBuffer[i], modelOutputSize, ACL_MEM_MALLOC_HUGE_FIRST); if (ret != ACL_SUCCESS) { @@ -505,6 +519,16 @@ void OmBackend::FreeOutputBuffer() { } } +void OmBackend::Unload() { + aclError ret = aclmdlUnload(modelId_); + if (ret != ACL_SUCCESS) { + FDERROR << "aclmdlUnload failed" + << ", errorCode is " << static_cast(ret); + return; + } + loadFlag_ = false; +} + void OmBackend::DestroyInput() { if (input_ == nullptr) { return; @@ -525,8 +549,6 @@ void OmBackend::DestroyOutput() { for (size_t i = 0; i < aclmdlGetDatasetNumBuffers(output_); ++i) { aclDataBuffer *dataBuffer = aclmdlGetDatasetBuffer(output_, i); - void *data = aclGetDataBufferAddr(dataBuffer); - (void)aclrtFree(data); (void)aclDestroyDataBuffer(dataBuffer); } diff --git a/deploy/ultra-infer/ultra_infer/runtime/backends/om/om_backend.h b/deploy/ultra-infer/ultra_infer/runtime/backends/om/om_backend.h index 5a31b21b8c..5f91892a30 100644 --- a/deploy/ultra-infer/ultra_infer/runtime/backends/om/om_backend.h +++ b/deploy/ultra-infer/ultra_infer/runtime/backends/om/om_backend.h @@ -59,6 +59,7 @@ class OmBackend : public BaseBackend { uint32_t modelId_; size_t modelWorkSize_; // model work memory buffer size size_t modelWeightSize_; // model weight memory buffer size + size_t outputSize_ = 0; void *modelWorkPtr_; // model work memory buffer void *modelWeightPtr_; // model weight memory buffer aclmdlDesc *modelDesc_; @@ -70,6 +71,7 @@ class OmBackend : public BaseBackend { bool LoadModel(const char *modelPath); bool Execute(); bool CreateInput(); + void Unload(); void DestroyInput(); bool CreateOutput(); void DestroyOutput(); @@ -78,5 +80,6 @@ class OmBackend : public BaseBackend { void FreeInputBuffer(); void FreeOutputBuffer(); bool InitResource(); + size_t GetOutputSizeFromENV(); }; } // namespace ultra_infer From 2e8eb2cd7088c810f8a75ef43e5e22d15c276fed Mon Sep 17 00:00:00 2001 From: leiqing10 Date: Sat, 28 Feb 2026 15:23:07 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=87=8A=E6=94=BE?= =?UTF-8?q?=E8=B5=84=E6=BA=90=E6=8A=A5=E9=94=99=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../runtime/backends/om/om_backend.cc | 42 +++++++++---------- .../runtime/backends/om/om_backend.h | 2 +- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/deploy/ultra-infer/ultra_infer/runtime/backends/om/om_backend.cc b/deploy/ultra-infer/ultra_infer/runtime/backends/om/om_backend.cc index db50b348a4..d180c0cc0f 100644 --- a/deploy/ultra-infer/ultra_infer/runtime/backends/om/om_backend.cc +++ b/deploy/ultra-infer/ultra_infer/runtime/backends/om/om_backend.cc @@ -21,11 +21,11 @@ namespace ultra_infer { bool OmBackend::aclInitFlag = false; +uint32_t OmBackend::initCount = 0; OmBackend::~OmBackend() { FreeInputBuffer(); FreeOutputBuffer(); - Unload(); DestroyInput(); DestroyOutput(); DestroyResource(); @@ -127,7 +127,7 @@ bool OmBackend::Infer(std::vector &inputs, } aclDataType dtype = aclmdlGetInputDataType(modelDesc_, i); aclTensorDesc *inputDesc = - aclCreateTensorDesc(dtype, inputs[i].Shape().size(), inputs[i].Shape(), ACL_FORMAT_NCHW); + aclCreateTensorDesc(dtype, inputs[i].Shape().size(), inputs[i].Shape().data(), ACL_FORMAT_NCHW); aclRet = aclmdlSetDatasetTensorDesc(input_, inputDesc, i); if (aclRet != ACL_SUCCESS) { FDERROR << "SetDatasetTensorDesc failed." @@ -160,13 +160,6 @@ bool OmBackend::Infer(std::vector &inputs, } (*outputs)[i].Resize(temp_shape, outputs_desc_[i].dtype, outputs_desc_[i].name); - size_t modelOutputSize = aclmdlGetOutputSizeByIndex(modelDesc_, i); - if (modelOutputSize != (*outputs)[i].Nbytes()) { - FDERROR << "output size is not match, index: " << i - << ", modelOutputSize:" << modelOutputSize - << ", (*outputs)[i].Nbytes():" << (*outputs)[i].Nbytes(); - return false; - } aclError aclRet = aclrtMemcpy( (*outputs)[i].MutableData(), (*outputs)[i].Nbytes(), outputBuffer[i], (*outputs)[i].Nbytes(), ACL_MEMCPY_DEVICE_TO_HOST); @@ -183,6 +176,7 @@ bool OmBackend::Infer(std::vector &inputs, bool OmBackend::InitResource() { // ACL init aclError ret; + initCount += 1; if (aclInitFlag == false) { ret = aclInit(NULL); if (ret != ACL_SUCCESS) { @@ -387,13 +381,14 @@ bool OmBackend::CreateInput() { size_t OmBackend::GetOutputSizeFromENV() { const char *outputSize = std::getenv("ASCEND_OM_OUTPUTSIZE"); - size defaultOutputSize = 64; // 64MB + size_t defaultOutputSize = 64; // 64MB try { - size_t size = static_cast(std::stoul(std::string(outputSize))) + size_t size = static_cast(std::stoul(std::string(outputSize))); if (size <= 0 || size >= 32000) { FDWARNING << "ASCEND_OM_OUTPUTSIZE is invalid, use default outputSize"; - size = defaultOutputSize + size = defaultOutputSize; } + return size; } catch (const std::exception &e) { return defaultOutputSize; } @@ -519,16 +514,6 @@ void OmBackend::FreeOutputBuffer() { } } -void OmBackend::Unload() { - aclError ret = aclmdlUnload(modelId_); - if (ret != ACL_SUCCESS) { - FDERROR << "aclmdlUnload failed" - << ", errorCode is " << static_cast(ret); - return; - } - loadFlag_ = false; -} - void OmBackend::DestroyInput() { if (input_ == nullptr) { return; @@ -564,6 +549,15 @@ void OmBackend::DestroyResource() { << ", errorCode is " << static_cast(ret); return; } + + ret = aclmdlUnload(modelId_); + if (ret != ACL_SUCCESS) { + FDERROR << "aclmdlUnload failed" + << ", errorCode is " << static_cast(ret); + return; + } + loadFlag_ = false; + if (stream_ != nullptr) { ret = aclrtDestroyStream(stream_); if (ret != ACL_SUCCESS) { @@ -588,13 +582,15 @@ void OmBackend::DestroyResource() { << " failed, errorCode = " << static_cast(ret); } - if (aclInitFlag == true) { + if (initCount == 1) { ret = aclFinalize(); if (ret != ACL_SUCCESS) { FDERROR << "finalize acl failed, errorCode = " << static_cast(ret); } aclInitFlag = false; + } else { + initCount -= 1; } } diff --git a/deploy/ultra-infer/ultra_infer/runtime/backends/om/om_backend.h b/deploy/ultra-infer/ultra_infer/runtime/backends/om/om_backend.h index 5f91892a30..f45962d70d 100644 --- a/deploy/ultra-infer/ultra_infer/runtime/backends/om/om_backend.h +++ b/deploy/ultra-infer/ultra_infer/runtime/backends/om/om_backend.h @@ -67,11 +67,11 @@ class OmBackend : public BaseBackend { aclmdlDataset *output_; aclrtContext context_; aclrtStream stream_; + static uint32_t initCount; bool LoadModel(const char *modelPath); bool Execute(); bool CreateInput(); - void Unload(); void DestroyInput(); bool CreateOutput(); void DestroyOutput();