diff --git a/build.sh b/build.sh index 65e28707cd5d629ccb6e78a134469141938705cf..3f78be7174eb0929b1d00de4c0c9b4516ab893f6 100755 --- a/build.sh +++ b/build.sh @@ -48,7 +48,7 @@ build_exit() exit 1 } -update_submodule +#update_submodule echo "---------------- MindSpore-Lite: build start ----------------" init_default_options process_options "$@" diff --git a/dsp_sdk b/dsp_sdk new file mode 160000 index 0000000000000000000000000000000000000000..3fbb290365c4ae5b0becb3bb8c6ad4ddc88c541e --- /dev/null +++ b/dsp_sdk @@ -0,0 +1 @@ +Subproject commit 3fbb290365c4ae5b0becb3bb8c6ad4ddc88c541e diff --git a/include/api/context.h b/include/api/context.h index fdcc08547cedf4fc2c5cab2d5ed6adc48f9de158..5601818a92b7d2f7373f011d6b4d69642a087146 100644 --- a/include/api/context.h +++ b/include/api/context.h @@ -163,6 +163,22 @@ class MS_API Context { /// \return Mutable reference of DeviceInfoContext vector in this context. std::vector> &MutableDeviceInfo(); + /// \brief Set the status whether to perform model inference or training in parallel. + /// + /// \param[in] is_enable: true, open kernel config; false, close kernel config. + /// \param[in] config_file: the config file path. + void SetEnableKernelConfig(bool is_enable, const std::string &config_file = ""); + + /// \brief Get the kernel config info file path. + /// + /// \return The kernel config info file path setting. + std::string GetKernelConfigFile() const; + + /// \brief Get the status whether to enable kernel config from file. + /// + /// \return Bool value that indicates whether enable kernel config from file. + bool GetKernelConfig() const; + private: std::shared_ptr data_; }; diff --git a/include/api/types.h b/include/api/types.h index 506288d7aadbc9a8b55171e2cb84a03f868435b3..2bc3223b5690d4d0e1be027effeb977f2d7d7caa 100644 --- a/include/api/types.h +++ b/include/api/types.h @@ -422,7 +422,12 @@ struct MSCallBackParam { std::string node_name; /**< node name argument */ std::string node_type; /**< node type argument */ double execute_time; /**< gpu execute time */ - int arch; /**< kernel arch argument, value range: enum KERNEL_ARCH */ +#ifdef ENABLE_DSP + int arch{0x0}; + int core_mask{0x1}; + std::string custom_kernel_name{""}; + int mem_type{0}; +#endif }; /// \brief KernelCallBack defined the function pointer for callBack. diff --git a/mindspore b/mindspore index 2365375a9d84065f7354601cb3b48ececb477bc8..11c2221d8727a31284580892c2db4d1964740c96 160000 --- a/mindspore +++ b/mindspore @@ -1 +1 @@ -Subproject commit 2365375a9d84065f7354601cb3b48ececb477bc8 +Subproject commit 11c2221d8727a31284580892c2db4d1964740c96 diff --git a/mindspore-lite/src/CMakeLists.txt b/mindspore-lite/src/CMakeLists.txt index f59e0dd0f79ea8e940dd0c245ef8d544eda1bbf0..9e15dad34e477d7868af42e61b1b7631a44b1923 100644 --- a/mindspore-lite/src/CMakeLists.txt +++ b/mindspore-lite/src/CMakeLists.txt @@ -137,6 +137,10 @@ set(LITE_SRC ${CMAKE_CURRENT_SOURCE_DIR}/infer/primitive_type.cc ) +if(TARGET_FT04 OR TARGET_FT78) + set(LITE_SRC ${LITE_SRC} ${CMAKE_CURRENT_SOURCE_DIR}/litert/storage_pass.cc) +endif() + file(GLOB FORMAT_PASS_SRC "${CMAKE_CURRENT_SOURCE_DIR}/litert/pass/format_pass/*.cc") set(LITE_SRC ${LITE_SRC} ${FORMAT_PASS_SRC}) diff --git a/mindspore-lite/src/common/ops/populate/custom_populate.cc b/mindspore-lite/src/common/ops/populate/custom_populate.cc index b19b673880351bce9466798510166ad34a6c0bc3..9e12be6c6535aaa915329dcec13209c09ef66ae4 100644 --- a/mindspore-lite/src/common/ops/populate/custom_populate.cc +++ b/mindspore-lite/src/common/ops/populate/custom_populate.cc @@ -185,7 +185,7 @@ OpParameter *CreateNpuOfflineModelParameter(const void *prim) { return nullptr; } memset(param, 0, sizeof(CustomParameter)); - param->op_parameter_.type_ = PrimType_Inner_ThirdPartyModel; + param->op_parameter_.type_ = PrimType::PrimType_Inner_ThirdPartyModel; // The offline model only uses the attr_data [0] field, and other fields do not need to be assigned values. param->attr_data[0] = static_cast(const_cast(prim)); return reinterpret_cast(param); diff --git a/mindspore-lite/src/executor/kernel_exec.h b/mindspore-lite/src/executor/kernel_exec.h index 36c34fed01bc07a6135196572a6d47ef4bfa5db1..3fc024d595303700a350fef4cff7d36cdad7213f 100644 --- a/mindspore-lite/src/executor/kernel_exec.h +++ b/mindspore-lite/src/executor/kernel_exec.h @@ -62,6 +62,19 @@ enum KERNEL_ARCH { kKernelArch_MAX = kAPU }; static const char *const kBuiltin = "Builtin"; +inline int GetCoreIdByCoreMask(int core_mask) { + for (int i = 0; i < 4; ++i) { + if ((core_mask >> i) & 1) return i; + } + return -1; +} +enum MEMORY_TYPE { + kShareMemory, // lite::dsp::MemType::SMC + kDDR, // lite::dsp::MemType::DDR + kPrivateMemory, // lite::dsp::MemType::L2 + kMemoryType_MIN = kShareMemory, + kMemoryType_MAX = kPrivateMemory +}; struct KernelKey { KERNEL_ARCH arch = kCPU; @@ -70,10 +83,13 @@ struct KernelKey { int type = 0; std::string kernel_arch; std::string provider{kBuiltin}; - + MEMORY_TYPE memory_type = kShareMemory; + int core_mask{0x1}; bool operator==(const KernelKey &dst) const { - return type == dst.type && kernel_arch == dst.kernel_arch && provider == dst.provider && arch == dst.arch && - data_type == dst.data_type && format == dst.format; + return type == dst.type && kernel_arch == dst.kernel_arch && + provider == dst.provider && arch == dst.arch && + data_type == dst.data_type && format == dst.format && + memory_type == dst.memory_type && core_mask == dst.core_mask; } }; @@ -119,14 +135,26 @@ class KernelExec { auto ret = DoExecute(); if (after != nullptr) { +#ifdef ENABLE_DSP auto desc = this->desc(); MSCallBackParam opInfo; opInfo.node_name = this->name(); opInfo.node_type = TypeName(type()); - opInfo.arch = desc.arch; + opInfo.arch = static_cast(desc.arch); + opInfo.core_mask = desc.core_mask; + opInfo.mem_type = static_cast(desc.memory_type); + if (type() == schema::PrimitiveType_Custom) { + auto type = kernel_->GetAttr("type"); + opInfo.custom_kernel_name = type; + } if (!after(this->in_tensors(), this->out_tensors(), opInfo)) { MS_LOG(WARNING) << "run kernel after_callback failed, name: " << this->name(); } +#else + if (!after(this->in_tensors(), this->out_tensors(), {this->name(), TypeName(type())})) { + MS_LOG(WARNING) << "run kernel after_callback failed, name: " << this->name(); + } +#endif } return ret; } diff --git a/mindspore-lite/src/litert/cxx_api/context.cc b/mindspore-lite/src/litert/cxx_api/context.cc index 435ab537df34fdf01e1d7253256e7451c48a7937..81a3c6a1d7edac25170cfeb85e1cffe8853c0f40 100644 --- a/mindspore-lite/src/litert/cxx_api/context.cc +++ b/mindspore-lite/src/litert/cxx_api/context.cc @@ -247,6 +247,39 @@ bool Context::GetMultiModalHW() const { return data_->float_mode; } +void Context::SetEnableKernelConfig(bool is_enable, const std::string &config_file) { + if (data_ == nullptr) { + MS_LOG(ERROR) << "Invalid context."; + return; + } + data_->enable_kernel_config_ = is_enable; + data_->kernel_config_file_ = config_file; + if(is_enable) + MS_LOG(INFO) << "Context::data_->enable_kernel_config_ is true"; + else + MS_LOG(INFO) << "Context::data_->enable_kernel_config_ is false"; + if(!config_file.empty()) + MS_LOG(INFO) << "Context::data_->kernel_config_file_ is " << config_file; + else + MS_LOG(INFO) << "Context::data_->kernel_config_file_ is empty"; +} + +bool Context::GetKernelConfig() const { + if (data_ == nullptr) { + MS_LOG(ERROR) << "Invalid context."; + return false; + } + return data_->enable_kernel_config_; +} + +std::string Context::GetKernelConfigFile() const { + if (data_ == nullptr) { + MS_LOG(ERROR) << "Invalid context."; + return ""; + } + return data_->kernel_config_file_; +} + std::vector> &Context::MutableDeviceInfo() { static std::vector> empty{}; if (data_ == nullptr) { diff --git a/mindspore-lite/src/litert/cxx_api/context.h b/mindspore-lite/src/litert/cxx_api/context.h index 0923d1a4160ea78e46964499b81817f1280183d6..a2425c07255f4291c8656738e0bd4861e6b746e2 100644 --- a/mindspore-lite/src/litert/cxx_api/context.h +++ b/mindspore-lite/src/litert/cxx_api/context.h @@ -41,6 +41,8 @@ struct Context::Data { DelegateMode delegate_mode_ = kNoDelegate; std::shared_ptr delegate = nullptr; bool float_mode = false; + bool enable_kernel_config_ = false; + std::string kernel_config_file_ = ""; }; struct DeviceInfoContext::Data { diff --git a/mindspore-lite/src/litert/cxx_api/converters.cc b/mindspore-lite/src/litert/cxx_api/converters.cc index 2f1a346cd769b824ad0f114e49803cf65b2de911..310ddd25a44ca9c998e1d0117735bd72676a4912 100644 --- a/mindspore-lite/src/litert/cxx_api/converters.cc +++ b/mindspore-lite/src/litert/cxx_api/converters.cc @@ -78,7 +78,7 @@ Status ContextUtils::AddAscendDevice(lite::InnerContext *inner_context, DeviceIn return kSuccess; } -Status ContextUtils::AddDspDevice(lite::InnerContext *inner_context, DeviceInfoContext *device) { +Status ContextUtils::AddDspDevice(lite::InnerContext *inner_context, DeviceInfoContext *device, bool enable_kernel_config, std::string kernel_config_file) { lite::DeviceInfo device_info; auto dsp_context = device->Cast(); if (dsp_context == nullptr) { @@ -88,6 +88,8 @@ Status ContextUtils::AddDspDevice(lite::InnerContext *inner_context, DeviceInfoC device_info.dsp_device_info_ = {dsp_context->GetDeviceID()}; inner_context->device_list_.push_back( {lite::DT_DSP, device_info, dsp_context->GetProvider(), dsp_context->GetProviderDevice()}); + inner_context->enable_kernel_config_ = enable_kernel_config; + inner_context->kernel_config_file_ = kernel_config_file; return kSuccess; } @@ -175,7 +177,7 @@ std::shared_ptr ContextUtils::Convert(Context *context) { } else if (device->GetDeviceType() == kAscend) { ret = AddAscendDevice(inner_context.get(), device.get()); } else if (device->GetDeviceType() == kDSP) { - ret = AddDspDevice(inner_context.get(), device.get()); + ret = AddDspDevice(inner_context.get(), device.get(), context->GetKernelConfig(), context->GetKernelConfigFile()); } else if (device->GetDeviceType() == kCustomDevice) { ret = AddCustomDevice(inner_context.get(), device); } diff --git a/mindspore-lite/src/litert/cxx_api/converters.h b/mindspore-lite/src/litert/cxx_api/converters.h index b19d7c0ec63cb3a01ee21aab685d8c21cc1d99ee..c5e565714f683affe2e2caa5b1d1ed099cf18f45 100644 --- a/mindspore-lite/src/litert/cxx_api/converters.h +++ b/mindspore-lite/src/litert/cxx_api/converters.h @@ -45,7 +45,7 @@ class MS_API ContextUtils { lite::InnerContext *inner_context); static Status AddNpuDevice(bool enable_fp16, int frequency, lite::InnerContext *inner_context); static Status AddAscendDevice(lite::InnerContext *inner_context, DeviceInfoContext *device); - static Status AddDspDevice(lite::InnerContext *inner_context, DeviceInfoContext *device); + static Status AddDspDevice(lite::InnerContext *inner_context, DeviceInfoContext *device, bool enable_kernel_config, std::string kernel_config_file); static Status AddCustomDevice(lite::InnerContext *inner_context, const std::shared_ptr &device); static bool IsAffinityModeValid(int affinity_mode) { return affinity_mode >= lite::NO_BIND && affinity_mode <= lite::MID_CPU; diff --git a/mindspore-lite/src/litert/inner_context.h b/mindspore-lite/src/litert/inner_context.h index 19f61ad84e955845fe9bae72204f79ec6710227e..cefbacd597b3214c48085fd548be080ee6df9ed3 100644 --- a/mindspore-lite/src/litert/inner_context.h +++ b/mindspore-lite/src/litert/inner_context.h @@ -147,6 +147,9 @@ struct MS_API InnerContext { std::unordered_map> link_info_{}; const ExecEnv *GetExecEnv() const { return &exec_env_; } + bool enable_kernel_config_ = false; + std::string kernel_config_file_ = ""; + private: int IsValid(); bool IsAllDeviceTypeValid() const; diff --git a/mindspore-lite/src/litert/kernel/dsp/dsp_allocator.cc b/mindspore-lite/src/litert/kernel/dsp/dsp_allocator.cc index 568599337f17415c459bdfa9937f8992899b7850..932ed64e1f884e8155601100e51859247c0362e4 100644 --- a/mindspore-lite/src/litert/kernel/dsp/dsp_allocator.cc +++ b/mindspore-lite/src/litert/kernel/dsp/dsp_allocator.cc @@ -40,7 +40,7 @@ void DSPAllocator::UnLock() { } } -void *DSPAllocator::MinimumFit(MemType mem_type, size_t size) { +void *DSPAllocator::MinimumFit(MEMORY_TYPE mem_type, size_t size) { auto iter = free_list_.lower_bound(size); while (iter != free_list_.end() && (iter->second->size_ >= size) && (iter->second->size_ < (size << shift_factor_))) { auto mem_buf = iter->second; @@ -58,7 +58,7 @@ void *DSPAllocator::MinimumFit(MemType mem_type, size_t size) { return nullptr; } -void *DSPAllocator::_Malloc(MemType mem_type, size_t size) { +void *DSPAllocator::_Malloc(MEMORY_TYPE mem_type, size_t size) { if (size > dsp_runtime_->GetMaxAllocSize()) { MS_LOG(ERROR) << "MallocData out of max_size, size: " << size; return nullptr; @@ -223,8 +223,8 @@ void DSPAllocator::Clear() { UnLock(); } -MemType DSPAllocator::GetMemType(void *host_ptr) { - MemType mem_type{MemType::DDR}; +MEMORY_TYPE DSPAllocator::GetMemType(void *host_ptr) { + MEMORY_TYPE mem_type{MEMORY_TYPE::kDDR}; Lock(); auto it = allocated_list_.find(host_ptr); if (it == allocated_list_.end()) { diff --git a/mindspore-lite/src/litert/kernel/dsp/dsp_allocator.h b/mindspore-lite/src/litert/kernel/dsp/dsp_allocator.h index 3cb2bae2b79f7c67a7729a8fdeb429dc67f19821..9744f533ec42e35186a57838acb43aaa11cdb74d 100644 --- a/mindspore-lite/src/litert/kernel/dsp/dsp_allocator.h +++ b/mindspore-lite/src/litert/kernel/dsp/dsp_allocator.h @@ -25,8 +25,10 @@ #include #include #include "src/litert/inner_allocator.h" +#include "src/executor/kernel_exec.h" namespace mindspore::lite::dsp { +using MEMORY_TYPE = mindspore::kernel::MEMORY_TYPE; enum class MemType : char { SMC, DDR, DDR36BIT, L2 }; #define UNLOCK_AND_RETURN_NULL(condition, ptr) \ do { \ @@ -43,8 +45,8 @@ class DSPAllocator : public mindspore::Allocator { ~DSPAllocator() override; using Allocator::Malloc; - void *Malloc(size_t size, MemType type) { return _Malloc(type, size); } - void *Malloc(size_t size) override { return _Malloc(MemType::DDR, size); } + void *Malloc(size_t size, MEMORY_TYPE type) { return _Malloc(type, size); } + void *Malloc(size_t size) override { return _Malloc(mem_type_, size); } void Free(void *ptr) override; int RefCount(void *ptr) override; @@ -54,15 +56,19 @@ class DSPAllocator : public mindspore::Allocator { size_t TotalSize(); void Clear(); - MemType GetMemType(void *host_ptr); + MEMORY_TYPE GetMemType(void *host_ptr); + void SetMemType(MEMORY_TYPE mem_type) { mem_type_ = mem_type; }; + int32_t GetCoreId() { return core_id_; }; + void SetCoreId(int32_t core_id) { core_id_ = core_id; }; + uint32_t GetCoreMask() { return 1U << core_id_; }; uint64_t GetDeviceMemPtr(void *buffer); void *Prepare(void *ptr) override { return ptr; } private: void Lock(); void UnLock(); - void *MinimumFit(MemType mem_type, size_t size); - void *_Malloc(MemType mem_type, size_t size = 0); + void *MinimumFit(MEMORY_TYPE mem_type, size_t size); + void *_Malloc(MEMORY_TYPE mem_type, size_t size = 0); template void ClearMemList(T *list); @@ -76,7 +82,7 @@ class DSPAllocator : public mindspore::Allocator { size_t size_{0}; uint64_t device_ptr_{0}; void *host_ptr_{nullptr}; - MemType mem_type_{MemType::DDR}; + MEMORY_TYPE mem_type_ {MEMORY_TYPE::kDDR}; }; // buf, membuf> @@ -86,6 +92,7 @@ class DSPAllocator : public mindspore::Allocator { // 6 is empirical value int shift_factor_ = 6; bool lock_flag_ = true; + MEMORY_TYPE mem_type_ {MEMORY_TYPE::kDDR}; }; } // namespace mindspore::lite::dsp diff --git a/mindspore-lite/src/litert/kernel/dsp/dsp_kernel.cc b/mindspore-lite/src/litert/kernel/dsp/dsp_kernel.cc index 137e2efe70a309c551ddd8ba045c56d90df6a451..8761389545261cf37498fc7819a568a7f0d59e05 100644 --- a/mindspore-lite/src/litert/kernel/dsp/dsp_kernel.cc +++ b/mindspore-lite/src/litert/kernel/dsp/dsp_kernel.cc @@ -53,6 +53,13 @@ int DSPKernel::PreProcess() { auto *output = out_tensors_.at(i); CHECK_NULL_RETURN(output); CHECK_NULL_RETURN(output->allocator()); + auto dsp_allocator = reinterpret_cast(output->allocator().get()); + dsp_allocator->SetMemType(out_mem_type_); + dsp_allocator->SetCoreId(core_id_); + MS_LOG(DEBUG) << "MallocData begin, tensor_name: " + << output->tensor_name() + << ", memory: " << static_cast(out_mem_type_) + << ", core_id: " << static_cast(core_id_); ret = output->MallocData(); if (ret != RET_OK) { MS_LOG(ERROR) << "MallocData failed"; diff --git a/mindspore-lite/src/litert/kernel/dsp/dsp_kernel.h b/mindspore-lite/src/litert/kernel/dsp/dsp_kernel.h index 9f9c16c992db3b72497f883d4b0ab74047586089..c4417f4e0ee577bbc2637bede415680e5f928ced 100644 --- a/mindspore-lite/src/litert/kernel/dsp/dsp_kernel.h +++ b/mindspore-lite/src/litert/kernel/dsp/dsp_kernel.h @@ -63,17 +63,21 @@ class DSPKernel : public LiteKernel { bool MallocDataDone(); virtual int CheckSpecs() { return RET_OK; } - lite::dsp::MemType GetMemType() { return out_mem_type_; } - void SetMemType(lite::dsp::MemType mem_type) { out_mem_type_ = mem_type; } + MEMORY_TYPE GetMemType() { return out_mem_type_; } + void SetMemType(MEMORY_TYPE mem_type) { out_mem_type_ = mem_type; } + int32_t GetCoreId() { return core_id_; } + void SetCoreId(int32_t core_id) { core_id_ = core_id; } void SetKernelArg(const std::vector &kernel_args = {}) { kernel_args_ = kernel_args; } int InferShape() override; + int GetCoreMask() {return core_mask_; } void SetCoreMask(int core_mask) { core_mask_ = core_mask; } void SetKernelName(const std::string &kernel_name) { kernel_name_ = kernel_name; } protected: lite::dsp::DSPRuntime *dsp_runtime_; std::vector kernel_args_; - lite::dsp::MemType out_mem_type_{lite::dsp::MemType::DDR}; + MEMORY_TYPE out_mem_type_{MEMORY_TYPE::kDDR}; + int32_t core_id_{0}; int core_mask_{0xf}; std::string kernel_name_; @@ -106,6 +110,13 @@ kernel::LiteKernel *DSPKernelCreator(const std::vector &inputs, delete kernel; return nullptr; } + auto dsp_kernel = reinterpret_cast(kernel); + dsp_kernel->SetMemType(desc.memory_type); + dsp_kernel->SetCoreId(GetCoreIdByCoreMask(desc.core_mask)); + MS_LOG(DEBUG) << "kernel " << opParameter->name_ + << " create success!" + << " memory type: " << static_cast(dsp_kernel->GetMemType()) + << " core id: " << static_cast(dsp_kernel->GetCoreId()); return kernel; } @@ -121,19 +132,19 @@ inline const std::string GetDataTypePrefix(int data_type) { return ""; } -inline char GetMemTypeSuffix(lite::dsp::MemType mem_type) { +inline char GetMemTypeSuffix(mindspore::kernel::MEMORY_TYPE mem_type) { switch (mem_type) { - case lite::dsp::MemType::DDR: - case lite::dsp::MemType::SMC: + case mindspore::kernel::MEMORY_TYPE::kDDR: + case mindspore::kernel::MEMORY_TYPE::kShareMemory: return 's'; - case lite::dsp::MemType::L2: + case mindspore::kernel::MEMORY_TYPE::kPrivateMemory: return 'p'; default: return 's'; } } -inline std::string GenerateKernelName(int data_type, lite::dsp::MemType mem_type, const std::string op_name) { +inline std::string GenerateKernelName(int data_type, mindspore::kernel::MEMORY_TYPE mem_type, const std::string op_name) { const std::string prefix = GetDataTypePrefix(data_type); char suffix = GetMemTypeSuffix(mem_type); auto result = prefix + "_" + op_name + "_" + suffix; diff --git a/mindspore-lite/src/litert/kernel/dsp/ft04/applymomentum.cc b/mindspore-lite/src/litert/kernel/dsp/ft04/applymomentum.cc index ce2bcfa537f2b3111c645c954eca8478c2edefe2..2ffee648fb8db3955ea2990b51936bab2d3d2413 100644 --- a/mindspore-lite/src/litert/kernel/dsp/ft04/applymomentum.cc +++ b/mindspore-lite/src/litert/kernel/dsp/ft04/applymomentum.cc @@ -26,6 +26,9 @@ #include "src/litert/kernel_registry.h" using mindspore::kernel::KERNEL_ARCH::kDSP; +using mindspore::kernel::MEMORY_TYPE::kShareMemory; +using mindspore::kernel::MEMORY_TYPE::kShareMemory; +using mindspore::kernel::MEMORY_TYPE::kPrivateMemory; using mindspore::lite::KernelRegistrar; using mindspore::lite::RET_ERROR; using mindspore::lite::RET_OK; @@ -197,6 +200,11 @@ int ApplyMomentumDSPKernel::Run() { return RET_OK; } -REG_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_ApplyMomentum, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_ApplyMomentum, DSPKernelCreator) +// register DSP kernels for different memory types +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_ApplyMomentum, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_ApplyMomentum, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_ApplyMomentum, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_ApplyMomentum, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_ApplyMomentum, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_ApplyMomentum, kPrivateMemory, DSPKernelCreator) } // namespace mindspore::kernel diff --git a/mindspore-lite/src/litert/kernel/dsp/ft04/assert.cc b/mindspore-lite/src/litert/kernel/dsp/ft04/assert.cc index 32e9835f1a9df541ee07a092f9f53cb79aa1313e..92d68762dfed49afdf9a6805842bbf44537c5673 100644 --- a/mindspore-lite/src/litert/kernel/dsp/ft04/assert.cc +++ b/mindspore-lite/src/litert/kernel/dsp/ft04/assert.cc @@ -20,6 +20,9 @@ #include "src/litert/kernel_registry.h" using mindspore::kernel::KERNEL_ARCH::kDSP; +using mindspore::kernel::MEMORY_TYPE::kShareMemory; +using mindspore::kernel::MEMORY_TYPE::kShareMemory; +using mindspore::kernel::MEMORY_TYPE::kPrivateMemory; using mindspore::lite::KernelRegistrar; using mindspore::lite::RET_ERROR; using mindspore::lite::RET_OK; @@ -70,6 +73,9 @@ int AssertDSPKernel::Run() { return RunInt(); } -REG_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_Assert, DSPKernelCreator) +// register DSP kernels for different memory types +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_Assert, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_Assert, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_Assert, kPrivateMemory, DSPKernelCreator) } // namespace mindspore::kernel diff --git a/mindspore-lite/src/litert/kernel/dsp/ft04/custom/custom_memory_copy.cc b/mindspore-lite/src/litert/kernel/dsp/ft04/custom/custom_memory_copy.cc new file mode 100644 index 0000000000000000000000000000000000000000..b57c22f1e22e5ada28ce283720b55e0700a0429b --- /dev/null +++ b/mindspore-lite/src/litert/kernel/dsp/ft04/custom/custom_memory_copy.cc @@ -0,0 +1,51 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/litert/kernel/dsp/ft04/custom/custom_memory_copy.h" +#include +#include "src/common/utils.h" + +namespace mindspore::lite { +const auto kComplex64 = DataType::kNumberTypeComplex64; +const auto kFloat32 = DataType::kNumberTypeFloat32; + +// Prepare will be called during graph compilation +int CustomMemoryCopyKernel::Prepare() { + allocator_ = dsp_runtime_->GetAllocator(); + std::string kernel_name = "custom_Memory_Copy"; + MS_LOG(DEBUG) << kernel_name << " Init Done!"; + return kSuccess; +} + +int CustomMemoryCopyKernel::Run() { + int ret = -1; + kernel_name_ = "FT04_datatrans"; + auto input_device_ptr = allocator_->GetDeviceMemPtr(in_tensors_.front()->data()); + auto output_device_ptr = allocator_->GetDeviceMemPtr(out_tensors_.front()->data()); + auto Size = (uint64_t)(out_tensors_.front()->Size()); + SetKernelArg({input_device_ptr, output_device_ptr, Size}); + ret = dsp_runtime_->RunKernel(kernel_name_, kernel_args_, allocator_->GetCoreMask()); + if (ret != kSuccess) { + MS_LOG(ERROR) << this->name() << " Run failed! "; + return kLiteError; + } + MS_LOG(DEBUG) << this->name() << " Run success! "; + return kSuccess; +} + +REGISTER_CUSTOM_KERNEL(DSP, FTMatrix, kFloat32, Custom_FT_MemoryCopy, CustomKernelCreator) +REGISTER_CUSTOM_KERNEL(DSP, FTMatrix, kComplex64, Custom_FT_MemoryCopy, CustomKernelCreator) +} // namespace mindspore \ No newline at end of file diff --git a/mindspore-lite/src/litert/kernel/dsp/ft04/custom/custom_memory_copy.h b/mindspore-lite/src/litert/kernel/dsp/ft04/custom/custom_memory_copy.h new file mode 100644 index 0000000000000000000000000000000000000000..5df10fa8514616df03afb65a0e3842bf914a3fff --- /dev/null +++ b/mindspore-lite/src/litert/kernel/dsp/ft04/custom/custom_memory_copy.h @@ -0,0 +1,45 @@ +/** + * Copyright 2025 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_CUSTOM_KERNEL_MEMORY_COPY_H_ +#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_CUSTOM_KERNEL_MEMORY_COPY_H_ + +#include +#include +#include +#include +#include +#include "include/registry/register_kernel_interface.h" +#include "include/registry/register_kernel.h" +#include "include/errorcode.h" +#include "src/litert/kernel/dsp/custom_kernel.h" + +namespace mindspore::lite { + +class CustomMemoryCopyKernel : public CustomKernel { + public: + using CustomKernel::CustomKernel; + ~CustomMemoryCopyKernel() override = default;; + int Prepare() override; + int Run() override; + + private: + std::string kernel_name_ = ""; + std::shared_ptr allocator_{}; +}; + +} // namespace mindspore::lite +#endif // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_CUSTOM_KERNEL_MEMORY_COPY_H_ diff --git a/mindspore-lite/src/litert/kernel/dsp/ft04/div.cc b/mindspore-lite/src/litert/kernel/dsp/ft04/div.cc index d88b8e40569173c4015368d8052220aed8deea1b..9423773fc89bf4281e60e5d10685e99568586774 100644 --- a/mindspore-lite/src/litert/kernel/dsp/ft04/div.cc +++ b/mindspore-lite/src/litert/kernel/dsp/ft04/div.cc @@ -22,6 +22,9 @@ #include "src/litert/kernel_registry.h" using mindspore::kernel::KERNEL_ARCH::kDSP; +using mindspore::kernel::MEMORY_TYPE::kShareMemory; +using mindspore::kernel::MEMORY_TYPE::kShareMemory; +using mindspore::kernel::MEMORY_TYPE::kPrivateMemory; using mindspore::lite::KernelRegistrar; using mindspore::lite::RET_ERROR; using mindspore::lite::RET_OK; @@ -102,9 +105,20 @@ int DivDSPKernel::Run() { return RET_OK; } -REG_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_DivFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_DivFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_DivFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_DivFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_DivFusion, DSPKernelCreator) +//register DSP kernels for different memory types +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_DivFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_DivFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_DivFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_DivFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_DivFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_DivFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_DivFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_DivFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_DivFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_DivFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_DivFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_DivFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_DivFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_DivFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_DivFusion, kPrivateMemory, DSPKernelCreator) } // namespace mindspore::kernel diff --git a/mindspore-lite/src/litert/kernel/dsp/ft04/exp.cc b/mindspore-lite/src/litert/kernel/dsp/ft04/exp.cc index aef12c9dbefb28e946b6414295711298a936640b..02ce9c978eda2b679659a3651adc4f26b698046e 100644 --- a/mindspore-lite/src/litert/kernel/dsp/ft04/exp.cc +++ b/mindspore-lite/src/litert/kernel/dsp/ft04/exp.cc @@ -24,6 +24,9 @@ #include "src/litert/kernel_registry.h" using mindspore::kernel::KERNEL_ARCH::kDSP; +using mindspore::kernel::MEMORY_TYPE::kShareMemory; +using mindspore::kernel::MEMORY_TYPE::kShareMemory; +using mindspore::kernel::MEMORY_TYPE::kPrivateMemory; using mindspore::lite::KernelRegistrar; using mindspore::lite::RET_ERROR; using mindspore::lite::RET_OK; @@ -116,9 +119,20 @@ int ExpDSPKernel::Run() { return RET_OK; } -REG_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_ExpFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_ExpFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_ExpFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_ExpFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_ExpFusion, DSPKernelCreator) +// register DSP kernels for different memory types +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_ExpFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_ExpFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_ExpFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_ExpFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_ExpFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_ExpFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_ExpFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_ExpFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_ExpFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_ExpFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_ExpFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_ExpFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_ExpFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_ExpFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_ExpFusion, kPrivateMemory, DSPKernelCreator) } // namespace mindspore::kernel diff --git a/mindspore-lite/src/litert/kernel/dsp/ft04/ft04_add.cc b/mindspore-lite/src/litert/kernel/dsp/ft04/ft04_add.cc index 89b5e55a59ea3627c347bb634d4b7ffdbf8663e8..63f1608be843af32e981b726b04bf1b758d1bfd7 100644 --- a/mindspore-lite/src/litert/kernel/dsp/ft04/ft04_add.cc +++ b/mindspore-lite/src/litert/kernel/dsp/ft04/ft04_add.cc @@ -22,6 +22,9 @@ #include "src/litert/kernel_registry.h" using mindspore::kernel::KERNEL_ARCH::kDSP; +using mindspore::kernel::MEMORY_TYPE::kShareMemory; +using mindspore::kernel::MEMORY_TYPE::kShareMemory; +using mindspore::kernel::MEMORY_TYPE::kPrivateMemory; using mindspore::lite::KernelRegistrar; using mindspore::lite::RET_ERROR; using mindspore::lite::RET_OK; @@ -47,9 +50,20 @@ int AddFT04DSPKernel::Prepare() { return RET_OK; } -REG_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_AddFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_AddFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_AddFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_AddFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_AddFusion, DSPKernelCreator) +// register DSP kernels for different memory types +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_AddFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_AddFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_AddFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_AddFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_AddFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_AddFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_AddFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_AddFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_AddFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_AddFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_AddFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_AddFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_AddFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_AddFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_AddFusion, kPrivateMemory, DSPKernelCreator) } // namespace mindspore::kernel diff --git a/mindspore-lite/src/litert/kernel/dsp/ft04/linspace.cc b/mindspore-lite/src/litert/kernel/dsp/ft04/linspace.cc index 1434ad836c42e9428042b82f0d2ccb7216a64826..0a0df1676f26470d701232ffb0d3ba5398900f5c 100644 --- a/mindspore-lite/src/litert/kernel/dsp/ft04/linspace.cc +++ b/mindspore-lite/src/litert/kernel/dsp/ft04/linspace.cc @@ -23,6 +23,9 @@ #include "schema/inner/model_generated.h" using mindspore::kernel::KERNEL_ARCH::kDSP; +using mindspore::kernel::MEMORY_TYPE::kShareMemory; +using mindspore::kernel::MEMORY_TYPE::kShareMemory; +using mindspore::kernel::MEMORY_TYPE::kPrivateMemory; using mindspore::lite::KernelRegistrar; using mindspore::lite::RET_ERROR; using mindspore::lite::RET_OK; @@ -100,7 +103,12 @@ int LinspaceDSPKernel::Run() { return RET_OK; } -REG_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_LinSpace, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_LinSpace, DSPKernelCreator) +// register DSP kernels for different memory types +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_LinSpace, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_LinSpace, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_LinSpace, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_LinSpace, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_LinSpace, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_LinSpace, kPrivateMemory, DSPKernelCreator) } // namespace mindspore::kernel diff --git a/mindspore-lite/src/litert/kernel/dsp/ft04/matmulfusion.cc b/mindspore-lite/src/litert/kernel/dsp/ft04/matmulfusion.cc index 602507e06123f48e0d3bfa8dfc4cb3fe365a1d0b..66f15838dcbee5c067ea294b61198b3b4264f33d 100644 --- a/mindspore-lite/src/litert/kernel/dsp/ft04/matmulfusion.cc +++ b/mindspore-lite/src/litert/kernel/dsp/ft04/matmulfusion.cc @@ -22,6 +22,9 @@ #include "src/litert/kernel/cpu/nnacl_c/matmul_parameter.h" using mindspore::kernel::KERNEL_ARCH::kDSP; +using mindspore::kernel::MEMORY_TYPE::kShareMemory; +using mindspore::kernel::MEMORY_TYPE::kShareMemory; +using mindspore::kernel::MEMORY_TYPE::kPrivateMemory; using mindspore::lite::KernelRegistrar; using mindspore::lite::RET_ERROR; using mindspore::lite::RET_OK; @@ -172,10 +175,21 @@ int MatMulFusionDSPKernel::Run() { return RET_OK; } -REG_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_MatMulFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_MatMulFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_MatMulFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_MatMulFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_MatMulFusion, DSPKernelCreator) +// register DSP kernels for different memory types +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_MatMulFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_MatMulFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_MatMulFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_MatMulFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_MatMulFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_MatMulFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_MatMulFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_MatMulFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_MatMulFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_MatMulFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_MatMulFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_MatMulFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_MatMulFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_MatMulFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_MatMulFusion, kPrivateMemory, DSPKernelCreator) } // namespace mindspore::kernel diff --git a/mindspore-lite/src/litert/kernel/dsp/ft04/mul.cc b/mindspore-lite/src/litert/kernel/dsp/ft04/mul.cc index ba979717d79562a79c7d024e9a8af385fec0ed7c..7b08333ab529a0f78a7a52fea13058005d489bea 100644 --- a/mindspore-lite/src/litert/kernel/dsp/ft04/mul.cc +++ b/mindspore-lite/src/litert/kernel/dsp/ft04/mul.cc @@ -22,6 +22,9 @@ #include "src/litert/kernel_registry.h" using mindspore::kernel::KERNEL_ARCH::kDSP; +using mindspore::kernel::MEMORY_TYPE::kShareMemory; +using mindspore::kernel::MEMORY_TYPE::kShareMemory; +using mindspore::kernel::MEMORY_TYPE::kPrivateMemory; using mindspore::lite::KernelRegistrar; using mindspore::lite::RET_ERROR; using mindspore::lite::RET_OK; @@ -102,9 +105,20 @@ int MulDSPKernel::Run() { return RET_OK; } -REG_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_MulFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_MulFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_MulFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_MulFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_MulFusion, DSPKernelCreator) +// register DSP kernels for different memory types +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_MulFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_MulFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_MulFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_MulFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_MulFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_MulFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_MulFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_MulFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_MulFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_MulFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_MulFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_MulFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_MulFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_MulFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_MulFusion, kPrivateMemory, DSPKernelCreator) } // namespace mindspore::kernel diff --git a/mindspore-lite/src/litert/kernel/dsp/ft04/ragged_range.cc b/mindspore-lite/src/litert/kernel/dsp/ft04/ragged_range.cc index 1acc8965b4570904ba3a58f3ddb81347d3c261cd..5b1118703a58e97616e6990f63da978f9d1f84ef 100644 --- a/mindspore-lite/src/litert/kernel/dsp/ft04/ragged_range.cc +++ b/mindspore-lite/src/litert/kernel/dsp/ft04/ragged_range.cc @@ -22,6 +22,9 @@ #include "schema/inner/model_generated.h" using mindspore::kernel::KERNEL_ARCH::kDSP; +using mindspore::kernel::MEMORY_TYPE::kShareMemory; +using mindspore::kernel::MEMORY_TYPE::kShareMemory; +using mindspore::kernel::MEMORY_TYPE::kPrivateMemory; using mindspore::lite::KernelRegistrar; using mindspore::lite::RET_ERROR; using mindspore::lite::RET_OK; @@ -124,9 +127,18 @@ int RaggedRangeDSPKernel::Run() { } } -REG_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_RaggedRange, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_RaggedRange, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_RaggedRange, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_RaggedRange, DSPKernelCreator) +// register DSP kernels for different memory types +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_RaggedRange, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_RaggedRange, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_RaggedRange, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_RaggedRange, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_RaggedRange, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_RaggedRange, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_RaggedRange, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_RaggedRange, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_RaggedRange, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_RaggedRange, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_RaggedRange, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_RaggedRange, kPrivateMemory, DSPKernelCreator) } // namespace mindspore::kernel diff --git a/mindspore-lite/src/litert/kernel/dsp/ft04/range.cc b/mindspore-lite/src/litert/kernel/dsp/ft04/range.cc index b33bf15341b8e58b7748f3e994b8fa76265b72b9..db2d98a13218729b7d55885c54b2fbbfba9c4c1e 100644 --- a/mindspore-lite/src/litert/kernel/dsp/ft04/range.cc +++ b/mindspore-lite/src/litert/kernel/dsp/ft04/range.cc @@ -23,6 +23,9 @@ #include "src/litert/kernel_registry.h" using mindspore::kernel::KERNEL_ARCH::kDSP; +using mindspore::kernel::MEMORY_TYPE::kShareMemory; +using mindspore::kernel::MEMORY_TYPE::kShareMemory; +using mindspore::kernel::MEMORY_TYPE::kPrivateMemory; using mindspore::lite::KernelRegistrar; using mindspore::lite::RET_ERROR; using mindspore::lite::RET_OK; @@ -110,8 +113,17 @@ int RangeDSPKernel::Run() { return RET_OK; } -REG_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_Range, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_Range, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_Range, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_Range, DSPKernelCreator) +// register DSP kernels for different memory types +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_Range, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_Range, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_Range, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_Range, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_Range, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_Range, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_Range, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_Range, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_Range, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_Range, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_Range, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_Range, kPrivateMemory, DSPKernelCreator) } // namespace mindspore::kernel diff --git a/mindspore-lite/src/litert/kernel/dsp/ft04/sub.cc b/mindspore-lite/src/litert/kernel/dsp/ft04/sub.cc index 1b2543251a55e6b96cc8a3904a2db4093ea0e47b..88b87f42aee4a2c6dd538e522291df4cc7ebe2e4 100644 --- a/mindspore-lite/src/litert/kernel/dsp/ft04/sub.cc +++ b/mindspore-lite/src/litert/kernel/dsp/ft04/sub.cc @@ -22,6 +22,9 @@ #include "src/litert/kernel_registry.h" using mindspore::kernel::KERNEL_ARCH::kDSP; +using mindspore::kernel::MEMORY_TYPE::kShareMemory; +using mindspore::kernel::MEMORY_TYPE::kShareMemory; +using mindspore::kernel::MEMORY_TYPE::kPrivateMemory; using mindspore::lite::KernelRegistrar; using mindspore::lite::RET_ERROR; using mindspore::lite::RET_OK; @@ -102,9 +105,20 @@ int SubDSPKernel::Run() { return RET_OK; } -REG_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_SubFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_SubFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_SubFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_SubFusion, DSPKernelCreator) -REG_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_SubFusion, DSPKernelCreator) +// register DSP kernels for different memory types +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_SubFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_SubFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat32, PrimitiveType_SubFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_SubFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_SubFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt16, PrimitiveType_SubFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_SubFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_SubFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeInt32, PrimitiveType_SubFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_SubFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_SubFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeComplex64, PrimitiveType_SubFusion, kPrivateMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_SubFusion, kShareMemory, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_SubFusion, kDDR, DSPKernelCreator) +REG_DSP_KERNEL(kDSP, kNumberTypeFloat16, PrimitiveType_SubFusion, kPrivateMemory, DSPKernelCreator) } // namespace mindspore::kernel diff --git a/mindspore-lite/src/litert/kernel/dsp/ft78/custom/custom_memory_copy.cc b/mindspore-lite/src/litert/kernel/dsp/ft78/custom/custom_memory_copy.cc new file mode 100644 index 0000000000000000000000000000000000000000..915d5e72d399e714649cea8ba3a103d0dfe963ba --- /dev/null +++ b/mindspore-lite/src/litert/kernel/dsp/ft78/custom/custom_memory_copy.cc @@ -0,0 +1,51 @@ +/** + * Copyright 2021 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "src/litert/kernel/dsp/ft04/custom/custom_memory_copy.h" +#include +#include "src/common/utils.h" + +namespace mindspore::lite { +const auto kComplex64 = DataType::kNumberTypeComplex64; +const auto kFloat32 = DataType::kNumberTypeFloat32; + +// Prepare will be called during graph compilation +int CustomMemoryCopyKernel::Prepare() { + allocator_ = dsp_runtime_->GetAllocator(); + std::string kernel_name = "custom_Memory_Copy"; + MS_LOG(DEBUG) << kernel_name << " Init Done!"; + return kSuccess; +} + +int CustomMemoryCopyKernel::Run() { + int ret = -1; + kernel_name_ = "FT78_datatrans"; + auto input_device_ptr = allocator_->GetDeviceMemPtr(in_tensors_.front()->data()); + auto output_device_ptr = allocator_->GetDeviceMemPtr(out_tensors_.front()->data()); + auto Size = (uint64_t)(out_tensors_.front()->Size()); + SetKernelArg({input_device_ptr, output_device_ptr, Size}); + ret = dsp_runtime_->RunKernel(kernel_name_, kernel_args_, allocator_->GetCoreMask()); + if (ret != kSuccess) { + MS_LOG(ERROR) << this->name() << " Run failed! "; + return kLiteError; + } + MS_LOG(DEBUG) << this->name() << " Run success! "; + return kSuccess; +} + +REGISTER_CUSTOM_KERNEL(DSP, FTMatrix, kFloat32, Custom_FT_MemoryCopy, CustomKernelCreator) +REGISTER_CUSTOM_KERNEL(DSP, FTMatrix, kComplex64, Custom_FT_MemoryCopy, CustomKernelCreator) +} // namespace mindspore \ No newline at end of file diff --git a/mindspore-lite/src/litert/kernel/dsp/ft78/custom/custom_memory_copy.h b/mindspore-lite/src/litert/kernel/dsp/ft78/custom/custom_memory_copy.h new file mode 100644 index 0000000000000000000000000000000000000000..5df10fa8514616df03afb65a0e3842bf914a3fff --- /dev/null +++ b/mindspore-lite/src/litert/kernel/dsp/ft78/custom/custom_memory_copy.h @@ -0,0 +1,45 @@ +/** + * Copyright 2025 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_LITE_SRC_RUNTIME_KERNEL_CUSTOM_KERNEL_MEMORY_COPY_H_ +#define MINDSPORE_LITE_SRC_RUNTIME_KERNEL_CUSTOM_KERNEL_MEMORY_COPY_H_ + +#include +#include +#include +#include +#include +#include "include/registry/register_kernel_interface.h" +#include "include/registry/register_kernel.h" +#include "include/errorcode.h" +#include "src/litert/kernel/dsp/custom_kernel.h" + +namespace mindspore::lite { + +class CustomMemoryCopyKernel : public CustomKernel { + public: + using CustomKernel::CustomKernel; + ~CustomMemoryCopyKernel() override = default;; + int Prepare() override; + int Run() override; + + private: + std::string kernel_name_ = ""; + std::shared_ptr allocator_{}; +}; + +} // namespace mindspore::lite +#endif // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_CUSTOM_KERNEL_MEMORY_COPY_H_ diff --git a/mindspore-lite/src/litert/kernel_registry.cc b/mindspore-lite/src/litert/kernel_registry.cc index 2d6230a137199463061f57146b50f7d3056d321f..a7f9e48b1e2635ddf883486df8ae615b86a33eab 100644 --- a/mindspore-lite/src/litert/kernel_registry.cc +++ b/mindspore-lite/src/litert/kernel_registry.cc @@ -40,6 +40,7 @@ using mindspore::kernel::kBuiltin; using mindspore::kernel::kCPU; using mindspore::kernel::KERNEL_ARCH; +using mindspore::kernel::MEMORY_TYPE; using mindspore::kernel::KernelKey; namespace mindspore::lite { @@ -98,12 +99,16 @@ int KernelRegistry::GetCreatorFuncIndex(const kernel::KernelKey desc) { int device_index = static_cast(desc.arch) - kKernelArch_MIN; int dType_index = desc.data_type == kObjectTypeString ? 0 : static_cast(desc.data_type) - kNumberTypeBegin; int op_index = static_cast(desc.type); + int mType_index = desc.memory_type - kMemoryType_MIN; int op_type_length = op_type_length_; if (op_index >= PrimType_InnerOpMin && desc.type < PrimType_InnerOpMax) { op_type_length = inner_op_type_length_; op_index -= PrimType_InnerOpMin; } - int index = device_index * data_type_length_ * op_type_length + dType_index * op_type_length + op_index; + int index = device_index * data_type_length_ * memory_type_length_ * op_type_length + + dType_index * memory_type_length_ * op_type_length + + mType_index * op_type_length + + op_index; return index; } @@ -148,6 +153,51 @@ void KernelRegistry::RegKernel(KERNEL_ARCH arch, TypeId data_type, int op_type, } } +void KernelRegistry::RegKernel(KERNEL_ARCH arch, TypeId data_type, int op_type, MEMORY_TYPE memory_type, kernel::KernelCreator creator) { + CreatorArraysInit(); + KernelKey desc = {arch, data_type, NHWC, op_type}; + desc.memory_type = memory_type; + int index = GetCreatorFuncIndex(desc); + if (desc.type >= PrimType_MIN && desc.type < PrimType_MAX) { + if (index >= array_size_ || index < 0) { + MS_LOG(ERROR) << "invalid kernel key, arch " << desc.arch << ", data_type" << desc.data_type << ",op type " + << desc.type; + return; + } + if (creator_arrays_ != nullptr) { + creator_arrays_[index] = creator; + } + } else if (desc.type >= PrimType_InnerOpMin && desc.type < PrimType_InnerOpMax) { + MS_CHECK_TRUE_RET_VOID(index >= 0 && index < inner_op_array_size_); + if (inner_op_creator_arrays_ != nullptr) { + inner_op_creator_arrays_[index] = creator; + } + } +} + +void KernelRegistry::RegKernel(KERNEL_ARCH arch, TypeId data_type, int op_type, MEMORY_TYPE memory_type, int core_mask, kernel::KernelCreator creator) { + CreatorArraysInit(); + KernelKey desc = {arch, data_type, NHWC, op_type}; + desc.memory_type = memory_type; + desc.core_mask = core_mask; + int index = GetCreatorFuncIndex(desc); + if (desc.type >= PrimType_MIN && desc.type < PrimType_MAX) { + if (index >= array_size_ || index < 0) { + MS_LOG(ERROR) << "invalid kernel key, arch " << desc.arch << ", data_type" << desc.data_type << ",op type " + << desc.type; + return; + } + if (creator_arrays_ != nullptr) { + creator_arrays_[index] = creator; + } + } else if (desc.type >= PrimType_InnerOpMin && desc.type < PrimType_InnerOpMax) { + MS_CHECK_TRUE_RET_VOID(index >= 0 && index < inner_op_array_size_); + if (inner_op_creator_arrays_ != nullptr) { + inner_op_creator_arrays_[index] = creator; + } + } +} + KernelRegistry::~KernelRegistry() { KernelRegistry *instance = GetInstance(); std::unique_lock malloc_creator_array(instance->lock_); diff --git a/mindspore-lite/src/litert/kernel_registry.h b/mindspore-lite/src/litert/kernel_registry.h index 1715a0fc36f110363105814ccb2913d64d6c8d1b..4d125784b3e74be52924e39b93ca806c7410f3c3 100644 --- a/mindspore-lite/src/litert/kernel_registry.h +++ b/mindspore-lite/src/litert/kernel_registry.h @@ -27,6 +27,8 @@ using mindspore::kernel::kKernelArch_MAX; using mindspore::kernel::kKernelArch_MIN; +using mindspore::kernel::kMemoryType_MAX; +using mindspore::kernel::kMemoryType_MIN; using mindspore::schema::PrimitiveType_MAX; using mindspore::schema::PrimitiveType_MIN; @@ -41,6 +43,8 @@ class MS_API KernelRegistry { int GetCreatorFuncIndex(kernel::KernelKey desc); void RegKernel(kernel::KernelKey desc, kernel::KernelCreator creator); void RegKernel(kernel::KERNEL_ARCH arch, TypeId data_type, int type, kernel::KernelCreator creator); + void RegKernel(kernel::KERNEL_ARCH arch, TypeId data_type, int type, kernel::MEMORY_TYPE memory_type, kernel::KernelCreator creator); + void RegKernel(kernel::KERNEL_ARCH arch, TypeId data_type, int type, kernel::MEMORY_TYPE memory_type, int core_mask, kernel::KernelCreator creator); bool SupportKernel(const kernel::KernelKey &key); int GetKernelExec(const std::vector &in_tensors, const std::vector &out_tensors, const InnerContext *ctx, const mindspore::Context *ms_ctx, const kernel::KernelKey &key, @@ -55,7 +59,8 @@ class MS_API KernelRegistry { static const int device_type_length_{kKernelArch_MAX - kKernelArch_MIN + 1}; static const int data_type_length_{kNumberTypeEnd - kNumberTypeBegin + 1}; static const int op_type_length_{PrimitiveType_MAX - PrimitiveType_MIN + 1}; - static const int array_size_{device_type_length_ * data_type_length_ * op_type_length_}; + static const int memory_type_length_{kMemoryType_MAX - kMemoryType_MIN + 1}; + static const int array_size_{device_type_length_ * data_type_length_ * op_type_length_ * memory_type_length_}; kernel::KernelCreator *creator_arrays_ = nullptr; static const int inner_op_type_length_{PrimType_InnerOpMax - PrimType_InnerOpMin}; static const int inner_op_array_size_{device_type_length_ * data_type_length_ * inner_op_type_length_}; @@ -73,16 +78,31 @@ class KernelRegistrar { KernelRegistrar(const kernel::KernelKey &desc, kernel::KernelCreator creator) { KernelRegistry::GetInstance()->RegKernel(desc, creator); } - ~KernelRegistrar() = default; KernelRegistrar(const kernel::KERNEL_ARCH arch, const TypeId data_type, const int op_type, kernel::KernelCreator creator) { KernelRegistry::GetInstance()->RegKernel(arch, data_type, op_type, creator); } + + KernelRegistrar(const kernel::KERNEL_ARCH arch, const TypeId data_type, const int op_type, + const kernel::MEMORY_TYPE memory_type, kernel::KernelCreator creator) { + KernelRegistry::GetInstance()->RegKernel(arch, data_type, op_type, memory_type, creator); + } + + KernelRegistrar(const kernel::KERNEL_ARCH arch, const TypeId data_type, const int op_type, + const kernel::MEMORY_TYPE memory_type, const int core_mask, kernel::KernelCreator creator) { + KernelRegistry::GetInstance()->RegKernel(arch, data_type, op_type, memory_type, core_mask, creator); + } + + ~KernelRegistrar() = default; }; #define REG_KERNEL(arch, data_type, op_type, kernelCreater) \ static KernelRegistrar g_##arch##data_type##op_type##kernelReg(arch, data_type, op_type, kernelCreater); + + +#define REG_DSP_KERNEL(arch, data_type, op_type, memory_type, kernelCreater) \ + static KernelRegistrar g_##arch##data_type##op_type##memory_type##kernelReg(arch, data_type, op_type, memory_type, kernelCreater); } // namespace mindspore::lite #endif // MINDSPORE_LITE_SRC_RUNTIME_KERNEL_REGISTRY_H_ diff --git a/mindspore-lite/src/litert/lite_session.cc b/mindspore-lite/src/litert/lite_session.cc index 77fc59e21aec69374146b39dfb549a756f4ffcdd..383ff02deea66b11b95daf0c6d1822abff39dbbf 100644 --- a/mindspore-lite/src/litert/lite_session.cc +++ b/mindspore-lite/src/litert/lite_session.cc @@ -599,6 +599,7 @@ int LiteSession::CompileGraph(Model *model) { delegate_device_type_); scheduler.SetupSchedulerCb(std::move(sched_cb_)); scheduler.SetConfig(config_info_); + scheduler.SetKernelConfig(); ret = scheduler.Schedule(&kernels_); if (ret != RET_OK) { MS_LOG(ERROR) << "Schedule kernels failed: " << ret; diff --git a/mindspore-lite/src/litert/scheduler.cc b/mindspore-lite/src/litert/scheduler.cc index 774b3811f068a52657be310612af6ed220f1dd81..e9ceea437d270bb716a1a7d80d42037d3b9c1151 100644 --- a/mindspore-lite/src/litert/scheduler.cc +++ b/mindspore-lite/src/litert/scheduler.cc @@ -21,6 +21,7 @@ #include #include #include +#include #include "src/tensorlist.h" #include "nnacl_c/partial_fusion_parameter.h" #include "include/errorcode.h" @@ -40,6 +41,9 @@ #include "src/common/context_util.h" #include "src/litert/infer_manager.h" #include "src/litert/runtime_pass.h" +#ifdef ENABLE_DSP +#include "src/litert/storage_pass.h" +#endif #include "src/litert/pass/format_pass/format_pass.h" #if !defined(AUTO_PARALLEL_CLIP) || !defined(RUNTIME_PASS_CLIP) #include "src/litert/sub_graph_split.h" @@ -421,7 +425,9 @@ int Scheduler::Schedule(std::vector *dst_kernels) { << "TypeId(" << kernel->desc().data_type << "); " << "OpType(" << PrimitiveCurVersionTypeName(kernel->desc().type) << "); " << "format(" << kernel->desc().format << "); " - << "arch(" << kernel->desc().arch << ")"; + << "arch(" << kernel->desc().arch << ")" + << "mem_type(" << kernel->desc().memory_type<< ")" + << "core_mask(" << kernel->desc().core_mask<< ")"; } ret = CheckCpuValid(dst_kernels); if (ret != RET_OK) { @@ -462,6 +468,15 @@ int Scheduler::Schedule(std::vector *dst_kernels) { return ret; } shape_fusion_pass_->RestoreState(); + +#ifdef ENABLE_DSP + status = StoragePass(dst_kernels, src_tensors_, ms_context_); + if (status != RET_OK) { + MS_LOG(ERROR) << "storage pass failed."; + return RET_ERROR; + } +#endif + if (IsPrintDebug()) { MS_LOG(DEBUG) << "schedule kernels success."; for (auto subgraph : *dst_kernels) { @@ -1126,7 +1141,7 @@ int Scheduler::FindDspKernel(const std::vector &in_tensors, const std: } // support more data type like int32 - kernel::KernelKey dsp_desc{kernel::KERNEL_ARCH::kDSP, desc.data_type, NHWC, desc.type}; + kernel::KernelKey dsp_desc{kernel::KERNEL_ARCH::kDSP, desc.data_type, NHWC, desc.type, "dsp", "Builtin", kernel::MEMORY_TYPE::kShareMemory, 0x1}; // memory_type && core_id // weight dequant auto ret = WeightDecoder::DequantNode(op_parameter, in_tensors, kNumberTypeFloat32, src_model_->graph_.version_, context_->float_mode); @@ -1226,45 +1241,16 @@ int Scheduler::FindProviderKernel(const std::vector &in_tensors, const return RET_NOT_SUPPORT; } -kernel::KernelExec *Scheduler::FindBackendKernel(const std::vector &in_tensors, - const std::vector &out_tensors, const LiteGraph::Node *node, - TypeId prefer_data_type) { - MS_ASSERT(node != nullptr); - // why we need this - TypeId data_type; - if (node->quant_type_ == schema::QuantType_QUANT_WEIGHT) { - if (in_tensors.front()->data_type() == kNumberTypeBool) { - data_type = kNumberTypeBool; - } else { - data_type = kNumberTypeFloat32; - } - } else { - data_type = GetFirstFp32Fp16OrInt8Type(in_tensors); - if (data_type == kTypeUnknown) { - MS_LOG(ERROR) << "GetFirstFp32Fp16OrInt8Type is unknown."; - return nullptr; - } - } - if (context_->float_mode) { - for (auto tensor : out_tensors) { - if (!tensor->quant_params().empty() && - (tensor->data_type() == kNumberTypeInt8 || tensor->data_type() == kNumberTypeUInt8)) { - data_type = kNumberTypeFloat32; - tensor->set_data_type(kNumberTypeFloat32); - } - } - } - kernel::KernelExec *kernel = nullptr; - auto status = FindProviderKernel(in_tensors, out_tensors, node, data_type, &kernel); - if (status == RET_OK && kernel != nullptr) { - return kernel; - } +int Scheduler::SelectKernelByDeviceList(const std::vector &in_tensors, + const std::vector &out_tensors, const LiteGraph::Node *node, + TypeId data_type, kernel::KernelExec **kernel, TypeId prefer_data_type) { + int status = RET_NOT_SUPPORT; MS_ASSERT(!node->output_indices_.empty()); OpParameter *op_parameter = op_parameters_[node->output_indices_.at(0)]; if (op_parameter == nullptr) { MS_LOG(ERROR) << "Can not find OpParameter!type: " << GetPrimitiveTypeName(node->primitive_, context_->get_schema_version()); - return nullptr; + return status; } #if (defined GPU_OPENCL) || (defined ENABLE_FP16) @@ -1277,9 +1263,9 @@ kernel::KernelExec *Scheduler::FindBackendKernel(const std::vector &in bool dsp_priority = DeviceTypePriority(context_, DT_DSP, DT_CPU); bool use_dsp_kernel = node->device_type_ == DT_DSP || node->device_type_ == kDefaultDeviceType; if (dsp_priority && use_dsp_kernel) { - status = FindDspKernel(in_tensors, out_tensors, op_parameter, desc, &kernel, prefer_data_type); + status = FindDspKernel(in_tensors, out_tensors, op_parameter, desc, kernel, prefer_data_type); if (status == RET_OK) { - return kernel; + return status; } else { MS_LOG(DEBUG) << "Get dsp op failed, scheduler to cpu: " << PrimitiveCurVersionTypeName(desc.type) << " " << node->name_; @@ -1290,7 +1276,7 @@ kernel::KernelExec *Scheduler::FindBackendKernel(const std::vector &in op_parameter = op_parameters_[node->output_indices_.at(0)]; } else { MS_LOG(ERROR) << "Try repeat infer fail: " << node->name_; - return nullptr; + return RET_NOT_SUPPORT; } } } @@ -1303,7 +1289,7 @@ kernel::KernelExec *Scheduler::FindBackendKernel(const std::vector &in if (gpu_priority && use_gpu_kernel) { status = FindGpuKernel(in_tensors, out_tensors, op_parameter, desc, &kernel, prefer_data_type); if (status == RET_OK) { - return kernel; + return status; } else { MS_LOG(DEBUG) << "Get gpu op failed, scheduler to cpu: " << PrimitiveCurVersionTypeName(desc.type) << " " << node->name_; @@ -1315,7 +1301,7 @@ kernel::KernelExec *Scheduler::FindBackendKernel(const std::vector &in op_parameter->thread_num_ = kernel_thread_count; } else { MS_LOG(ERROR) << "Try repeat infer fail: " << node->name_; - return nullptr; + return RET_NOT_SUPPORT; } } } @@ -1326,7 +1312,7 @@ kernel::KernelExec *Scheduler::FindBackendKernel(const std::vector &in ((is_train_session_ == false) || (sched_cb_ && sched_cb_->SchedFp16Kernel(node)))) { status = FindCpuKernel(in_tensors, out_tensors, op_parameter, desc, kNumberTypeFloat16, &kernel); if (status == RET_OK) { - return kernel; + return status; } else { MS_LOG(INFO) << "Get op failed, scheduler to CPU by node: " << node->name_ << ", prefer_data_type is " << (prefer_data_type == kNumberTypeFloat16 ? "fp16" : "unknown") << ", desc.type index:" << desc.type @@ -1339,7 +1325,7 @@ kernel::KernelExec *Scheduler::FindBackendKernel(const std::vector &in op_parameter->thread_num_ = kernel_thread_count; } else { MS_LOG(ERROR) << "Try repeat infer fail: " << node->name_; - return nullptr; + return RET_NOT_SUPPORT; } } } @@ -1349,9 +1335,164 @@ kernel::KernelExec *Scheduler::FindBackendKernel(const std::vector &in MS_LOG(DEBUG) << "Get fp16 op failed, back to fp32 op."; desc.data_type = kNumberTypeFloat32; } - status = FindCpuKernel(in_tensors, out_tensors, op_parameter, desc, kNumberTypeFloat32, &kernel); + status = FindCpuKernel(in_tensors, out_tensors, op_parameter, desc, kNumberTypeFloat32, kernel); if (status == RET_OK) { - return kernel; + return status; + } else if (status == RET_ERROR) { + op_parameters_.erase(node->output_indices_.at(0)); + auto ret = InferNodeShape(node); + if (!(ret == RET_INFER_INVALID || ret == RET_OK)) { + MS_LOG(ERROR) << "Try repeat infer fail: " << node->name_; + } + } +#ifdef OP_INT8_CLIP + if (desc.data_type == kNumberTypeInt8) { + MS_LOG(ERROR) << unsupport_int8_log; + } +#endif + return status; +} + +int Scheduler::SelectKernelByConfigFile(const std::vector &in_tensors, + const std::vector &out_tensors, const LiteGraph::Node *node, + TypeId data_type, kernel::KernelExec **kernel, TypeId prefer_data_type) { + int status = RET_NOT_SUPPORT; + MS_ASSERT(!node->output_indices_.empty()); + OpParameter *op_parameter = op_parameters_[node->output_indices_.at(0)]; + if (op_parameter == nullptr) { + MS_LOG(ERROR) << "Can not find OpParameter!type: " + << GetPrimitiveTypeName(node->primitive_, context_->get_schema_version()); + return status; + } + + op_parameter->is_train_session_ = is_train_session_; + + if (kernel_info_.empty()) { + MS_LOG(ERROR) << "Kernel config parse error."; + return status; + } + const KernelInfo *kernel_info = nullptr; + for (const auto &kinfo : kernel_info_) { + if (kinfo.name == node->name_) { + kernel_info = &kinfo; + break; + } + } + if (kernel_info == nullptr) { + MS_LOG(ERROR) << "kernel name: " << node->name_ << " not found in kernel config file"; + return status; + } + kernel::KernelKey desc{kernel::KERNEL_ARCH::kCPU, data_type, NHWC, op_parameter->type_}; + +#ifdef ENABLE_DSP + bool dsp_priority = DeviceTypePriority(context_, DT_DSP, DT_CPU); + bool use_dsp_kernel = node->device_type_ == DT_DSP || node->device_type_ == kDefaultDeviceType; + if (dsp_priority && use_dsp_kernel) { + status = FindDspKernel(in_tensors, out_tensors, op_parameter, desc, kernel, prefer_data_type); + if (status == RET_OK) { + auto current_desc = (*kernel)->desc(); + current_desc.memory_type = kernel_info->memory_type; + current_desc.core_mask = kernel_info->core_mask; + (*kernel)->set_desc(current_desc); + return status; + } else { + MS_LOG(DEBUG) << "Get dsp op failed, scheduler to cpu: " << PrimitiveCurVersionTypeName(desc.type) << " " + << node->name_; + if (status == RET_ERROR) { + op_parameters_.erase(node->output_indices_.at(0)); + auto ret = InferNodeShape(node); + if (ret == RET_INFER_INVALID || ret == RET_OK) { + op_parameter = op_parameters_[node->output_indices_.at(0)]; + } else { + MS_LOG(ERROR) << "Try repeat infer fail: " << node->name_; + return RET_NOT_SUPPORT; + } + } + } + } +#endif + + if (data_type == kNumberTypeFloat16) { + MS_LOG(DEBUG) << "Get fp16 op failed, back to fp32 op."; + desc.data_type = kNumberTypeFloat32; + } + status = FindCpuKernel(in_tensors, out_tensors, op_parameter, desc, kNumberTypeFloat32, kernel); + if (status == RET_OK) { + return status; + } else if (status == RET_ERROR) { + op_parameters_.erase(node->output_indices_.at(0)); + auto ret = InferNodeShape(node); + if (!(ret == RET_INFER_INVALID || ret == RET_OK)) { + MS_LOG(ERROR) << "Try repeat infer fail: " << node->name_; + } + } +#ifdef OP_INT8_CLIP + if (desc.data_type == kNumberTypeInt8) { + MS_LOG(ERROR) << unsupport_int8_log; + } +#endif + return status; +} + +int Scheduler::SelectKernelRandomly(const std::vector &in_tensors, + const std::vector &out_tensors, const LiteGraph::Node *node, + TypeId data_type, kernel::KernelExec **kernel, TypeId prefer_data_type) { + int status = RET_NOT_SUPPORT; + MS_ASSERT(!node->output_indices_.empty()); + OpParameter *op_parameter = op_parameters_[node->output_indices_.at(0)]; + if (op_parameter == nullptr) { + MS_LOG(ERROR) << "Can not find OpParameter!type: " + << GetPrimitiveTypeName(node->primitive_, context_->get_schema_version()); + return status; + } + + op_parameter->is_train_session_ = is_train_session_; + std::random_device random_device; + auto rng = std::mt19937(random_device()); + kernel::KernelKey desc{kernel::KERNEL_ARCH::kCPU, data_type, NHWC, op_parameter->type_}; + +#ifdef ENABLE_DSP + std::uniform_int_distribution<> generate_priority(false, true); + bool dsp_priority = generate_priority(rng); + bool use_dsp_kernel = node->device_type_ == DT_DSP || node->device_type_ == kDefaultDeviceType; + if (dsp_priority && use_dsp_kernel) { + status = FindDspKernel(in_tensors, out_tensors, op_parameter, desc, kernel, prefer_data_type); + if (status == RET_OK) { + std::vector coremark = {1, 3, 15}; + std::vector all_mem_types = {kernel::MEMORY_TYPE::kShareMemory, + kernel::MEMORY_TYPE::kDDR, + kernel::MEMORY_TYPE::kPrivateMemory}; + std::uniform_int_distribution<> generate_coremark(0, coremark.size() - 1); + std::uniform_int_distribution<> generate_memtype(0, all_mem_types.size() - 1); + auto current_desc = (*kernel)->desc(); + current_desc.memory_type = all_mem_types[generate_memtype(rng)]; + current_desc.core_mask = coremark[generate_coremark(rng)]; + (*kernel)->set_desc(current_desc); + return status; + } else { + MS_LOG(DEBUG) << "Get dsp op failed, scheduler to cpu: " << PrimitiveCurVersionTypeName(desc.type) << " " + << node->name_; + if (status == RET_ERROR) { + op_parameters_.erase(node->output_indices_.at(0)); + auto ret = InferNodeShape(node); + if (ret == RET_INFER_INVALID || ret == RET_OK) { + op_parameter = op_parameters_[node->output_indices_.at(0)]; + } else { + MS_LOG(ERROR) << "Try repeat infer fail: " << node->name_; + return RET_NOT_SUPPORT; + } + } + } + } +#endif + + if (data_type == kNumberTypeFloat16) { + MS_LOG(DEBUG) << "Get fp16 op failed, back to fp32 op."; + desc.data_type = kNumberTypeFloat32; + } + status = FindCpuKernel(in_tensors, out_tensors, op_parameter, desc, kNumberTypeFloat32, kernel); + if (status == RET_OK) { + return status; } else if (status == RET_ERROR) { op_parameters_.erase(node->output_indices_.at(0)); auto ret = InferNodeShape(node); @@ -1364,6 +1505,57 @@ kernel::KernelExec *Scheduler::FindBackendKernel(const std::vector &in MS_LOG(ERROR) << unsupport_int8_log; } #endif + return status; +} + +kernel::KernelExec *Scheduler::FindBackendKernel(const std::vector &in_tensors, + const std::vector &out_tensors, const LiteGraph::Node *node, + TypeId prefer_data_type) { + MS_ASSERT(node != nullptr); + // why we need this + TypeId data_type; + if (node->quant_type_ == schema::QuantType_QUANT_WEIGHT) { + if (in_tensors.front()->data_type() == kNumberTypeBool) { + data_type = kNumberTypeBool; + } else { + data_type = kNumberTypeFloat32; + } + } else { + data_type = GetFirstFp32Fp16OrInt8Type(in_tensors); + if (data_type == kTypeUnknown) { + MS_LOG(ERROR) << "GetFirstFp32Fp16OrInt8Type is unknown."; + return nullptr; + } + } + if (context_->float_mode) { + for (auto tensor : out_tensors) { + if (!tensor->quant_params().empty() && + (tensor->data_type() == kNumberTypeInt8 || tensor->data_type() == kNumberTypeUInt8)) { + data_type = kNumberTypeFloat32; + tensor->set_data_type(kNumberTypeFloat32); + } + } + } + kernel::KernelExec *kernel = nullptr; + auto status = FindProviderKernel(in_tensors, out_tensors, node, data_type, &kernel); + if (status == RET_OK && kernel != nullptr) { + return kernel; + } + + if(context_ != nullptr) { + if (context_->enable_kernel_config_) { + if (context_->kernel_config_file_.empty()) { + status = SelectKernelRandomly(in_tensors, out_tensors, node, data_type, &kernel, prefer_data_type); + } else { + status = SelectKernelByConfigFile(in_tensors, out_tensors, node, data_type, &kernel, prefer_data_type); + } + } else { + status = SelectKernelByDeviceList(in_tensors, out_tensors, node, data_type, &kernel, prefer_data_type); + } + } + if (status == RET_OK && kernel != nullptr) { + return kernel; + } return nullptr; } diff --git a/mindspore-lite/src/litert/scheduler.h b/mindspore-lite/src/litert/scheduler.h index fa80db9b6a7346ee6b7331128029cd2e44aae5f7..fcd227baf4315a9c7798cd04b6c2d41997e8c37e 100644 --- a/mindspore-lite/src/litert/scheduler.h +++ b/mindspore-lite/src/litert/scheduler.h @@ -32,7 +32,22 @@ #include "include/api/delegate.h" #include "src/control_flow/control_flow_scheduler.h" #include "src/litert/runtime_shape_fusion_pass.h" +#include "src/common/file_utils.h" +#include +using MEMORY_TYPE = mindspore::kernel::MEMORY_TYPE; +struct ModelTime { + double min; + double avg; + double max; +}; + +struct KernelInfo { + std::string name; + int arch; + int core_mask; + MEMORY_TYPE memory_type; +}; namespace mindspore::lite { constexpr int kDefaultDeviceType = -1; class Scheduler { @@ -61,6 +76,51 @@ class Scheduler { void SetConfig(const std::map> *config_info) { config_info_ = config_info; } + void SetKernelConfig() { + if (context_->kernel_config_file_.empty() || !context_->enable_kernel_config_) { + MS_LOG(INFO) << "Kernel config file is empty or kernel config is disabled."; + return; + } + auto real_path = RealPath(context_->kernel_config_file_.c_str()); + std::ifstream ifs(real_path); + if (!ifs.good()) { + MS_LOG(ERROR) << "file: " << real_path << " is not exist"; + return; + } + if (!ifs.is_open()) { + MS_LOG(ERROR) << "file: " << real_path << " open failed"; + return; + } + + nlohmann::json kernel_config_json; + ifs >> kernel_config_json; + double best_avg = std::numeric_limits::max(); + + for (const auto &item : kernel_config_json) { + if (!item.contains("model_time") || !item["model_time"].contains("avg") || !item.contains("kernel_info")) { + MS_LOG(WARNING) << "Invalid kernel config item format"; + continue; + } + double temp_avg = item["model_time"]["avg"]; + if (temp_avg < best_avg) { + best_avg = temp_avg; + kernel_info_.clear(); + for (const auto &kernel : item["kernel_info"]) { + if (!kernel.contains("name") || !kernel.contains("arch") || !kernel.contains("core_mask") || + !kernel.contains("mem_type")) { + MS_LOG(WARNING) << "Invalid kernel info format"; + continue; + } + KernelInfo kinfo; + kinfo.name = kernel["name"]; + kinfo.arch = kernel["arch"]; + kinfo.core_mask = kernel["core_mask"]; + kinfo.memory_type = kernel["mem_type"]; + kernel_info_.push_back(kinfo); + } + } + } + } std::vector NonTailCallNodes(); private: @@ -104,6 +164,18 @@ class Scheduler { int FindProviderKernel(const std::vector &in_tensors, const std::vector &out_tensors, const LiteGraph::Node *node, TypeId data_type, kernel::KernelExec **kernel); + int SelectKernelByDeviceList(const std::vector &in_tensors, const std::vector &out_tensors, + const LiteGraph::Node *node, TypeId data_type, kernel::KernelExec **kernel, + TypeId prefer_data_type); + + int SelectKernelByConfigFile(const std::vector &in_tensors, const std::vector &out_tensors, + const LiteGraph::Node *node, TypeId data_type, kernel::KernelExec **kernel, + TypeId prefer_data_type); + + int SelectKernelRandomly(const std::vector &in_tensors, const std::vector &out_tensors, + const LiteGraph::Node *node, TypeId data_type, kernel::KernelExec **kernel, + TypeId prefer_data_type); + int InitKernels(std::vector &&dst_kernels); kernel::KernelExec *SchedulePartialToKernel(const lite::LiteGraph::Node *src_node); // schedule a partial node to a subgraph_kernel @@ -187,6 +259,7 @@ class Scheduler { const std::map> *config_info_ = nullptr; std::shared_ptr shape_fusion_pass_ = nullptr; std::vector infer_subgraph_index_; + std::vector kernel_info_; }; } // namespace mindspore::lite diff --git a/mindspore-lite/src/litert/storage_pass.cc b/mindspore-lite/src/litert/storage_pass.cc new file mode 100644 index 0000000000000000000000000000000000000000..deb5b313383ec5192e7fd1121fdea86f7e8359b3 --- /dev/null +++ b/mindspore-lite/src/litert/storage_pass.cc @@ -0,0 +1,331 @@ +#include "src/litert/storage_pass.h" +#include "mindspore-lite/include/registry/register_kernel.h" +#include "mindspore-lite/src/common/tensor_util.h" +#include "mindspore-lite/src/litert/kernel_exec_util.h" + +using MEMORY_TYPE = mindspore::kernel::MEMORY_TYPE; +using KernelKey = mindspore::kernel::KernelKey; + +namespace mindspore::lite { + +/** + * @brief 对子图进行存储优化。 + * + * 对传入的子图进行存储选择操作，并在不同存储类型的算子之间插入内存拷贝算子 。 + * + * @param subgraphs 子图的指针数组，每个子图是一个KernelExec的指针 + * @param tensors Tensor指针数组，存储子图中使用的Tensor + * @param ms_ctx MindSpore的上下文指针，包含执行环境信息 + * + * @return 操作结果，如果成功返回RET_OK，否则返回RET_ERROR + */ +STATUS StoragePass(std::vector *subgraphs, std::vector *tensors, + const mindspore::Context *ms_ctx) { + for (auto subgraph : *subgraphs) { + auto arch = subgraph->desc().arch; + if (arch == kernel::KERNEL_ARCH::kDelegate || arch == kernel::KERNEL_ARCH::kCPU) { + continue; // 跳过Delegate和CPU子图 + } + auto status = ProcessSubgraph(subgraph, tensors, ms_ctx); // 处理每个子图 + if (status != RET_OK) { + return status; + } + } + return RET_OK; +} + +/** + * @brief 处理图输入或常量张量 + * + * 根据给定的图、当前算子、内存类型、输入张量、张量列表、索引和MindSpore上下文，处理图输入或常量张量。 + * 如果内存类型不是DDR，则会插入内存复制算子。 + * + * @param graph 图指针，指向需要处理的子图 + * @param origin_kernel 当前算子指针 + * @param origin_memory_type 内存类型 + * @param in_tensor 输入张量指针 + * @param tensors 张量列表指针 + * @param index 索引 + * @param ms_ctx MindSpore上下文指针 + * + * @return 处理成功返回true，否则返回false + */ +bool HandleGraphInputOrConstTensor(kernel::SubGraphKernel *graph, kernel::KernelExec *origin_kernel, + MEMORY_TYPE origin_memory_type, Tensor *in_tensor, std::vector *tensors, + size_t index, const mindspore::Context *ms_ctx) { + bool is_ddr_type = origin_memory_type == MEMORY_TYPE::kDDR; + if (!is_ddr_type) { + std::string name = in_tensor->tensor_name() + "_memcpy_to_" + origin_kernel->name(); + auto mem_copy_kernel = InsertMemoryCopy(graph, origin_kernel, name, tensors, index, ms_ctx); + if (mem_copy_kernel == nullptr) { + MS_LOG(ERROR) << "InsertMemoryCopy failed."; + return false; + } + auto mem_copy_kernel_desc = mem_copy_kernel->desc(); + mem_copy_kernel_desc.core_mask = origin_kernel->desc().core_mask; + mem_copy_kernel->set_desc(mem_copy_kernel_desc); + graph->SetInNodes(kernel::KernelExecUtil::SubgraphInputNodes(graph->nodes())); + graph->SetOutNodes(kernel::KernelExecUtil::SubgraphOutputNodes(graph->nodes())); + auto ret = graph->TopologicalSortNodes(); + if (ret != RET_OK) { + MS_LOG(ERROR) << "Topological sort kernels failed."; + return false; + } + } + return true; +} + +/** + * @brief 处理非图输入或常量张量的输入 + * + * 处理输入张量不是图输入或常量张量的情况，通过插入内存拷贝算子（Memory Copy Kernel）来处理。 + * + * @param graph 子图指针，指向当前处理的子图。 + * @param origin_kernel 原始算子执行对象指针，指向原始算子执行对象。 + * @param in_kernel 输入算子执行对象指针，指向输入算子执行对象。 + * @param tensors 张量指针向量指针，指向包含所有张量的指针向量。 + * @param index 张量索引，指定需要处理的张量在tensors中的位置。 + * @param ms_ctx MindSpore上下文指针，指向MindSpore上下文对象。 + * + * @return 如果处理成功返回true，否则返回false。 + */ +bool HandleNonGraphInputOrConstTensor(kernel::SubGraphKernel *graph, kernel::KernelExec *origin_kernel, + kernel::KernelExec *in_kernel, std::vector *tensors, size_t index, + const mindspore::Context *ms_ctx) { + std::string name = in_kernel->name() + "_memcpy_to_" + origin_kernel->name(); + auto mem_copy_kernel = InsertMemoryCopy(graph, origin_kernel, name, tensors, index, ms_ctx); + if (mem_copy_kernel == nullptr) { + MS_LOG(ERROR) << "InsertMemoryCopy failed."; + return false; + } + auto origin_kernel_desc = origin_kernel->desc(); + if (origin_kernel_desc.memory_type == MEMORY_TYPE::kPrivateMemory) { + origin_kernel_desc.core_mask = origin_kernel->desc().core_mask; + } else { + origin_kernel_desc.core_mask = in_kernel->desc().core_mask; + } + origin_kernel->set_desc(origin_kernel_desc); + auto in_tensors = origin_kernel->in_tensors(); + for (size_t j = 0; j < in_tensors.size(); j++) { + auto in_tensor = in_tensors.at(j); + if (in_tensor->tensor_name() == in_kernel->name()) { + origin_kernel->set_in_tensor(mem_copy_kernel->out_tensors()[0], j); + } + } + graph->SetInNodes(kernel::KernelExecUtil::SubgraphInputNodes(graph->nodes())); + graph->SetOutNodes(kernel::KernelExecUtil::SubgraphOutputNodes(graph->nodes())); + auto ret = graph->TopologicalSortNodes(); + if (ret != RET_OK) { + MS_LOG(ERROR) << "Topological sort kernels failed."; + return false; + } + return true; +} + +/** + * @brief 处理输入张量 + * + * 该函数用于处理输入张量，并根据张量的类型进行不同的处理。 + * + * @param graph 子图指针 + * @param origin_kernel 当前算子执行对象指针 + * @param in_kernels 输入算子执行对象指针数组 + * @param tensors 张量指针数组指针 + * @param origin_memory_type 内存类型 + * @param in_t 输入张量指针 + * @param ms_ctx MindSpore上下文指针 + * @param index 输入张量的索引 + * + * @return 如果处理成功，则返回true；否则返回false + */ +bool HandleInputTensors(kernel::SubGraphKernel *graph, kernel::KernelExec *origin_kernel, + std::vector in_kernels, std::vector *tensors, + MEMORY_TYPE origin_memory_type, Tensor *in_t, const mindspore::Context *ms_ctx, + const size_t &index) { + if (in_t->IsGraphInput() || in_t->IsConst()) { + if (!HandleGraphInputOrConstTensor(graph, origin_kernel, origin_memory_type, in_t, tensors, index, ms_ctx)) { + return false; + } + } else { + int in_kernel_index = -1; + for (size_t i = 0; i < in_kernels.size(); ++i) { + if (find(in_kernels[i]->out_tensors().begin(), in_kernels[i]->out_tensors().end(), in_t) != + in_kernels[i]->out_tensors().end()) { + in_kernel_index = i; + break; + } + } + if (in_kernel_index < 0) { + MS_LOG(ERROR) << "Can not find the kernel of input tensor: " << in_t->tensor_name(); + return false; + } + auto in_kernel = in_kernels.at(in_kernel_index); + bool need_copy = false; + +#ifdef ENBALE_DSP + MEMORY_TYPE in_memory_type = in_kernel->desc().memory_type; + KernelKey in_desc = in_kernel->desc(); + int in_core_mask = in_desc.core_mask; + int in_core_id = kernel::GetCoreIdByCoreMask(in_core_mask); + + KernelKey origin_desc = origin_kernel->desc(); + int origin_core_mask = origin_desc.core_mask; + int origin_core_id = kernel::GetCoreIdByCoreMask(origin_core_mask); + + bool is_mem_type_diff = origin_memory_type != in_memory_type; + bool is_am_mem_type_diff_core = origin_memory_type == MEMORY_TYPE::kPrivateMemory && + in_memory_type == MEMORY_TYPE::kPrivateMemory && + in_core_id != origin_core_id; + + need_copy = is_mem_type_diff || is_am_mem_type_diff_core; +#endif + + if (need_copy && !HandleNonGraphInputOrConstTensor(graph, origin_kernel, in_kernel, tensors, index, ms_ctx)) { + return false; + } + } + return true; +} + +/** + * @brief 处理算子函数 + * + * 处理传入的子图、原始算子、张量向量和上下文，插入内存拷贝操作，确保内存类型一致。 + * + * @param graph 子图指针 + * @param origin_kernel 当前算子指针 + * @param tensors 张量向量指针 + * @param ms_ctx 上下文指针 + * @return STATUS 状态码，RET_OK 表示成功，RET_ERROR 表示失败 + */ +STATUS ProcessKernel(kernel::SubGraphKernel *graph, kernel::KernelExec *origin_kernel, std::vector *tensors, + const mindspore::Context *ms_ctx) { + KernelKey origin_kernel_desc = origin_kernel->desc(); + MEMORY_TYPE origin_kernel_memory_type = origin_kernel_desc.memory_type; + auto in_kernels = origin_kernel->in_kernels(); + auto in_ts = origin_kernel->in_tensors(); + auto in_tensor_size = in_ts.size(); + for (size_t i = 0; i < in_tensor_size; i++) { + auto in_t = in_ts.at(i); + if (!HandleInputTensors(graph, origin_kernel, in_kernels, tensors, origin_kernel_memory_type, in_t, ms_ctx, i)) { + return RET_ERROR; + } + } + return RET_OK; +} + +/** + * @brief 处理子图 + * + * @param subgraph 子图指针 + * @param tensors 张量指针数组 + * @param ms_ctx MindSpore上下文指针 + * + * @return STATUS 状态码 + */ +STATUS ProcessSubgraph(kernel::KernelExec *subgraph, std::vector *tensors, const mindspore::Context *ms_ctx) { + auto graph = static_cast(subgraph); + auto kernels = graph->nodes(); + auto origin_kernel_size = kernels.size(); + for (size_t kernel_index = 0; kernel_index < origin_kernel_size; kernel_index++) { + kernel::KernelExec *origin_kernel = kernels.at(kernel_index); + auto status = ProcessKernel(graph, origin_kernel, tensors, ms_ctx); + if (status != RET_OK) { + return status; + } + } + return RET_OK; +} + +/** + * @brief 在子图中插入内存拷贝操作 + * + * 在给定的子图中，根据指定的索引，在原始算子操作后插入一个内存拷贝操作。 + * + * @param subgraph 子图指针，指向要插入内存拷贝操作的子图 + * @param origin_kernel 原始算子操作指针，指向要在其后插入内存拷贝操作的算子操作 + * @param name 内存拷贝操作的名称 + * @param all_tensors 所有张量的指针，用于存储新创建的内存拷贝张量 + * @param index 索引，指定在哪个输入张量后进行内存拷贝操作 + * @param ms_ctx MindSpore上下文指针，提供必要的上下文信息 + * + * @return 插入后的内存拷贝算子操作指针，如果插入失败则返回nullptr + */ +kernel::KernelExec *InsertMemoryCopy(kernel::SubGraphKernel *subgraph, kernel::KernelExec *origin_kernel, + std::string &name, std::vector *all_tensors, const size_t &index, + const mindspore::Context *ms_ctx) { + auto mem_copy_name = name; + auto in_tensor = origin_kernel->in_tensors().at(index); + auto out_tensor = new (std::nothrow) Tensor(in_tensor->data_type(), in_tensor->shape(), in_tensor->format()); + out_tensor->set_tensor_name(mem_copy_name); + auto mem_copy_kernel = CreateMemoryCopyFunc(in_tensor, out_tensor, mem_copy_name, ms_ctx); + if (mem_copy_kernel == nullptr) { + MS_LOG(ERROR) << "create memory copy kernel failed"; + return nullptr; + } + auto mem_copy_kernel_desc = mem_copy_kernel->desc(); + mem_copy_kernel_desc.memory_type = origin_kernel->desc().memory_type; + mem_copy_kernel->set_desc(mem_copy_kernel_desc); + all_tensors->push_back(out_tensor); + subgraph->InsertInEdge(origin_kernel, mem_copy_kernel, index); + return mem_copy_kernel; +} + +/** + * @brief 创建一个内存拷贝算子的函数。 + * + * 根据输入和输出张量，以及MindSpore上下文，创建一个内存拷贝的函数。 + * + * @param in_tensor 输入张量。 + * @param out_tensor 输出张量。 + * @param name 函数名称。 + * @param ms_ctx MindSpore上下文指针。 + * + * @return 返回创建的KernelExec指针，如果创建失败则返回nullptr。 + */ +kernel::KernelExec *CreateMemoryCopyFunc(Tensor *in_tensor, Tensor *out_tensor, std::string &name, + const mindspore::Context *ms_ctx) { + std::vector in_tensors; + std::vector out_tensors; + in_tensors.push_back(in_tensor); + out_tensors.push_back(out_tensor); + auto ms_in_tensors = LiteTensorsToMSTensors(in_tensors); + auto ms_out_tensors = LiteTensorsToMSTensors(out_tensors); + + flatbuffers::FlatBufferBuilder fbb(1024); + // create custom kernel + auto val_offset = schema::CreateCustomDirect(fbb, "Custom_FT_MemoryCopy"); + flatbuffers::Offset primitive = + schema::CreatePrimitive(fbb, static_cast(PrimType::PrimType_Custom), val_offset.o); + fbb.Finish(primitive); + const mindspore::schema::Primitive *primitive_ptr = + flatbuffers::GetRoot(fbb.GetBufferPointer()); + auto data_type = in_tensor->data_type(); + auto prim_type = mindspore::schema::PrimitiveType::PrimitiveType_Custom; + std::string kernel_arch = "DSP"; + std::string provider = "FTMatrix"; + auto arch = kernel::KERNEL_ARCH::kDSP; + + kernel::KernelKey key{arch, data_type, NHWC, prim_type, kernel_arch, provider}; + registry::KernelDesc desc{static_cast(key.data_type), key.type, key.kernel_arch, key.provider}; + auto creator = registry::RegisterKernel::GetCreator(primitive_ptr, &desc); + if (creator == nullptr) { + MS_LOG(ERROR) << "creator memory copy kernel error"; + return nullptr; + } + + auto base_kernel = creator(ms_in_tensors, ms_out_tensors, primitive_ptr, ms_ctx); + if (base_kernel != nullptr) { + auto *kernel_exec = new (std::nothrow) kernel::KernelExec(base_kernel); + if (kernel_exec != nullptr) { + kernel::KernelKey tmp_key = key; + tmp_key.arch = kernel::KERNEL_ARCH::kDSP; + kernel_exec->set_desc(tmp_key); + kernel_exec->set_name(name); + MS_LOG(DEBUG) << "create " << name << " kernel success"; + return kernel_exec; + } + } + return nullptr; +} +} // namespace mindspore::lite \ No newline at end of file diff --git a/mindspore-lite/src/litert/storage_pass.h b/mindspore-lite/src/litert/storage_pass.h new file mode 100644 index 0000000000000000000000000000000000000000..99997549deed02e1a110c00c1993ea41be339913 --- /dev/null +++ b/mindspore-lite/src/litert/storage_pass.h @@ -0,0 +1,40 @@ +#ifndef MINDSPORE_LITE_SRC_STORAGE_STORAGE_PASS_H_ +#define MINDSPORE_LITE_SRC_STORAGE_STORAGE_PASS_H_ + +#include +#include "src/executor/kernel_exec.h" +#include "src/executor/sub_graph_kernel.h" +#include "schema/ops_generated.h" +#include "schema/model_generated.h" + +namespace mindspore::lite { + +STATUS StoragePass(std::vector *subgraphs, std::vector *tensors, + const mindspore::Context *ms_ctx); + +kernel::KernelExec *InsertMemoryCopy(kernel::SubGraphKernel *subgraph, kernel::KernelExec *origin_kernel, + std::string &name, std::vector *all_tensors, const size_t &index, + const mindspore::Context *ms_ctx); + +kernel::KernelExec *CreateMemoryCopyFunc(Tensor *in_tensor, Tensor *out_tensor, std::string &name, + const mindspore::Context *ms_ctx); + +STATUS ProcessKernel(kernel::SubGraphKernel *graph, kernel::KernelExec *origin_kernel, std::vector *tensors, + const mindspore::Context *ms_ctx); + +STATUS ProcessSubgraph(kernel::KernelExec *subgraph, std::vector *tensors, const mindspore::Context *ms_ctx); + +bool HandleGraphInputOrConstTensor(kernel::SubGraphKernel *graph, kernel::KernelExec *origin_kernel, + kernel::MEMORY_TYPE origin_memory_type, Tensor *in_tensor, std::vector *tensors, + size_t index, const mindspore::Context *ms_ctx); + +bool HandleNonGraphInputOrConstTensor(kernel::SubGraphKernel *graph, kernel::KernelExec *origin_kernel, + kernel::KernelExec *in_kernel, std::vector *tensors, size_t index, + const mindspore::Context *ms_ctx); + +bool HandleInputTensors(kernel::SubGraphKernel *graph, kernel::KernelExec *origin_kernel, + std::vector in_kernels, std::vector *tensors, + kernel::MEMORY_TYPE origin_memory_type, Tensor *in_t, const mindspore::Context *ms_ctx, + const size_t &index); +} // namespace mindspore::lite +#endif // MINDSPORE_LITE_SRC_STORAGE_STORAGE_PASS_H_ \ No newline at end of file diff --git a/mindspore-lite/tools/benchmark/CMakeLists.txt b/mindspore-lite/tools/benchmark/CMakeLists.txt index 5a22fd12c0f1da42bcad036d8b0e6ae115d7fac2..4f29389ceca835686c8945ed1b73b61a26c590cb 100644 --- a/mindspore-lite/tools/benchmark/CMakeLists.txt +++ b/mindspore-lite/tools/benchmark/CMakeLists.txt @@ -58,6 +58,8 @@ else() endif() include_directories(${NNACL_DIR}/../) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) + set(COMMON_SRC ${CMAKE_CURRENT_SOURCE_DIR}/../common/flag_parser.cc ${CMAKE_CURRENT_SOURCE_DIR}/../common/string_util.cc @@ -93,7 +95,8 @@ set(BENCHMARK_SRC ${CMAKE_CURRENT_SOURCE_DIR}/main.cc ${CMAKE_CURRENT_SOURCE_DIR}/run_benchmark.cc ${CMAKE_CURRENT_SOURCE_DIR}/benchmark_base.cc - ${CMAKE_CURRENT_SOURCE_DIR}/benchmark_unified_api.cc) + ${CMAKE_CURRENT_SOURCE_DIR}/benchmark_unified_api.cc + ${CMAKE_CURRENT_SOURCE_DIR}/kernel_config.cc) if(MSLITE_ENABLE_CLOUD_FUSION_INFERENCE OR MSLITE_ENABLE_CLOUD_INFERENCE) set_property(SOURCE ${BENCHMARK_SRC} PROPERTY COMPILE_DEFINITIONS diff --git a/mindspore-lite/tools/benchmark/benchmark_base.cc b/mindspore-lite/tools/benchmark/benchmark_base.cc index 333e77ff31063f4e76cc72f142e53131c403dbd3..b839da8acba0faa29be5364b5e7ff3fb1ded5e95 100644 --- a/mindspore-lite/tools/benchmark/benchmark_base.cc +++ b/mindspore-lite/tools/benchmark/benchmark_base.cc @@ -606,6 +606,158 @@ int BenchmarkBase::Init() { return RET_OK; } +void BenchmarkBase::LoadBenchmarkConfig() { + std::string config_file_path = flags_->config_file_; + if (config_file_path.empty()) { + MS_LOG(WARNING) << "Config file path is empty, using default settings"; + return; + } + std::ifstream config_file(config_file_path); + if (!config_file.is_open()) { + MS_LOG(ERROR) << "Failed to open config file: " << config_file_path; + return; + } + + MS_LOG(INFO) << "Loading config file: " << config_file_path; + + std::string line; + bool in_benchmark_section = false; + std::string kernel_config_value; + std::string run_count_value; + + while (std::getline(config_file, line)) { + // 移除行尾的\r (处理Windows格式文件) + if (!line.empty() && line.back() == '\r') { + line.pop_back(); + } + + // 去除行首尾空格和制表符 + auto trim = [](std::string &str) { + // 删除首部空格 + size_t start = str.find_first_not_of(" \t\r\n"); + if (start == std::string::npos) { + str = ""; + return; + } + // 删除尾部空格 + size_t end = str.find_last_not_of(" \t\r\n"); + str = str.substr(start, end - start + 1); + }; + trim(line); + + MS_LOG(DEBUG) << "Processing line: \"" << line << "\""; + + // 跳过空行和注释 + if (line.empty() || line[0] == '#' || line[0] == ';') continue; + + // 处理节头 + if (line.front() == '[' && line.back() == ']') { + std::string section = line.substr(1, line.size() - 2); + trim(section); + + // 不区分大小写的节名比较 + std::string lower_section = section; + std::transform(lower_section.begin(), lower_section.end(), lower_section.begin(), ::tolower); + + MS_LOG(DEBUG) << "Found section: " << section << ", lowercased: " << lower_section; + in_benchmark_section = (lower_section == "benchmark"); + continue; + } + + // 仅处理benchmark节内的配置 + if (!in_benchmark_section) { + MS_LOG(DEBUG) << "Skipping line outside benchmark section: " << line; + continue; + } + + // 解析键值对 + size_t pos = line.find('='); + if (pos == std::string::npos) { + MS_LOG(DEBUG) << "Line has no '=' character: " << line; + continue; + } + + std::string key = line.substr(0, pos); + std::string value = line.substr(pos + 1); + trim(key); + trim(value); + + // 不区分大小写的键名比较 + std::string lower_key = key; + std::transform(lower_key.begin(), lower_key.end(), lower_key.begin(), ::tolower); + + MS_LOG(DEBUG) << "Found key-value pair: " << key << " = " << value; + + if (lower_key == "kernel_config_file") { + kernel_config_value = value; + MS_LOG(DEBUG) << "Set kernel_config_file to: " << kernel_config_value; + } else if (lower_key == "run_benchmark_count") { + run_count_value = value; + MS_LOG(DEBUG) << "Set run_benchmark_count to: " << run_count_value; + } + } + config_file.close(); + + MS_LOG(INFO) << "Finished parsing config file. kernel_config_value: \"" + << kernel_config_value << "\", run_count_value: \"" << run_count_value << "\""; + + // 4. 处理变量替换（${CURRENT_DIR}） + if (!kernel_config_value.empty()) { + // 替换${CURRENT_DIR}为实际路径 + char cwd[PATH_MAX]; + if (getcwd(cwd, sizeof(cwd)) != nullptr) { + size_t pos = 0; + std::string current_dir(cwd); + std::string placeholder = "${CURRENT_DIR}"; + + while ((pos = kernel_config_value.find(placeholder, pos)) != std::string::npos) { + kernel_config_value.replace(pos, placeholder.length(), current_dir); + pos += current_dir.length(); + } + MS_LOG(INFO) << "After variable replacement, kernel_config_file: " << kernel_config_value; + } else { + MS_LOG(WARNING) << "Failed to get current working directory"; + } + } else { + MS_LOG(WARNING) << "kernel_config_file not specified in config file"; + } + + flags_->kernel_config_file_ = kernel_config_value; + + int run_count = 1; // 默认值 + if (!run_count_value.empty()) { + try { + run_count = std::stoi(run_count_value); + if (run_count < 1) { + MS_LOG(WARNING) << "Invalid run count " << run_count_value + << ", using default value 1"; + run_count = 1; + } else { + MS_LOG(INFO) << "Using specified run count: " << run_count; + } + } catch (const std::exception &e) { + MS_LOG(WARNING) << "Invalid run count format: " << run_count_value + << ", error: " << e.what() + << ", using default value 1"; + } + } + flags_->run_benchmark_count_ = run_count; + + MS_LOG(INFO) << "Kernel config file set to: " << flags_->kernel_config_file_; + MS_LOG(INFO) << "Benchmark will run " << flags_->run_benchmark_count_ << " times"; +} +// 配置文件示例(benchmark.cfg) +/* +[benchmark] +kernel_config_file = ./model.json +run_benchmark_count = 10 +*/ + +// 使用方法 +/* +./benchmark --modelFile=your_model.ms --device=DSP --configFile=benchmark.cfg +*/ + int BenchmarkBase::PrintResult(const std::vector &title, const std::map> &result) { std::vector columnLenMax(kPrintColNum); diff --git a/mindspore-lite/tools/benchmark/benchmark_base.h b/mindspore-lite/tools/benchmark/benchmark_base.h index cd156d564b3a726dbb75da611886741786c0cc7c..360f95cea01bc1ee84946ce980ca32ce8cb01b4e 100644 --- a/mindspore-lite/tools/benchmark/benchmark_base.h +++ b/mindspore-lite/tools/benchmark/benchmark_base.h @@ -42,6 +42,7 @@ #include "ir/dtype/type_id.h" #include "schema/model_generated.h" #include "nnacl_c/op_base.h" +#include "kernel_config.h" namespace mindspore::lite { #define BENCHMARK_LOG_ERROR(str) \ @@ -210,6 +211,10 @@ class MS_API BenchmarkFlags : public virtual FlagParser { bool enable_gl_texture_ = false; bool enable_parallel_ = false; int warm_up_loop_count_ = 3; + //Config + int run_benchmark_count_ = 0; + bool enable_kernel_config_ = false; + std::string kernel_config_file_ = ""; // MarkAccuracy std::string benchmark_data_file_; std::string benchmark_data_type_ = "FLOAT"; @@ -249,6 +254,33 @@ class MS_API BenchmarkBase { int Init(); virtual int RunBenchmark() = 0; + void LoadBenchmarkConfig(); + + void GetKernelConfigApi(const std::string &filename) { + int start = filename.find_last_of("/"); + int end = filename.find_last_of("."); + std::string result_name = filename.substr(start + 1, end - start - 1); + kernel_config_ = new KernelConfig(result_name + ".json"); + } + + void DeleteKernelConfigApi() { + if (kernel_config_ != nullptr) { + delete kernel_config_; + kernel_config_ = nullptr; + } + } + + int GetRunBenchmarkCount() { + return flags_->run_benchmark_count_; + } + + bool IsEnableKernelConfig() { + if (flags_->run_benchmark_count_ > 0 && flags_->kernel_config_file_.empty()) { + flags_->enable_kernel_config_ = true; + } + return flags_->enable_kernel_config_; + } + protected: virtual int LoadInput() = 0; @@ -530,9 +562,17 @@ class MS_API BenchmarkBase { float op_cost_total_ = 0.0f; std::map> op_times_by_type_; std::map> op_times_by_name_; - std::map op_start_times_by_name_; + std::map op_start_times_by_name_; + std::map op_end_times_by_name_; + std::map>> op_input_shape_; + std::map> op_input_dtype_; + std::map op_arch_; + std::map op_custom_name_; + std::map op_core_mask_; + std::map op_mem_type_; + KernelConfig *kernel_config_; std::mutex op_times_mutex_; -#ifndef BENCHMARK_CLIP_JSON + #ifndef BENCHMARK_CLIP_JSON // dump data nlohmann::json dump_cfg_json_; #endif diff --git a/mindspore-lite/tools/benchmark/benchmark_unified_api.cc b/mindspore-lite/tools/benchmark/benchmark_unified_api.cc index 722d7f74c38944f40828082104bf4c47436b1efa..0f94ebcb2a59c6e8611f930fbf3d71aec360faa1 100644 --- a/mindspore-lite/tools/benchmark/benchmark_unified_api.cc +++ b/mindspore-lite/tools/benchmark/benchmark_unified_api.cc @@ -21,6 +21,11 @@ #include #include #include +#include +#include +#include +#include +#include #include #include #include @@ -534,6 +539,11 @@ int BenchmarkUnifiedApi::InitMSContext(const std::shared_ptr std::shared_ptr dsp_device_info = std::make_shared(); device_list.push_back(dsp_device_info); context->SetEnableParallel(flags_->enable_parallel_); + if (!flags_->kernel_config_file_.empty()) { + context->SetEnableKernelConfig(true, flags_->kernel_config_file_); + } else { + context->SetEnableKernelConfig(flags_->enable_kernel_config_); + } } // CPU priority is behind GPU and NPU @@ -544,6 +554,7 @@ int BenchmarkUnifiedApi::InitMSContext(const std::shared_ptr return RET_OK; } + #ifdef MSLITE_ENABLE_CLOUD_INFERENCE int BenchmarkUnifiedApi::CompareOutputForModelPool(std::vector *outputs) { if (outputs->empty()) { @@ -977,6 +988,23 @@ int BenchmarkUnifiedApi::MarkPerformance() { flags_->model_file_.substr(flags_->model_file_.find_last_of(DELIM_SLASH) + 1).c_str(), flags_->num_threads_, time_min / kFloatMSEC, time_max / kFloatMSEC, time_avg / kFloatMSEC); } +#ifdef ENABLE_DSP + // If kernel config is enabled, always attempt to write model/kernel info. + if (flags_->enable_kernel_config_ && kernel_config_ != nullptr) { + // decrement remaining run count used to format JSON array entries correctly + if (flags_->run_benchmark_count_ > 0) { + flags_->run_benchmark_count_--; + } + // Ensure maps exist; GetKernelConfig handles empty maps by writing empty kernel_info + kernel_config_->GetKernelConfig(time_min / kFloatMSEC, time_avg / kFloatMSEC, time_max / kFloatMSEC, op_arch_, + op_core_mask_, op_mem_type_, flags_->run_benchmark_count_); + op_times_by_name_.clear(); + op_times_by_type_.clear(); + op_arch_.clear(); + op_core_mask_.clear(); + op_mem_type_.clear(); + } +#endif return RET_OK; } @@ -1422,6 +1450,7 @@ int BenchmarkUnifiedApi::RunBenchmark() { } UpdateConfigInfo(); + #ifdef MSLITE_ENABLE_CLOUD_INFERENCE if (flags_->enable_parallel_predict_) { MS_CHECK_FALSE_MSG(RunParallelBenchmark(context) != RET_OK, RET_ERROR, "run model pool failed."); @@ -1548,6 +1577,26 @@ int BenchmarkUnifiedApi::InitTimeProfilingCallbackParameter() { op_times_by_type_[call_param.node_type].second += cost; op_times_by_name_[call_param.node_name].first++; op_times_by_name_[call_param.node_name].second += cost; +#ifdef ENABLE_DSP + if (!call_param.node_name.empty()) { + op_end_times_by_name_[call_param.node_name] = opEnd; + std::vector> in_shape; + std::vector in_dtype; + for (auto after_input : after_inputs) { + auto dtype = static_cast(after_input.DataType()); + in_shape.push_back(after_input.Shape()); + op_input_shape_[call_param.node_name] = in_shape; + in_dtype.push_back(dtype); + op_input_dtype_[call_param.node_name] = in_dtype; + } + op_arch_[call_param.node_name] = call_param.arch; + if (call_param.node_type == "Custom") { + op_custom_name_[call_param.node_name] = call_param.custom_kernel_name; + } + op_core_mask_[call_param.node_name] = call_param.core_mask; + op_mem_type_[call_param.node_name] = call_param.mem_type; + } +#endif } return true; }; @@ -1571,6 +1620,11 @@ int BenchmarkUnifiedApi::InitTimeProfilingCallbackParameter() { op_call_times_total_++; op_begin_ = GetTimeUs(); +#ifdef ENABLE_DSP + if (!call_param.node_name.empty()) { + op_start_times_by_name_[call_param.node_name] = op_begin_; + } +#endif return true; }; @@ -1596,6 +1650,26 @@ int BenchmarkUnifiedApi::InitTimeProfilingCallbackParameter() { op_times_by_type_[call_param.node_type].second += cost; op_times_by_name_[call_param.node_name].first++; op_times_by_name_[call_param.node_name].second += cost; +#ifdef ENABLE_DSP + if (!call_param.node_name.empty()) { + op_end_times_by_name_[call_param.node_name] = opEnd; + std::vector> in_shape; + std::vector in_dtype; + for (auto after_input : after_inputs) { + auto dtype = static_cast(after_input.DataType()); + in_shape.push_back(after_input.Shape()); + op_input_shape_[call_param.node_name] = in_shape; + in_dtype.push_back(dtype); + op_input_dtype_[call_param.node_name] = in_dtype; + } + op_arch_[call_param.node_name] = call_param.arch; + if (call_param.node_type == "Custom") { + op_custom_name_[call_param.node_name] = call_param.custom_kernel_name; + } + op_core_mask_[call_param.node_name] = call_param.core_mask; + op_mem_type_[call_param.node_name] = call_param.mem_type; + } +#endif return true; }; } diff --git a/mindspore-lite/tools/benchmark/kernel_config.cc b/mindspore-lite/tools/benchmark/kernel_config.cc new file mode 100644 index 0000000000000000000000000000000000000000..8446645cc38bb5b4bcbb568be080ae6f76c6577c --- /dev/null +++ b/mindspore-lite/tools/benchmark/kernel_config.cc @@ -0,0 +1,167 @@ +#include "kernel_config.h" +#include + +namespace mindspore::lite { + +static const char *kDefaultKernelConfigFilename = "model.json"; + +void KernelConfig::OpenFile() { output_file_stream_.open(filename_, std::ofstream::out | std::ofstream::trunc); } + +KernelConfig::KernelConfig(const std::string &filename) { + filename_ = filename.empty() ? kDefaultKernelConfigFilename : filename; + OpenFile(); + StartLog(); +} + +KernelConfig::KernelConfig(const char *filename_cstr) { + std::string filename(filename_cstr); + filename_ = filename.empty() ? kDefaultKernelConfigFilename : filename; + OpenFile(); + StartLog(); +} + +KernelConfig::~KernelConfig() { + EndLog(); + output_file_stream_.close(); +} + +void KernelConfig::StartLog() { + output_file_stream_ << std::string( + R"JSON([ + )JSON"); +} + +void KernelConfig::EndLog() { + output_file_stream_ << std::string(R"JSON( +])JSON"); +} + +// void KernelConfig::GetKernelConfig(float min_time, float avg_time, float max_time, +// const std::map &op_arch, +// const std::map &op_core_mask, +// const std::map &op_mem_type, int run_count) { +// output_file_stream_ << string_format(std::string(R"JSON({ +// "model_time": {"min": %.3f, "avg": %.3f, "max": %.3f}, +// "kernel_info": [)JSON"), +// min_time, avg_time, max_time); + +// size_t kernel_count = 0; +// std::map result; +// for (auto &iter : op_arch) { +// auto arch = iter.second; +// // delete pnna kernel +// if (arch == 8) { +// continue; +// } +// result[iter.first] = arch; +// } +// size_t result_size = result.size(); +// for (auto &iter : result) { +// auto kernel_name = iter.first; +// ++kernel_count; +// auto arch = op_arch.at(kernel_name); +// auto core_mask = op_core_mask.at(kernel_name); +// auto mem_type = op_mem_type.at(kernel_name); +// auto is_last = (kernel_count == result_size); +// if (!is_last) { +// output_file_stream_ << string_format(std::string(R"JSON( +// {"name": "%s","arch": %d,"core_mask": %d,"mem_type": %d},)JSON"), +// kernel_name.c_str(), arch, core_mask, mem_type); +// } else { +// output_file_stream_ << string_format(std::string(R"JSON( +// {"name": "%s","arch": %d,"core_mask": %d,"mem_type": %d})JSON"), +// kernel_name.c_str(), arch, core_mask, mem_type); +// } +// } +// if (run_count > 0) { +// output_file_stream_ << (std::string(R"JSON( +// ] +// }, +// )JSON")); +// } else { +// output_file_stream_ << (std::string(R"JSON( +// ] +// })JSON")); +// } +// } + +void KernelConfig::GetKernelConfig(float min_time, float avg_time, float max_time, + const std::map &op_arch, + const std::map &op_core_mask, + const std::map &op_mem_type, int run_count) { + // 文件输出 + output_file_stream_ << string_format(std::string(R"JSON({ + "model_time": {"min": %.3f, "avg": %.3f, "max": %.3f}, + "kernel_info": [)JSON"), + min_time, avg_time, max_time); + + // 命令行输出 - 模型时间信息 + std::cout << "\n===== Kernel Configuration Summary =====" << std::endl; + std::cout << "Model Performance (ms):" << std::endl; + std::cout << " Min Time: " << std::fixed << std::setprecision(3) << min_time << " ms" << std::endl; + std::cout << " Avg Time: " << std::fixed << std::setprecision(3) << avg_time << " ms" << std::endl; + std::cout << " Max Time: " << std::fixed << std::setprecision(3) << max_time << " ms" << std::endl; + std::cout << "Kernel Details:" << std::endl; + + size_t kernel_count = 0; + std::map result; + for (auto &iter : op_arch) { + auto arch = iter.second; + // delete pnna kernel + if (arch == 8) { + continue; + } + result[iter.first] = arch; + } + size_t result_size = result.size(); + + // 如果没有内核信息，命令行输出提示 + if (result_size == 0) { + std::cout << " No kernel information available" << std::endl; + } + + for (auto &iter : result) { + auto kernel_name = iter.first; + ++kernel_count; + auto arch = op_arch.at(kernel_name); + auto core_mask = op_core_mask.at(kernel_name); + auto mem_type = op_mem_type.at(kernel_name); + auto is_last = (kernel_count == result_size); + + // 命令行输出 - 每个内核的详细信息 + std::cout << " [" << kernel_count << "/" << result_size << "] "; + std::cout << "Kernel: " << std::left << std::setw(30) << kernel_name; + std::cout << " Arch: " << std::setw(2) << arch; + std::cout << " CoreMask: 0x" << std::hex << std::setw(4) << std::setfill('0') << core_mask; + std::cout << std::dec << std::setfill(' ') << " MemType: " << std::setw(2) << mem_type << std::endl; + + // 文件输出 + if (!is_last) { + output_file_stream_ << string_format(std::string(R"JSON( + {"name": "%s","arch": %d,"core_mask": %d,"mem_type": %d},)JSON"), + kernel_name.c_str(), arch, core_mask, mem_type); + } else { + output_file_stream_ << string_format(std::string(R"JSON( + {"name": "%s","arch": %d,"core_mask": %d,"mem_type": %d})JSON"), + kernel_name.c_str(), arch, core_mask, mem_type); + } + } + + // 命令行输出 - 总结信息 + std::cout << "Total Kernels: " << result_size << std::endl; + std::cout << "Config saved to: " << filename_ << std::endl; + std::cout << "=========================================" << std::endl << std::endl; + + // 文件输出 - 结尾 + if (run_count > 0) { + output_file_stream_ << (std::string(R"JSON( + ] + }, + )JSON")); + } else { + output_file_stream_ << (std::string(R"JSON( + ] + })JSON")); + } +} +} // namespace mindspore::lite \ No newline at end of file diff --git a/mindspore-lite/tools/benchmark/kernel_config.h b/mindspore-lite/tools/benchmark/kernel_config.h new file mode 100644 index 0000000000000000000000000000000000000000..c2e36ec102897fb1664b5416a6a5fc367221fc2c --- /dev/null +++ b/mindspore-lite/tools/benchmark/kernel_config.h @@ -0,0 +1,43 @@ +#ifndef MINDSPORE_LITE_TOOLS_BENCHMARK_KERNEL_CONFIG_H_ +#define MINDSPORE_LITE_TOOLS_BENCHMARK_KERNEL_CONFIG_H_ + +#include +#include +#include +#include +#include +#include +#include "mindspore-lite/src/common/log.h" + +namespace mindspore::lite { + +class KernelConfig { + public: + KernelConfig(const std::string &filename); + KernelConfig(const char *filename); + ~KernelConfig(); + std::string filename() { return filename_; } + void GetKernelConfig(float min_time, float avg_time, float max_time, const std::map &op_arch, + const std::map &op_core_mask, const std::map &op_mem_type, + int run_count); + + template + std::string string_format(const std::string &format, Args... args) { + int size_s = std::snprintf(nullptr, 0, format.c_str(), args...) + 1; // Extra space for '\0' + auto size = static_cast(size_s); + auto buf = std::make_unique(size); + std::snprintf(buf.get(), size, format.c_str(), args...); + return std::string(buf.get(), size - 1); // exclude the '\0' + } + + private: + void OpenFile(); + void StartLog(); + void EndLog(); + + std::string filename_; + std::ofstream output_file_stream_; +}; +} // namespace mindspore::lite + +#endif // MINDSPORE_LITE_TOOLS_BENCHMARK_KERNEL_CONFIG_H_ \ No newline at end of file diff --git a/mindspore-lite/tools/benchmark/run_benchmark.cc b/mindspore-lite/tools/benchmark/run_benchmark.cc index c13d90dd0513cc9c6e7298c46260adce570db2e1..5083fddf0a870718be7cd4db0f7494514a054dee 100644 --- a/mindspore-lite/tools/benchmark/run_benchmark.cc +++ b/mindspore-lite/tools/benchmark/run_benchmark.cc @@ -71,9 +71,26 @@ int RunBenchmark(int argc, const char **argv) { delete benchmark; return RET_ERROR; } +#ifdef ENABLE_DSP + if(!flags.config_file_.empty()) + benchmark->LoadBenchmarkConfig(); // from benchmark.cfg load kernel_config_file and run_benchmark_count into flags auto model_name = flags.model_file_.substr(flags.model_file_.find_last_of(DELIM_SLASH) + 1); + int run_benchmark_count = benchmark->GetRunBenchmarkCount(); + bool enable_kernel_config = benchmark->IsEnableKernelConfig(); + if (enable_kernel_config) { + benchmark->GetKernelConfigApi(model_name); + } + for (int i = 0; i < run_benchmark_count; i++) { + status = benchmark->RunBenchmark(); + } + + if (enable_kernel_config) { + benchmark->DeleteKernelConfigApi(); + } +#else status = benchmark->RunBenchmark(); +#endif if (status != 0) { BENCHMARK_LOG_ERROR("Run Benchmark " << model_name << " Failed : " << status); delete benchmark; diff --git a/mindspore-lite/tools/converter/adapter/dsp/infer/custom_memory_copy_infer.cc b/mindspore-lite/tools/converter/adapter/dsp/infer/custom_memory_copy_infer.cc new file mode 100644 index 0000000000000000000000000000000000000000..1625e846e41827452e654c615668cc519885151c --- /dev/null +++ b/mindspore-lite/tools/converter/adapter/dsp/infer/custom_memory_copy_infer.cc @@ -0,0 +1,47 @@ +/** + * Copyright 2025 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "tools/converter/adapter/dsp/infer/custom_memory_copy_infer.h" +#include +#include +#include "include/api/status.h" +#include "include/registry/register_kernel_interface.h" +#include "tools/converter/adapter/dsp/infer/custom_common.h" + +namespace mindspore::lite { + +Status CustomMemoryCopyInfer::Infer(std::vector *inputs, std::vector *outputs, + const schema::Primitive *primitive) { + // if (inputs->size() != 1 || outputs->size() != 1) { + // return kLiteInferInvalid; + // } + outputs->front().SetFormat(outputs->front().format()); + outputs->front().SetDataType(outputs->front().DataType()); + auto ret = common::CheckIsDynamicShape(*inputs); + if (ret == kLiteInferInvalid) { + outputs->front().SetShape({-1}); // shape{-1} shows that shape need to be inferred when running. + return kLiteInferInvalid; + } else if (ret != kSuccess) { + return kLiteError; + } + auto input_shape = inputs->front().Shape(); + outputs->front().SetShape(inputs->front().Shape()); + return kSuccess; +} + +std::shared_ptr CustomMemoryCopyInferCreator() { return std::make_shared(); } +REGISTER_CUSTOM_KERNEL_INTERFACE(FTMatrix, Custom_FT_MemoryCopy, CustomMemoryCopyInferCreator) +} // namespace mindspore::lite \ No newline at end of file diff --git a/mindspore-lite/tools/converter/adapter/dsp/infer/custom_memory_copy_infer.h b/mindspore-lite/tools/converter/adapter/dsp/infer/custom_memory_copy_infer.h new file mode 100644 index 0000000000000000000000000000000000000000..2af0d9ac8bd9d1a691970c808d9df6b71a3a6cb7 --- /dev/null +++ b/mindspore-lite/tools/converter/adapter/dsp/infer/custom_memory_copy_infer.h @@ -0,0 +1,33 @@ +/** + * Copyright 2025 Huawei Technologies Co., Ltd + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef MINDSPORE_LITE_TOOLS_CONVERTER_ADAPTER_DSP_INFER_CUSTOM_MEMORY_COPY_INFER_H +#define MINDSPORE_LITE_TOOLS_CONVERTER_ADAPTER_DSP_INFER_CUSTOM_MEMORY_COPY_INFER_H + +#include "tools/converter/adapter/dsp/infer/custom_infer.h" +#include +#include + +namespace mindspore::lite { +class CustomMemoryCopyInfer : public CustomInterface { + public: + CustomMemoryCopyInfer() = default; + ~CustomMemoryCopyInfer() = default; + Status Infer(std::vector *inputs, std::vector *outputs, + const mindspore::schema::Primitive *primitive) override; +}; +} // namespace mindspore::lite +#endif // MINDSPORE_LITE_TOOLS_CONVERTER_ADAPTER_DSP_INFER_CUSTOM_MEMORY_COPY_INFER_H