Skip to content

[NPUW] Update NPUW caching properties #31876

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
+66 −64
Open

[NPUW] Update NPUW caching properties #31876

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
3330f75
Update NPUW caching properties
smirnov-alexey Aug 26, 2025
5e5d992
Add macro
smirnov-alexey Aug 27, 2025
68e7da5
Fix build
smirnov-alexey Aug 27, 2025
ac29563
Fix clang format
smirnov-alexey Aug 27, 2025
e84c4d1
Merge branch 'master' into as/npuw_update_caching_props
smirnov-alexey Aug 28, 2025
8895928
Merge branch 'master' into as/npuw_update_caching_props
smirnov-alexey Aug 29, 2025
3f51b10
Merge branch 'master' into as/npuw_update_caching_props
smirnov-alexey Sep 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 39 additions & 62 deletions src/plugins/intel_npu/src/al/include/intel_npu/config/npuw.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,37 @@ void registerNPUWLLMOptions(OptionsDesc& desc);
} \
};

#define DEFINE_ANYMAP_OPT(Name, PropertyKey) \
struct Name final : OptionBase<Name, ov::AnyMap> { \
static std::string_view key() { \
return ov::intel_npu::npuw::llm::prefill_config.name(); \
} \
\
static constexpr std::string_view getTypeName() { \
return "::intel_npu::" #PropertyKey; \
} \
\
static ov::AnyMap defaultValue() { \
return {}; \
} \
\
static ov::AnyMap parse(std::string_view val) { \
return ov::npuw::s11n::stringToAnyMap(std::string(val)); \
} \
\
static std::string toString(const ov::AnyMap& val) { \
return ov::npuw::s11n::anyMapToString(val); \
} \
\
static OptionMode mode() { \
return OptionMode::RunTime; \
} \
\
static bool isPublic() { \
return false; \
} \
};

DEFINE_OPT(NPU_USE_NPUW, bool, false, use_npuw, RunTime);
DEFINE_OPT(NPUW_DEVICES, std::string, "NPU,CPU", npuw::devices, RunTime);
DEFINE_OPT(NPUW_SUBMODEL_DEVICE, std::string, "", npuw::submodel_device, RunTime);
Expand Down Expand Up @@ -109,10 +140,16 @@ DEFINE_OPT(NPUW_LLM_SEQ_LEN_DIM, uint32_t, 2, npuw::llm::seq_len_dim, RunTime);
DEFINE_OPT(NPUW_LLM_MAX_PROMPT_LEN, uint32_t, 1024, npuw::llm::max_prompt_len, RunTime);
DEFINE_OPT(NPUW_LLM_MIN_RESPONSE_LEN, uint32_t, 128, npuw::llm::min_response_len, RunTime);
DEFINE_OPT(NPUW_LLM_OPTIMIZE_V_TENSORS, bool, true, npuw::llm::optimize_v_tensors, RunTime);
DEFINE_OPT(NPUW_LLM_CACHE_ROPE, bool, true, npuw::llm::cache_rope, CompileTime);
DEFINE_OPT(NPUW_LLM_CACHE_ROPE, bool, true, npuw::llm::cache_rope, RunTime);
Comment on lines -112 to +143
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It impacts the IR that's being compiled, how it is a runtime option?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

All of our properties are RunTime. CompileTime is used for the NPU compiler. Also caching property won't work if it's not RunTime since we would to define isAvailable in filteredConfig somewhere in the plugin

DEFINE_OPT(NPUW_LLM_PREFILL_CHUNK_SIZE, uint64_t, 1024, npuw::llm::prefill_chunk_size, RunTime);
DEFINE_OPT(NPUW_LLM_SHARED_HEAD, bool, true, npuw::llm::shared_lm_head, CompileTime);
DEFINE_OPT(NPUW_LLM_SHARED_HEAD, bool, true, npuw::llm::shared_lm_head, RunTime);
Comment on lines -114 to +145
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same thing here

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Answered above

DEFINE_OPT(NPUW_LLM_MAX_LORA_RANK, uint32_t, 32, npuw::llm::max_lora_rank, RunTime);
DEFINE_ANYMAP_OPT(NPUW_LLM_PREFILL_CONFIG, npuw::llm::prefill_config);
DEFINE_ANYMAP_OPT(NPUW_LLM_ADDITIONAL_PREFILL_CONFIG, npuw::llm::additional_prefill_config);
DEFINE_ANYMAP_OPT(NPUW_LLM_GENERATE_CONFIG, npuw::llm::generate_config);
DEFINE_ANYMAP_OPT(NPUW_LLM_ADDITIONAL_GENERATE_CONFIG, npuw::llm::additional_generate_config);
DEFINE_ANYMAP_OPT(NPUW_LLM_SHARED_LM_HEAD_CONFIG, npuw::llm::shared_lm_head_config);
DEFINE_ANYMAP_OPT(NPUW_LLM_ADDITIONAL_SHARED_LM_HEAD_CONFIG, npuw::llm::additional_shared_lm_head_config);

namespace npuw {
namespace llm {
Expand Down Expand Up @@ -216,64 +253,4 @@ struct NPUW_LLM_GENERATE_HINT final : OptionBase<NPUW_LLM_GENERATE_HINT, ::intel
return false;
}
};

struct NPUW_LLM_PREFILL_CONFIG final : OptionBase<NPUW_LLM_PREFILL_CONFIG, ov::AnyMap> {
static std::string_view key() {
return ov::intel_npu::npuw::llm::prefill_config.name();
}

static constexpr std::string_view getTypeName() {
return "::intel_npu::npuw::llm::prefill_config";
}

static ov::AnyMap defaultValue() {
return {};
}

static ov::AnyMap parse(std::string_view val) {
return ov::npuw::s11n::stringToAnyMap(std::string(val));
}

static std::string toString(const ov::AnyMap& val) {
return ov::npuw::s11n::anyMapToString(val);
}

static OptionMode mode() {
return OptionMode::RunTime;
}

static bool isPublic() {
return false;
}
};

struct NPUW_LLM_GENERATE_CONFIG final : OptionBase<NPUW_LLM_GENERATE_CONFIG, ov::AnyMap> {
static std::string_view key() {
return ov::intel_npu::npuw::llm::generate_config.name();
}

static constexpr std::string_view getTypeName() {
return "::intel_npu::npuw::llm::generate_config";
}

static ov::AnyMap defaultValue() {
return {};
}

static ov::AnyMap parse(std::string_view val) {
return ov::npuw::s11n::stringToAnyMap(std::string(val));
}

static std::string toString(const ov::AnyMap& val) {
return ov::npuw::s11n::anyMapToString(val);
}

static OptionMode mode() {
return OptionMode::RunTime;
}

static bool isPublic() {
return false;
}
};
} // namespace intel_npu
10 changes: 9 additions & 1 deletion src/plugins/intel_npu/src/plugin/include/properties.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,10 +113,18 @@ class Properties final {
ov::intel_npu::npuw::llm::max_prompt_len.name(),
ov::intel_npu::npuw::llm::min_response_len.name(),
ov::intel_npu::npuw::llm::optimize_v_tensors.name(),
ov::intel_npu::npuw::llm::cache_rope.name(),
ov::intel_npu::npuw::llm::prefill_chunk_size.name(),
ov::intel_npu::npuw::llm::shared_lm_head.name(),
ov::intel_npu::npuw::llm::max_lora_rank.name(),
ov::intel_npu::npuw::llm::prefill_hint.name(),
ov::intel_npu::npuw::llm::prefill_config.name(),
ov::intel_npu::npuw::llm::additional_prefill_config.name(),
ov::intel_npu::npuw::llm::generate_hint.name(),
ov::intel_npu::npuw::llm::generate_config.name()};
ov::intel_npu::npuw::llm::generate_config.name(),
ov::intel_npu::npuw::llm::additional_generate_config.name(),
ov::intel_npu::npuw::llm::shared_lm_head_config.name(),
ov::intel_npu::npuw::llm::additional_shared_lm_head_config.name()};

const std::vector<ov::PropertyName> _internalSupportedProperties = {ov::internal::caching_properties.name(),
ov::internal::caching_with_mmap.name()};
Expand Down
7 changes: 7 additions & 0 deletions src/plugins/intel_npu/src/plugin/src/plugin.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,10 +332,17 @@ void Plugin::init_options() {
REGISTER_OPTION(NPUW_LLM_MIN_RESPONSE_LEN);
REGISTER_OPTION(NPUW_LLM_OPTIMIZE_V_TENSORS);
REGISTER_OPTION(NPUW_LLM_CACHE_ROPE);
REGISTER_OPTION(NPUW_LLM_PREFILL_CHUNK_SIZE);
REGISTER_OPTION(NPUW_LLM_SHARED_HEAD);
REGISTER_OPTION(NPUW_LLM_MAX_LORA_RANK);
REGISTER_OPTION(NPUW_LLM_PREFILL_HINT);
REGISTER_OPTION(NPUW_LLM_PREFILL_CONFIG);
REGISTER_OPTION(NPUW_LLM_ADDITIONAL_PREFILL_CONFIG);
REGISTER_OPTION(NPUW_LLM_GENERATE_HINT);
REGISTER_OPTION(NPUW_LLM_GENERATE_CONFIG);
REGISTER_OPTION(NPUW_LLM_ADDITIONAL_GENERATE_CONFIG);
REGISTER_OPTION(NPUW_LLM_SHARED_LM_HEAD_CONFIG);
REGISTER_OPTION(NPUW_LLM_ADDITIONAL_SHARED_LM_HEAD_CONFIG);
}

void Plugin::filter_config_by_compiler_support(FilteredConfig& cfg) const {
Expand Down
12 changes: 11 additions & 1 deletion src/plugins/intel_npu/src/plugin/src/properties.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -448,11 +448,21 @@ void Properties::registerPluginProperties() {
TRY_REGISTER_SIMPLE_PROPERTY(ov::intel_npu::npuw::llm::max_prompt_len, NPUW_LLM_MAX_PROMPT_LEN);
TRY_REGISTER_SIMPLE_PROPERTY(ov::intel_npu::npuw::llm::min_response_len, NPUW_LLM_MIN_RESPONSE_LEN);
TRY_REGISTER_SIMPLE_PROPERTY(ov::intel_npu::npuw::llm::optimize_v_tensors, NPUW_LLM_OPTIMIZE_V_TENSORS);
TRY_REGISTER_SIMPLE_PROPERTY(ov::intel_npu::npuw::llm::cache_rope, NPUW_LLM_CACHE_ROPE);
TRY_REGISTER_SIMPLE_PROPERTY(ov::intel_npu::npuw::llm::prefill_chunk_size, NPUW_LLM_PREFILL_CHUNK_SIZE);
TRY_REGISTER_SIMPLE_PROPERTY(ov::intel_npu::npuw::llm::shared_lm_head, NPUW_LLM_SHARED_HEAD);
TRY_REGISTER_SIMPLE_PROPERTY(ov::intel_npu::npuw::llm::max_lora_rank, NPUW_LLM_MAX_LORA_RANK);
TRY_REGISTER_SIMPLE_PROPERTY(ov::intel_npu::npuw::llm::prefill_hint, NPUW_LLM_PREFILL_HINT);
TRY_REGISTER_SIMPLE_PROPERTY(ov::intel_npu::npuw::llm::prefill_config, NPUW_LLM_PREFILL_CONFIG);
TRY_REGISTER_SIMPLE_PROPERTY(ov::intel_npu::npuw::llm::additional_prefill_config,
NPUW_LLM_ADDITIONAL_PREFILL_CONFIG);
TRY_REGISTER_SIMPLE_PROPERTY(ov::intel_npu::npuw::llm::generate_hint, NPUW_LLM_GENERATE_HINT);
TRY_REGISTER_SIMPLE_PROPERTY(ov::intel_npu::npuw::llm::generate_config, NPUW_LLM_GENERATE_CONFIG);
TRY_REGISTER_SIMPLE_PROPERTY(ov::intel_npu::npuw::llm::cache_rope, NPUW_LLM_CACHE_ROPE);
TRY_REGISTER_SIMPLE_PROPERTY(ov::intel_npu::npuw::llm::additional_generate_config,
NPUW_LLM_ADDITIONAL_GENERATE_CONFIG);
TRY_REGISTER_SIMPLE_PROPERTY(ov::intel_npu::npuw::llm::shared_lm_head_config, NPUW_LLM_SHARED_LM_HEAD_CONFIG);
TRY_REGISTER_SIMPLE_PROPERTY(ov::intel_npu::npuw::llm::additional_shared_lm_head_config,
NPUW_LLM_ADDITIONAL_SHARED_LM_HEAD_CONFIG);

// 2. Metrics (static device and enviroment properties)
// ========
Expand Down
Loading