From 519d098bca4f3a851c5c5a2a8597ee3675f6ced1 Mon Sep 17 00:00:00 2001 From: Christian Nilsendahl Date: Fri, 28 Oct 2022 15:23:44 +0200 Subject: [PATCH 1/8] Multi type libs default data Loading multiple type libraries with default data resulted in DL_ERROR_OUT_OF_DEFAULT_VALUE_SLOTS. --- include/dl/dl.h | 1 - src/dl.cpp | 1 - src/dl_typelib_read_bin.cpp | 25 +++++++++++++------------ tests/dl_tests_typelib.cpp | 20 ++++++++++++++++++++ 4 files changed, 33 insertions(+), 14 deletions(-) diff --git a/include/dl/dl.h b/include/dl/dl.h index fae87e5..f37d0f0 100644 --- a/include/dl/dl.h +++ b/include/dl/dl.h @@ -67,7 +67,6 @@ typedef enum DL_NODISCARD DL_ERROR_OUT_OF_LIBRARY_MEMORY, DL_ERROR_OUT_OF_INSTANCE_MEMORY, DL_ERROR_DYNAMIC_SIZE_TYPES_AND_NO_INSTANCE_ALLOCATOR, - DL_ERROR_OUT_OF_DEFAULT_VALUE_SLOTS, DL_ERROR_TYPE_MISMATCH, DL_ERROR_TYPE_NOT_FOUND, DL_ERROR_BUFFER_TOO_SMALL, diff --git a/src/dl.cpp b/src/dl.cpp index dac1548..d088600 100644 --- a/src/dl.cpp +++ b/src/dl.cpp @@ -503,7 +503,6 @@ const char* dl_error_to_string( dl_error_t error ) DL_ERR_TO_STR(DL_ERROR_OUT_OF_LIBRARY_MEMORY); DL_ERR_TO_STR(DL_ERROR_OUT_OF_INSTANCE_MEMORY); DL_ERR_TO_STR(DL_ERROR_DYNAMIC_SIZE_TYPES_AND_NO_INSTANCE_ALLOCATOR); - DL_ERR_TO_STR(DL_ERROR_OUT_OF_DEFAULT_VALUE_SLOTS); DL_ERR_TO_STR(DL_ERROR_TYPE_MISMATCH); DL_ERR_TO_STR(DL_ERROR_TYPE_NOT_FOUND); DL_ERR_TO_STR(DL_ERROR_BUFFER_TOO_SMALL); diff --git a/src/dl_typelib_read_bin.cpp b/src/dl_typelib_read_bin.cpp index c673b8f..695432a 100644 --- a/src/dl_typelib_read_bin.cpp +++ b/src/dl_typelib_read_bin.cpp @@ -2,21 +2,16 @@ #include "dl_internal_util.h" #include "dl_types.h" -static dl_error_t dl_internal_load_type_library_defaults( dl_ctx_t dl_ctx, +static void dl_internal_load_type_library_defaults( dl_ctx_t dl_ctx, const uint8_t* default_data, unsigned int default_data_size ) { - if( default_data_size == 0 ) return DL_ERROR_OK; + if( default_data_size == 0 ) + return; - if( dl_ctx->default_data != 0x0 ) - return DL_ERROR_OUT_OF_DEFAULT_VALUE_SLOTS; - - dl_ctx->default_data = (uint8_t*)dl_alloc( &dl_ctx->alloc, default_data_size ); - dl_ctx->default_data_size = default_data_size; - - memcpy( dl_ctx->default_data, default_data, default_data_size ); - - return DL_ERROR_OK; + dl_ctx->default_data = (uint8_t*)dl_realloc( &dl_ctx->alloc, dl_ctx->default_data, default_data_size + dl_ctx->default_data_size, dl_ctx->default_data_size ); + memcpy( dl_ctx->default_data + dl_ctx->default_data_size, default_data, default_data_size ); + dl_ctx->default_data_size += default_data_size; } static void dl_internal_read_typelibrary_header( dl_typelib_header* header, const uint8_t* data ) @@ -141,8 +136,13 @@ dl_error_t dl_context_load_type_library( dl_ctx_t dl_ctx, const unsigned char* l } for( unsigned int i = 0; i < header.member_count; ++i ) + { dl_ctx->member_descs[ dl_ctx->member_count + i ].name += td_str_offset; + if( dl_ctx->member_descs[dl_ctx->member_count + i].default_value_offset != UINT32_MAX ) + dl_ctx->member_descs[dl_ctx->member_count + i].default_value_offset += (uint32_t)dl_ctx->default_data_size; + } + for( unsigned int i = 0; i < header.enum_count; ++i ) { dl_ctx->enum_descs[ dl_ctx->enum_count + i ].name += td_str_offset; @@ -181,5 +181,6 @@ dl_error_t dl_context_load_type_library( dl_ctx_t dl_ctx, const unsigned char* l dl_ctx->enum_alias_capacity = dl_ctx->enum_alias_count; dl_ctx->typedata_strings_cap = dl_ctx->typedata_strings_size; - return dl_internal_load_type_library_defaults( dl_ctx, lib_data + defaults_offset, header.default_value_size ); + dl_internal_load_type_library_defaults( dl_ctx, lib_data + defaults_offset, header.default_value_size ); + return DL_ERROR_OK; } diff --git a/tests/dl_tests_typelib.cpp b/tests/dl_tests_typelib.cpp index 6b3437c..1253e6b 100644 --- a/tests/dl_tests_typelib.cpp +++ b/tests/dl_tests_typelib.cpp @@ -441,3 +441,23 @@ TEST_F( DLTypeLib, enum_in_2_tlds ) // enum that do not fit in type! +TEST_F( DLTypeLib, defaults_in_2_tlds ) +{ + // capturing bug where default values in a second tld would generate DL_ERROR_OUT_OF_DEFAULT_VALUE_SLOTS if the + // first tld had default values. + + const char typelib1[] = STRINGIFY({ "module" : "tl1", "types" : { "tl1_type" : { "members" : [ { "name" : "m1", "type" : "fp32", "default" : 1.0 } ] } } }); + const char typelib2[] = STRINGIFY({ "module" : "tl2", "types" : { "tl2_type" : { "members" : [ { "name" : "m2", "type" : "fp32", "default" : 2.0 } ] } } }); + + size_t tl1_size; + size_t tl2_size; + uint8_t* tl1 = test_pack_txt_type_lib( typelib1, sizeof(typelib1)-1, &tl1_size ); + uint8_t* tl2 = test_pack_txt_type_lib( typelib2, sizeof(typelib2)-1, &tl2_size ); + + EXPECT_DL_ERR_OK( dl_context_load_type_library( ctx, tl1, tl1_size ) ); + EXPECT_DL_ERR_OK( dl_context_load_type_library( ctx, tl2, tl2_size ) ); + + free(tl1); + free(tl2); +} + From f1eaddfa196f0ed856c95898dac9c674d08882a1 Mon Sep 17 00:00:00 2001 From: Christian Nilsendahl Date: Mon, 31 Oct 2022 11:49:53 +0100 Subject: [PATCH 2/8] Exposed default values through reflection --- include/dl/dl_reflect.h | 1 + src/dl_reflect.cpp | 16 +++++++++++++ src/dl_txt_pack.cpp | 2 +- src/dl_typelib_read_bin.cpp | 20 ++++++++++++++-- src/dl_typelib_read_txt.cpp | 15 ++++++++++++ src/dl_typelib_write_bin.cpp | 15 ++++++++++++ src/dl_types.h | 1 + tests/dl_tests_base.cpp | 2 +- tests/dl_tests_reflect.cpp | 46 ++++++++++++++++++++++++++++++++---- tests/unittest2.tld | 5 ++++ 10 files changed, 114 insertions(+), 9 deletions(-) diff --git a/include/dl/dl_reflect.h b/include/dl/dl_reflect.h index bd97976..605ad15 100755 --- a/include/dl/dl_reflect.h +++ b/include/dl/dl_reflect.h @@ -48,6 +48,7 @@ typedef struct dl_member_info { const char* name; const char* comment; + const void* default_data; dl_type_atom_t atom; dl_type_storage_t storage; dl_typeid_t type_id; diff --git a/src/dl_reflect.cpp b/src/dl_reflect.cpp index b22d8d9..441f8e6 100755 --- a/src/dl_reflect.cpp +++ b/src/dl_reflect.cpp @@ -1,6 +1,7 @@ /* copyright (c) 2010 Fredrik Kihlander, see LICENSE for more info */ #include "dl_types.h" +#include "dl_patch_ptr.h" #include
@@ -116,12 +117,27 @@ dl_error_t DL_DLL_EXPORT dl_reflect_get_type_members( dl_ctx_t dl_ctx, dl_typeid if(members_size < type->member_count) return DL_ERROR_BUFFER_TOO_SMALL; + if (!dl_ctx->default_data_patched) + { + for( unsigned int i = 0; i < dl_ctx->member_count; ++i ) + { + dl_member_desc& member_desc = dl_ctx->member_descs[ i ]; + if( member_desc.default_value_size && member_desc.default_value_size != member_desc.size[DL_PTR_SIZE_HOST] ) + { + uint8_t* default_data = dl_ctx->default_data + member_desc.default_value_offset; + dl_internal_patch_member( dl_ctx, &member_desc, default_data, 0, (uintptr_t)default_data ); + } + } + dl_ctx->default_data_patched = true; // This is not thread safe. It would be better to avoid moving around default data and keep it patched like meta data + } + for( uint32_t member_index = 0; member_index < type->member_count; ++member_index ) { const dl_member_desc* member = dl_get_type_member( dl_ctx, type, member_index ); out_members[member_index].name = dl_internal_member_name( dl_ctx, member ); out_members[member_index].comment = dl_internal_member_comment( dl_ctx, member ); + out_members[member_index].default_data = reinterpret_cast(dl_ctx->default_data + member->default_value_offset); out_members[member_index].atom = member->AtomType(); out_members[member_index].storage = member->StorageType(); out_members[member_index].type_id = member->type_id; diff --git a/src/dl_txt_pack.cpp b/src/dl_txt_pack.cpp index 0a3e8cc..867b1e8 100755 --- a/src/dl_txt_pack.cpp +++ b/src/dl_txt_pack.cpp @@ -782,7 +782,7 @@ static void dl_txt_pack_write_default_value( dl_ctx_t dl_ctx, uint8_t* member_data = packctx->writer->data + member_pos; if( !packctx->writer->dummy ) - dl_internal_patch_member( dl_ctx, member, member_data, (uintptr_t)packctx->writer->data, subdata_pos - member_size ); + dl_internal_patch_member( dl_ctx, member, member_data, (uintptr_t)packctx->writer->data, subdata_pos - member_size - (dl_ctx->default_data_patched ? (uintptr_t)member_default_value : 0) ); } } diff --git a/src/dl_typelib_read_bin.cpp b/src/dl_typelib_read_bin.cpp index 695432a..b45ed6e 100644 --- a/src/dl_typelib_read_bin.cpp +++ b/src/dl_typelib_read_bin.cpp @@ -1,10 +1,11 @@ #include
#include "dl_internal_util.h" #include "dl_types.h" +#include "dl_patch_ptr.h" static void dl_internal_load_type_library_defaults( dl_ctx_t dl_ctx, - const uint8_t* default_data, - unsigned int default_data_size ) + const uint8_t* default_data, + unsigned int default_data_size ) { if( default_data_size == 0 ) return; @@ -76,6 +77,20 @@ dl_error_t dl_context_load_type_library( dl_ctx_t dl_ctx, const unsigned char* l { if(lib_data_size < sizeof(dl_typelib_header)) return DL_ERROR_MALFORMED_DATA; + + if (dl_ctx->default_data_patched) + { + for( unsigned int i = 0; i < dl_ctx->member_count; ++i ) + { + dl_member_desc& member_desc = dl_ctx->member_descs[ i ]; + if( member_desc.default_value_size && member_desc.default_value_size != member_desc.size[DL_PTR_SIZE_HOST] ) + { + uint8_t* default_data = dl_ctx->default_data + member_desc.default_value_offset; + dl_internal_patch_member( dl_ctx, &member_desc, default_data, 0, static_cast(-(ptrdiff_t)default_data) ); + } + } + dl_ctx->default_data_patched = false; // This is not thread safe. It would be better to avoid moving around default data and keep it patched like meta data + } dl_typelib_header header; dl_internal_read_typelibrary_header(&header, lib_data); @@ -182,5 +197,6 @@ dl_error_t dl_context_load_type_library( dl_ctx_t dl_ctx, const unsigned char* l dl_ctx->typedata_strings_cap = dl_ctx->typedata_strings_size; dl_internal_load_type_library_defaults( dl_ctx, lib_data + defaults_offset, header.default_value_size ); + return DL_ERROR_OK; } diff --git a/src/dl_typelib_read_txt.cpp b/src/dl_typelib_read_txt.cpp index 77504c8..929ca7d 100644 --- a/src/dl_typelib_read_txt.cpp +++ b/src/dl_typelib_read_txt.cpp @@ -4,6 +4,7 @@ #include "dl_types.h" #include "dl_alloc.h" #include "dl_txt_read.h" +#include "dl_patch_ptr.h" #include // strtoul #include @@ -1187,6 +1188,20 @@ static const dl_type_desc* dl_internal_member_owner( dl_ctx_t ctx, const dl_memb static void dl_context_load_txt_type_library_inner( dl_ctx_t ctx, dl_txt_read_ctx* read_state ) { + if (ctx->default_data_patched) + { + for( unsigned int i = 0; i < ctx->member_count; ++i ) + { + dl_member_desc& member_desc = ctx->member_descs[ i ]; + if( member_desc.default_value_size && member_desc.default_value_size != member_desc.size[DL_PTR_SIZE_HOST] ) + { + uint8_t* default_data = ctx->default_data + member_desc.default_value_offset; + dl_internal_patch_member( ctx, &member_desc, default_data, 0, static_cast(-(ptrdiff_t)default_data) ); + } + } + ctx->default_data_patched = false; // This is not thread safe. It would be better to avoid moving around default data and keep it patched like meta data + } + #if defined(_MSC_VER ) #pragma warning(push) #pragma warning(disable:4611) diff --git a/src/dl_typelib_write_bin.cpp b/src/dl_typelib_write_bin.cpp index a2fd8e9..60fa968 100644 --- a/src/dl_typelib_write_bin.cpp +++ b/src/dl_typelib_write_bin.cpp @@ -1,5 +1,6 @@ #include
#include "dl_binary_writer.h" +#include "dl_patch_ptr.h" dl_error_t dl_context_write_type_library( dl_ctx_t dl_ctx, unsigned char* out_lib, size_t out_lib_size, size_t* produced_bytes ) { @@ -19,6 +20,20 @@ dl_error_t dl_context_write_type_library( dl_ctx_t dl_ctx, unsigned char* out_li header.default_value_size = (uint32_t)dl_ctx->default_data_size; header.typeinfo_strings_size = (uint32_t)dl_ctx->typedata_strings_size; header.c_includes_size = (uint32_t)dl_ctx->c_includes_size; + + if (dl_ctx->default_data_patched) + { + for( unsigned int i = 0; i < dl_ctx->member_count; ++i ) + { + dl_member_desc& member_desc = dl_ctx->member_descs[ i ]; + if( member_desc.default_value_size && member_desc.default_value_size != member_desc.size[DL_PTR_SIZE_HOST] ) + { + uint8_t* default_data = dl_ctx->default_data + member_desc.default_value_offset; + dl_internal_patch_member( dl_ctx, &member_desc, default_data, 0, static_cast(-(ptrdiff_t)default_data) ); + } + } + dl_ctx->default_data_patched = false; // This is not thread safe. It would be better to avoid moving around default data and keep it patched like meta data + } dl_binary_writer_write( &writer, &header, sizeof( dl_typelib_header ) ); if(dl_ctx->type_count) dl_binary_writer_write( &writer, dl_ctx->type_ids, sizeof( dl_typeid_t ) * dl_ctx->type_count ); diff --git a/src/dl_types.h b/src/dl_types.h index d41fabc..8a4c23c 100755 --- a/src/dl_types.h +++ b/src/dl_types.h @@ -307,6 +307,7 @@ struct dl_context uint8_t* default_data; size_t default_data_size; + bool default_data_patched; }; struct dl_substr diff --git a/tests/dl_tests_base.cpp b/tests/dl_tests_base.cpp index 248abf2..4f258b7 100644 --- a/tests/dl_tests_base.cpp +++ b/tests/dl_tests_base.cpp @@ -30,9 +30,9 @@ void DL::SetUp() p.error_msg_func = test_log_error; EXPECT_DL_ERR_EQ( DL_ERROR_OK, dl_context_create( &Ctx, &p ) ); - EXPECT_DL_ERR_EQ( DL_ERROR_OK, dl_context_load_type_library(Ctx, typelib1, sizeof(typelib1)) ); EXPECT_DL_ERR_EQ( DL_ERROR_OK, dl_context_load_type_library(Ctx, typelib2, sizeof(typelib2)) ); EXPECT_DL_ERR_EQ( DL_ERROR_OK, dl_context_load_type_library(Ctx, typelib3, sizeof(typelib3)) ); + EXPECT_DL_ERR_EQ( DL_ERROR_OK, dl_context_load_type_library(Ctx, typelib1, sizeof(typelib1)) ); // Putting this last did expose errors } void DL::TearDown() diff --git a/tests/dl_tests_reflect.cpp b/tests/dl_tests_reflect.cpp index 0b508cd..bb63e93 100755 --- a/tests/dl_tests_reflect.cpp +++ b/tests/dl_tests_reflect.cpp @@ -23,12 +23,12 @@ TEST_F(DLReflect, pods) dl_member_info_t Members[128]; memset( &Info, 0x0, sizeof(dl_type_info_t) ); - EXPECT_DL_ERR_OK(dl_reflect_get_type_info( Ctx, Pods::TYPE_ID, &Info )); - EXPECT_DL_ERR_OK(dl_reflect_get_type_members( Ctx, Pods::TYPE_ID, Members, DL_ARRAY_LENGTH(Members) )); + EXPECT_DL_ERR_OK(dl_reflect_get_type_info( Ctx, PodsDefaults::TYPE_ID, &Info )); + EXPECT_DL_ERR_OK(dl_reflect_get_type_members( Ctx, PodsDefaults::TYPE_ID, Members, DL_ARRAY_LENGTH(Members) )); - EXPECT_EQ ((uint32_t)Pods::TYPE_ID, Info.tid ); - EXPECT_STREQ("Pods", Info.name); - EXPECT_EQ (10u, Info.member_count); + EXPECT_EQ ((uint32_t)PodsDefaults::TYPE_ID, Info.tid ); + EXPECT_STREQ("PodsDefaults", Info.name); + EXPECT_EQ (10u, Info.member_count); EXPECT_STREQ("i8", Members[0].name); EXPECT_EQ (DL_TYPE_ATOM_POD, Members[0].atom); @@ -69,6 +69,42 @@ TEST_F(DLReflect, pods) EXPECT_STREQ("f64", Members[9].name); EXPECT_EQ (DL_TYPE_ATOM_POD, Members[9].atom); EXPECT_EQ (DL_TYPE_STORAGE_FP64, Members[9].storage); + + EXPECT_EQ( 2, *reinterpret_cast( Members[0].default_data) ); + EXPECT_EQ( 3, *reinterpret_cast( Members[1].default_data) ); + EXPECT_EQ( 4, *reinterpret_cast( Members[2].default_data) ); + EXPECT_EQ( 5LL, *reinterpret_cast( Members[3].default_data) ); + EXPECT_EQ( 6U, *reinterpret_cast( Members[4].default_data) ); + EXPECT_EQ( 7U, *reinterpret_cast( Members[5].default_data) ); + EXPECT_EQ( 8U, *reinterpret_cast( Members[6].default_data) ); + EXPECT_EQ( 9ULL, *reinterpret_cast( Members[7].default_data) ); + EXPECT_EQ( 10.f, *reinterpret_cast( Members[8].default_data) ); + EXPECT_EQ( 11.0, *reinterpret_cast( Members[9].default_data) ); +} + +TEST_F(DLReflect, array_array_default) +{ + dl_type_info_t Info; + dl_member_info_t Members[16]; + memset( &Info, 0x0, sizeof(dl_type_info_t) ); + + EXPECT_DL_ERR_OK(dl_reflect_get_type_info( Ctx, DefaultArrayArray::TYPE_ID, &Info )); + EXPECT_DL_ERR_OK(dl_reflect_get_type_members( Ctx, DefaultArrayArray::TYPE_ID, Members, DL_ARRAY_LENGTH(Members) )); + + EXPECT_EQ ((uint32_t)DefaultArrayArray::TYPE_ID, Info.tid ); + EXPECT_STREQ("DefaultArrayArray", Info.name); + EXPECT_EQ (1u, Info.member_count); + + EXPECT_STREQ("Arr", Members[0].name); + EXPECT_EQ (nullptr, Members[0].comment); + + const DefaultArrayArray* array_array = reinterpret_cast(Members[0].default_data); + EXPECT_EQ (2U, array_array->Arr.count); + EXPECT_EQ (2U, array_array->Arr[0].u32_arr.count); + EXPECT_EQ (1U, array_array->Arr[0].u32_arr[0]); + EXPECT_EQ (3U, array_array->Arr[0].u32_arr[1]); + EXPECT_EQ (3U, array_array->Arr[1].u32_arr[0]); + EXPECT_EQ (7U, array_array->Arr[1].u32_arr[1]); } TEST_F(DLReflect, enums) diff --git a/tests/unittest2.tld b/tests/unittest2.tld index d221350..a117e5a 100755 --- a/tests/unittest2.tld +++ b/tests/unittest2.tld @@ -94,6 +94,11 @@ "members" : [ { "name" : "members", "type" : "complex_member[]" } ] + }, + "int_default" : { + "members" : [ + { "name" : "member", "type" : "int32", "default" : 1 } + ] } } } From bab72e5965e29852009519797468033e3f84b2ea Mon Sep 17 00:00:00 2001 From: Christian Nilsendahl Date: Thu, 3 Nov 2022 23:54:53 +0100 Subject: [PATCH 3/8] Still work left here Merge remote-tracking branch 'origin/master' into default_data_reflection --- include/dl/dl_txt.h | 51 ++++++++-- src/dl.cpp | 116 ++++++++++++++++----- src/dl_binary_writer.h | 2 +- src/dl_convert.cpp | 194 +++++++++++++++++++++++------------- src/dl_patch_ptr.cpp | 104 ++++++++++--------- src/dl_patch_ptr.h | 10 +- src/dl_store.h | 74 ++++++++++++++ src/dl_txt_pack.cpp | 183 +++++++++++++++++++++++----------- src/dl_txt_unpack.cpp | 120 +++++++++++----------- src/dl_typelib_read_txt.cpp | 4 +- src/dl_types.h | 10 +- tests/dl_tests_error.cpp | 4 +- tests/dl_tests_ptr.cpp | 2 +- 13 files changed, 591 insertions(+), 283 deletions(-) create mode 100644 src/dl_store.h diff --git a/include/dl/dl_txt.h b/include/dl/dl_txt.h index d53593d..f6efc69 100755 --- a/include/dl/dl_txt.h +++ b/include/dl/dl_txt.h @@ -49,7 +49,7 @@ dl_error_t DL_DLL_EXPORT dl_txt_pack_calc_size( dl_ctx_t dl_ctx, const char* txt /* Function: dl_txt_unpack - Unpack binary packed instance to text-format. + Unpack a packed binary (with header and offsets) instance to text-format. Parameters: dl_ctx - Context to use. @@ -61,16 +61,35 @@ dl_error_t DL_DLL_EXPORT dl_txt_pack_calc_size( dl_ctx_t dl_ctx, const char* txt produced_bytes - Number of bytes that would have been written to out_txt_instance if it was large enough. Note: - Packed instance to unpack is required to be in current platform endian, if not DL_ERROR_ENDIAN_ERROR will be returned. + A stored packed instance to unpack is required to be in current platform endian, if not DL_ERROR_ENDIAN_ERROR will be returned. +*/ +dl_error_t DL_DLL_EXPORT dl_txt_unpack( dl_ctx_t dl_ctx, dl_typeid_t type, + unsigned char* packed_instance, size_t packed_instance_size, + char* out_txt_instance, size_t out_txt_instance_size, + size_t* produced_bytes ); + +/* + Function: dl_txt_unpack_loaded + Unpack a loaded binary (without header, with pointers) instance to text-format. + + Parameters: + dl_ctx - Context to use. + type - Type stored in packed_instace. + loaded_instance - Buffer with loaded data. + out_txt_instance - Ptr to buffer where to write txt-data. + out_txt_instance_size - Size of out_txt_instance. + produced_bytes - Number of bytes that would have been written to out_txt_instance if it was large enough. + + Note: + A loaded instance to unpack is required to be in current platform endian, if not DL_ERROR_ENDIAN_ERROR will be returned. */ -dl_error_t DL_DLL_EXPORT dl_txt_unpack( dl_ctx_t dl_ctx, dl_typeid_t type, - const unsigned char* packed_instance, size_t packed_instance_size, - char* out_txt_instance, size_t out_txt_instance_size, - size_t* produced_bytes ); +dl_error_t DL_DLL_EXPORT dl_txt_unpack_loaded( dl_ctx_t dl_ctx, dl_typeid_t type, + const void* loaded_instance, char* out_txt_instance, + size_t out_txt_instance_size, size_t* produced_bytes ); /* Function: dl_txt_unpack_calc_size - Calculate the amount of memory needed to unpack binary data to intermediate data. + Calculate the amount of memory needed to unpack packed binary data (with header and offsets) to text-format. Parameters: dl_ctx - Context to use. @@ -83,9 +102,25 @@ dl_error_t DL_DLL_EXPORT dl_txt_unpack( dl_ctx_t dl_ctx, d Packed instance to unpack is required to be in current platform endian, if not DL_ERROR_ENDIAN_ERROR will be returned. */ dl_error_t DL_DLL_EXPORT dl_txt_unpack_calc_size( dl_ctx_t dl_ctx, dl_typeid_t type, - const unsigned char* packed_instance, size_t packed_instance_size, + unsigned char* packed_instance, size_t packed_instance_size, size_t* out_txt_instance_size ); +/* + Function: dl_txt_unpack_loaded_calc_size + Calculate the amount of memory needed to unpack loaded binary data (without header, with pointers) to text-format. + + Parameters: + dl_ctx - Context to use. + type - Type stored in packed_instace. + loaded_instance - Buffer with loaded data. + out_txt_instance_size - Size required to unpack packed_instance. + + Note: + Packed instance to unpack is required to be in current platform endian, if not DL_ERROR_ENDIAN_ERROR will be returned. +*/ +dl_error_t DL_DLL_EXPORT dl_txt_unpack_loaded_calc_size( dl_ctx_t dl_ctx, dl_typeid_t type, + const void* loaded_instance, size_t* out_txt_instance_size ); + #ifdef __cplusplus } #endif diff --git a/src/dl.cpp b/src/dl.cpp index 3cb071f..5b4f4b5 100644 --- a/src/dl.cpp +++ b/src/dl.cpp @@ -5,6 +5,8 @@ #include
+#include + dl_error_t dl_context_create( dl_ctx_t* dl_ctx, dl_create_params_t* create_params ) { dl_allocator alloc; @@ -47,6 +49,27 @@ dl_error_t dl_context_destroy(dl_ctx_t dl_ctx) return DL_ERROR_OK; } +static dl_error_t dl_ptr_chain_patching( const dl_data_header* header, uint8_t* instance ) +{ + uintptr_t offset_shift = sizeof(uintptr_t) * 4; + uintptr_t offset_mask = (1ULL << offset_shift) - 1; + uint8_t* base_ptr = (uint8_t*)instance - sizeof(dl_data_header); + uint8_t* patch_mem = base_ptr; + uintptr_t offset_to_pointer = header->first_pointer_to_patch; // 0 is the patch terminator + while( offset_to_pointer != 0 ) + { + patch_mem += offset_to_pointer; + if( patch_mem > instance + header->instance_size ) + return DL_ERROR_MALFORMED_DATA; + uintptr_t offsets = *(uintptr_t*)patch_mem; + if( (offsets & offset_mask) > (header->instance_size + sizeof(*header)) ) + return DL_ERROR_MALFORMED_DATA; + offset_to_pointer = offsets >> offset_shift; + *(uint8_t**)patch_mem = (offsets & offset_mask) + base_ptr; + } + return DL_ERROR_OK; +} + dl_error_t dl_instance_load( dl_ctx_t dl_ctx, dl_typeid_t type_id, void* instance, size_t instance_size, const unsigned char* packed_instance, size_t packed_instance_size, @@ -73,11 +96,14 @@ dl_error_t dl_instance_load( dl_ctx_t dl_ctx, dl_typeid_t DL_ASSERT( (uint8_t*)instance + header->instance_size > packed_instance || (uint8_t*)instance < packed_instance + header->instance_size ); memcpy( instance, packed_instance + sizeof(dl_data_header), header->instance_size ); - dl_internal_patch_instance( dl_ctx, root_type, (uint8_t*)instance, 0x0, (uintptr_t)instance ); - - if( consumed ) + if (consumed) *consumed = (size_t)header->instance_size + sizeof(dl_data_header); + if( header->not_using_ptr_chain_patching ) + dl_internal_patch_instance( dl_ctx, root_type, (uint8_t*)instance, 0x0, (uintptr_t)instance - sizeof( dl_data_header ) ); + else + return dl_ptr_chain_patching( header, (uint8_t*)instance ); + return DL_ERROR_OK; } @@ -93,18 +119,23 @@ dl_error_t DL_DLL_EXPORT dl_instance_load_inplace( dl_ctx_t dl_ctx, if( header->version != DL_INSTANCE_VERSION ) return DL_ERROR_VERSION_MISMATCH; if( header->root_instance_type != type_id ) return DL_ERROR_TYPE_MISMATCH; - const dl_type_desc* type = dl_internal_find_type(dl_ctx, header->root_instance_type); - if( type == 0x0 ) - return DL_ERROR_TYPE_NOT_FOUND; - - uint8_t* instance_ptr = packed_instance + sizeof(dl_data_header); - dl_internal_patch_instance( dl_ctx, type, instance_ptr, 0x0, (uintptr_t)instance_ptr ); + *loaded_instance = packed_instance + sizeof(dl_data_header); - *loaded_instance = instance_ptr; - - if( consumed ) + if (consumed) *consumed = header->instance_size + sizeof(dl_data_header); + if( header->version == DL_INSTANCE_VERSION && !header->not_using_ptr_chain_patching ) + { + return dl_ptr_chain_patching( header, (uint8_t*)*loaded_instance ); + } + else if( header->not_using_ptr_chain_patching ) + { + const dl_type_desc* type = dl_internal_find_type( dl_ctx, header->root_instance_type ); + if( type == 0x0 ) + return DL_ERROR_TYPE_NOT_FOUND; + + dl_internal_patch_instance( dl_ctx, type, packed_instance + sizeof( dl_data_header ), 0x0, (uintptr_t)packed_instance ); + } return DL_ERROR_OK; } @@ -112,6 +143,7 @@ struct CDLBinStoreContext { CDLBinStoreContext( uint8_t* out_data, size_t out_data_size, bool is_dummy, dl_allocator alloc ) : written_ptrs(alloc) + , ptrs(alloc) , strings(alloc) { dl_binary_writer_init( &writer, out_data, out_data_size, is_dummy, DL_ENDIAN_HOST, DL_ENDIAN_HOST, DL_PTR_SIZE_HOST ); @@ -119,11 +151,15 @@ struct CDLBinStoreContext uintptr_t FindWrittenPtr( void* ptr ) { - for (size_t i = 0; i < written_ptrs.Len(); ++i) + if( ptr == 0 ) + return (uintptr_t)0; + + size_t len = written_ptrs.Len(); + for( size_t i = 0; i < len; ++i ) if( written_ptrs[i].ptr == ptr ) return written_ptrs[i].pos; - return (uintptr_t)-1; + return (uintptr_t)0; } void AddWrittenPtr( const void* ptr, uintptr_t pos ) @@ -148,6 +184,7 @@ struct CDLBinStoreContext const void* ptr; }; CArrayStatic written_ptrs; + CArrayStatic ptrs; struct SString { @@ -179,6 +216,8 @@ static void dl_internal_store_string( const uint8_t* instance, CDLBinStoreContex store_ctx->strings.Add( {str, length, hash, (uint32_t) offset} ); dl_binary_writer_seek_set(&store_ctx->writer, pos); } + if( !store_ctx->writer.dummy ) + store_ctx->ptrs.Add( dl_binary_writer_tell( &store_ctx->writer ) ); dl_binary_writer_write( &store_ctx->writer, &offset, sizeof(uintptr_t) ); } @@ -191,10 +230,10 @@ static dl_error_t dl_internal_store_ptr( dl_ctx_t dl_ctx, uint8_t* instance, con if( data == 0x0 ) // Null-pointer, store pint(-1) to signal to patching! { - DL_ASSERT(offset == (uintptr_t)-1 && "This pointer should not have been found among the written ptrs!"); - // keep the -1 in Offset and store it to ptr. + DL_ASSERT(offset == (uintptr_t)0 && "This pointer should not have been found among the written ptrs!"); + // keep the 0 in Offset and store it to ptr. } - else if( offset == (uintptr_t)-1 ) // has not been written yet! + else if( offset == (uintptr_t)0 ) // has not been written yet! { uintptr_t pos = dl_binary_writer_tell( &store_ctx->writer ); dl_binary_writer_seek_end( &store_ctx->writer ); @@ -216,6 +255,9 @@ static dl_error_t dl_internal_store_ptr( dl_ctx_t dl_ctx, uint8_t* instance, con dl_binary_writer_seek_set( &store_ctx->writer, pos ); } + if( !store_ctx->writer.dummy && data != 0x0 ) + store_ctx->ptrs.Add( dl_binary_writer_tell( &store_ctx->writer ) ); + dl_binary_writer_write( &store_ctx->writer, &offset, sizeof(uintptr_t) ); return DL_ERROR_OK; } @@ -370,6 +412,9 @@ static dl_error_t dl_internal_store_member( dl_ctx_t dl_ctx, const dl_member_des if (DL_ERROR_OK != err) return err; dl_binary_writer_seek_set( &store_ctx->writer, pos ); + + if( !store_ctx->writer.dummy ) + store_ctx->ptrs.Add( dl_binary_writer_tell( &store_ctx->writer ) ); } // make room for ptr @@ -451,8 +496,9 @@ dl_error_t dl_instance_store( dl_ctx_t dl_ctx, dl_typeid_t type_id, return DL_ERROR_TYPE_NOT_FOUND; bool store_ctx_is_dummy = out_buffer_size == 0; - CDLBinStoreContext store_context(out_buffer + sizeof(dl_data_header), out_buffer_size - sizeof(dl_data_header), store_ctx_is_dummy, dl_ctx->alloc); + CDLBinStoreContext store_context( out_buffer, out_buffer_size, store_ctx_is_dummy, dl_ctx->alloc ); + dl_data_header* header = (dl_data_header*)out_buffer; if( out_buffer_size > 0 ) { if( instance == out_buffer + sizeof(dl_data_header) ) @@ -460,26 +506,46 @@ dl_error_t dl_instance_store( dl_ctx_t dl_ctx, dl_typeid_t type_id, else memset( out_buffer, 0, out_buffer_size ); - dl_data_header* header = (dl_data_header*)out_buffer; header->id = DL_INSTANCE_ID; header->version = DL_INSTANCE_VERSION; header->root_instance_type = type_id; header->is_64_bit_ptr = sizeof( void* ) == 8 ? 1 : 0; } + dl_binary_writer_seek_set( &store_context.writer, sizeof( dl_data_header ) ); + dl_binary_writer_update_needed_size( &store_context.writer ); dl_binary_writer_reserve( &store_context.writer, type->size[DL_PTR_SIZE_HOST] ); - store_context.AddWrittenPtr(instance, 0); // if pointer refere to root-node, it can be found at offset 0 + store_context.AddWrittenPtr( instance, sizeof( dl_data_header ) ); // if pointer refere to root-node, it can be found at offset "sizeof(dl_data_header)" dl_error_t err = dl_internal_instance_store( dl_ctx, type, (uint8_t*)instance, &store_context ); // write instance size! - dl_data_header* out_header = (dl_data_header*)out_buffer; dl_binary_writer_seek_end( &store_context.writer ); - if( out_buffer ) - out_header->instance_size = (uint32_t)dl_binary_writer_tell( &store_context.writer ); + if( header ) + { + header->instance_size = uint32_t( dl_binary_writer_tell( &store_context.writer ) - sizeof( dl_data_header ) ); + + uintptr_t offset_shift = sizeof( uintptr_t ) * 4; + if( dl_binary_writer_tell( &store_context.writer ) >= ( 1ULL << offset_shift ) ) + header->not_using_ptr_chain_patching = 1; + else + { + std::sort( store_context.ptrs.m_Ptr, store_context.ptrs.m_Ptr + store_context.ptrs.m_nElements ); + if( out_buffer && store_context.ptrs.Len() ) + { + store_context.ptrs.Add( store_context.ptrs[store_context.ptrs.Len() - 1] ); // Adding last pointer again so the offset to next pointer becomes 0 which terminates patching + for( size_t i = 0; i < store_context.ptrs.Len() - 1; ++i ) + { + uintptr_t offset = *(uintptr_t*)&out_buffer[store_context.ptrs[i]]; + *(uintptr_t*)&out_buffer[store_context.ptrs[i]] = offset | ( ( (uintptr_t)( store_context.ptrs[i + 1] - store_context.ptrs[i] ) ) << offset_shift ); + } + header->first_pointer_to_patch = (uint32_t)store_context.ptrs[0]; + } + } + } if( produced_bytes ) - *produced_bytes = (uint32_t)dl_binary_writer_tell( &store_context.writer ) + sizeof(dl_data_header); + *produced_bytes = (uint32_t)dl_binary_writer_tell( &store_context.writer ); if( out_buffer_size > 0 && dl_binary_writer_tell( &store_context.writer ) > out_buffer_size ) return DL_ERROR_BUFFER_TOO_SMALL; @@ -539,7 +605,7 @@ dl_error_t dl_instance_get_info( const unsigned char* packed_instance, size_t pa if( packed_instance_size < sizeof(dl_data_header) && header->id != DL_INSTANCE_ID_SWAPED && header->id != DL_INSTANCE_ID ) return DL_ERROR_MALFORMED_DATA; - if(header->version != DL_INSTANCE_VERSION && header->version != DL_INSTANCE_VERSION_SWAPED) + if( header->version != DL_INSTANCE_VERSION && header->version != DL_INSTANCE_VERSION_SWAPED ) return DL_ERROR_VERSION_MISMATCH; out_info->ptrsize = header->is_64_bit_ptr ? 8 : 4; diff --git a/src/dl_binary_writer.h b/src/dl_binary_writer.h index f365c6f..adb0add 100755 --- a/src/dl_binary_writer.h +++ b/src/dl_binary_writer.h @@ -174,7 +174,7 @@ static inline void dl_binary_writer_write_array( dl_binary_writer* writer, const } // val is expected to be in host-endian!!! -static inline void dl_binary_writer_write_ptr( dl_binary_writer* writer, size_t val ) +static inline void dl_binary_writer_write_ptr( dl_binary_writer* writer, uint64_t val ) { if( writer->target_endian != DL_ENDIAN_HOST ) { diff --git a/src/dl_convert.cpp b/src/dl_convert.cpp index 6d5eef2..91dc129 100644 --- a/src/dl_convert.cpp +++ b/src/dl_convert.cpp @@ -73,10 +73,11 @@ class SConvertContext static inline void dl_swap_header( dl_data_header* header ) { - header->id = dl_swap_endian_uint32( header->id ); - header->version = dl_swap_endian_uint32( header->version ); - header->root_instance_type = dl_swap_endian_uint32( header->root_instance_type ); - header->instance_size = dl_swap_endian_uint32( header->instance_size ); + header->id = dl_swap_endian_uint32( header->id ); + header->version = dl_swap_endian_uint32( header->version ); + header->root_instance_type = dl_swap_endian_uint32( header->root_instance_type ); + header->instance_size = dl_swap_endian_uint32( header->instance_size ); + header->first_pointer_to_patch = dl_swap_endian_uint32( header->first_pointer_to_patch ); } static uintptr_t dl_internal_read_ptr_data( const uint8_t* data, @@ -808,20 +809,14 @@ static dl_error_t dl_internal_convert_write_instance( dl_ctx_t dl_ctx, #include bool dl_internal_sort_pred( const SInstance& i1, const SInstance& i2 ) { return i1.address < i2.address; } +bool dl_internal_search_pred( const SInstance& i1, uintptr_t address ) { return i1.address < (const uint8_t*)address; } +bool dl_internal_sort_patchpos_pred( const SConvertContext::PatchPos& i1, const SConvertContext::PatchPos& i2 ) { return i1.pos < i2.pos; } -dl_error_t dl_internal_convert_no_header( dl_ctx_t dl_ctx, - unsigned char* packed_instance, unsigned char* packed_instance_base, - unsigned char* out_instance, size_t out_instance_size, - size_t* needed_size, - dl_endian_t src_endian, dl_endian_t out_endian, - dl_ptr_size_t src_ptr_size, dl_ptr_size_t out_ptr_size, - const dl_type_desc* root_type, size_t base_offset ) +dl_error_t dl_internal_convert_no_header( dl_ctx_t dl_ctx, unsigned char* packed_instance, + unsigned char* packed_instance_base, SConvertContext& conv_ctx, + dl_binary_writer* writer, size_t* needed_size, + const dl_type_desc* root_type ) { - dl_binary_writer writer; - dl_binary_writer_init( &writer, out_instance, out_instance_size, out_instance == 0x0, src_endian, out_endian, out_ptr_size ); - - SConvertContext conv_ctx( src_endian, out_endian, src_ptr_size, out_ptr_size, dl_ctx->alloc ); - conv_ctx.instances.Add(SInstance(packed_instance, root_type, 0x0, dl_make_type(DL_TYPE_ATOM_POD, DL_TYPE_STORAGE_STRUCT))); dl_error_t err = dl_internal_convert_collect_instances(dl_ctx, root_type, packed_instance, packed_instance_base, conv_ctx); @@ -836,41 +831,56 @@ dl_error_t dl_internal_convert_no_header( dl_ctx_t dl_ctx, if (dl_type_storage_t((conv_ctx.instances[i].type_id & DL_TYPE_STORAGE_MASK) >> DL_TYPE_STORAGE_MIN_BIT) == DL_TYPE_STORAGE_STR && last_address == conv_ctx.instances[i].address) continue; // Merge identical strings last_address = conv_ctx.instances[i].address; - err = dl_internal_convert_write_instance( dl_ctx, conv_ctx.instances[i], &conv_ctx.instances[i].offset_after_patch, conv_ctx, &writer ); + err = dl_internal_convert_write_instance( dl_ctx, conv_ctx.instances[i], &conv_ctx.instances[i].offset_after_patch, conv_ctx, writer ); if(err != DL_ERROR_OK) return err; } - if(out_instance != 0x0) // no need to patch data if we are only calculating size + dl_binary_writer_seek_end( writer ); + *needed_size = (unsigned int)dl_binary_writer_tell( writer ); + + if( !writer->dummy ) // no need to patch data if we are only calculating size { - for(unsigned int i = 0; i < conv_ctx.m_lPatchOffset.Len(); ++i) - { - SConvertContext::PatchPos& pp = conv_ctx.m_lPatchOffset[i]; + dl_data_header* new_header = (dl_data_header*)writer->data; + uintptr_t offset_shift = ( conv_ctx.target_ptr_size == DL_PTR_SIZE_32BIT ? 4 : 8 ) * 4; - // find new offset - uintptr_t new_offset = (uintptr_t)-1; + if( *needed_size >= ( 1ULL << offset_shift ) ) + new_header->not_using_ptr_chain_patching = 1; + else + new_header->not_using_ptr_chain_patching = 0; + new_header->instance_size = uint32_t( *needed_size - sizeof( dl_data_header ) ); - for( size_t j = 0; j < conv_ctx.instances.Len(); ++j ) + if( conv_ctx.m_lPatchOffset.Len() ) + { + std::sort( conv_ctx.m_lPatchOffset.m_Ptr, conv_ctx.m_lPatchOffset.m_Ptr + conv_ctx.m_lPatchOffset.Len(), dl_internal_sort_patchpos_pred ); + conv_ctx.m_lPatchOffset.Add( conv_ctx.m_lPatchOffset[conv_ctx.m_lPatchOffset.Len() - 1] ); // Adding last pointer again so the offset to next pointer becomes 0 which terminates patching + for( unsigned int i = 0; i < conv_ctx.m_lPatchOffset.Len() - 1; ++i ) { - uintptr_t old_offset = (uintptr_t)(conv_ctx.instances[j].address - packed_instance_base); + SConvertContext::PatchPos& pp = conv_ctx.m_lPatchOffset[i]; + + if( i == 0 && !new_header->not_using_ptr_chain_patching ) + new_header->first_pointer_to_patch = conv_ctx.tgt_endian != DL_ENDIAN_HOST ? dl_swap_endian_uint32( (uint32_t)pp.pos ) : (uint32_t)pp.pos; - if(old_offset == pp.old_offset) + // find new offset + uint64_t new_offset = 0; + const SInstance* instance = std::lower_bound( conv_ctx.instances.m_Ptr, conv_ctx.instances.m_Ptr + conv_ctx.instances.Len(), pp.old_offset + (uintptr_t)packed_instance_base, dl_internal_search_pred ); + if( instance != conv_ctx.instances.m_Ptr + conv_ctx.instances.Len() ) { - new_offset = conv_ctx.instances[j].offset_after_patch; - break; + uintptr_t old_offset = (uintptr_t)( instance->address - packed_instance_base ); + if( old_offset == pp.old_offset ) + new_offset = instance->offset_after_patch; } - } - DL_ASSERT(new_offset != (uintptr_t)-1 && "We should have found the instance!"); + DL_ASSERT_MSG( new_offset != (uintptr_t)0, "We should have found the instance!" ); + dl_binary_writer_seek_set( writer, pp.pos ); + if( !new_header->not_using_ptr_chain_patching ) + new_offset = new_offset | ( ( (uint64_t)( conv_ctx.m_lPatchOffset[i + 1].pos - conv_ctx.m_lPatchOffset[i].pos ) ) << offset_shift ); - dl_binary_writer_seek_set( &writer, pp.pos ); - dl_binary_writer_write_ptr( &writer, new_offset + base_offset ); + dl_binary_writer_write_ptr( writer, new_offset ); + } } } - dl_binary_writer_seek_end( &writer ); - *needed_size = (unsigned int)dl_binary_writer_tell( &writer ); - return err; } @@ -880,18 +890,20 @@ static dl_error_t dl_internal_convert_instance( dl_ctx_t dl_ctx, dl_endian_t out_endian, size_t out_ptr_size, size_t* out_size ) { - dl_data_header* header = (dl_data_header*)packed_instance; - - if( packed_instance_size < sizeof(dl_data_header) ) return DL_ERROR_MALFORMED_DATA; - if( header->id != DL_INSTANCE_ID && - header->id != DL_INSTANCE_ID_SWAPED ) return DL_ERROR_MALFORMED_DATA; - if( header->version != DL_INSTANCE_VERSION && - header->version != DL_INSTANCE_VERSION_SWAPED ) return DL_ERROR_VERSION_MISMATCH; - if( header->root_instance_type != type && - header->root_instance_type != dl_swap_endian_uint32(type) ) return DL_ERROR_TYPE_MISMATCH; - if( out_ptr_size != 4 && out_ptr_size != 8 ) return DL_ERROR_INVALID_PARAMETER; - - dl_ptr_size_t src_ptr_size = header->is_64_bit_ptr != 0 ? DL_PTR_SIZE_64BIT : DL_PTR_SIZE_32BIT; + dl_data_header header = *(dl_data_header*)packed_instance; + + if( packed_instance_size < sizeof(dl_data_header) ) return DL_ERROR_MALFORMED_DATA; + if( header.id != DL_INSTANCE_ID && + header.id != DL_INSTANCE_ID_SWAPED ) return DL_ERROR_MALFORMED_DATA; + if( header.version != DL_INSTANCE_VERSION && + header.version != DL_INSTANCE_VERSION_SWAPED && + header.version != 1 && + header.version != dl_swap_endian_uint32(1) ) return DL_ERROR_VERSION_MISMATCH; + if( header.root_instance_type != type && + header.root_instance_type != dl_swap_endian_uint32(type) ) return DL_ERROR_TYPE_MISMATCH; + if( out_ptr_size != 4 && out_ptr_size != 8 ) return DL_ERROR_INVALID_PARAMETER; + + dl_ptr_size_t src_ptr_size = header.is_64_bit_ptr != 0 ? DL_PTR_SIZE_64BIT : DL_PTR_SIZE_32BIT; dl_ptr_size_t dst_ptr_size; switch(out_ptr_size) @@ -905,7 +917,10 @@ static dl_error_t dl_internal_convert_instance( dl_ctx_t dl_ctx, if( dst_ptr_size > src_ptr_size && packed_instance == out_instance ) return DL_ERROR_UNSUPPORTED_OPERATION; - dl_endian_t src_endian = header->id == DL_INSTANCE_ID ? DL_ENDIAN_HOST : dl_other_endian( DL_ENDIAN_HOST ); + dl_endian_t src_endian = header.id == DL_INSTANCE_ID ? DL_ENDIAN_HOST : dl_other_endian( DL_ENDIAN_HOST ); + + if (src_endian != DL_ENDIAN_HOST) + dl_swap_header(&header); if(src_endian == out_endian && src_ptr_size == dst_ptr_size) { @@ -916,24 +931,17 @@ static dl_error_t dl_internal_convert_instance( dl_ctx_t dl_ctx, return DL_ERROR_OK; } - dl_typeid_t root_type_id = src_endian != DL_ENDIAN_HOST ? dl_swap_endian_uint32( header->root_instance_type ) : header->root_instance_type; - - const dl_type_desc* root_type = dl_internal_find_type(dl_ctx, root_type_id); + const dl_type_desc* root_type = dl_internal_find_type(dl_ctx, header.root_instance_type); if(root_type == 0x0) return DL_ERROR_TYPE_NOT_FOUND; - dl_error_t err = dl_internal_convert_no_header( dl_ctx, - packed_instance + sizeof(dl_data_header), - packed_instance + sizeof(dl_data_header), - out_instance == 0x0 ? 0x0 : out_instance + sizeof(dl_data_header), - out_instance_size - sizeof(dl_data_header), - out_size, - src_endian, - out_endian, - src_ptr_size, - dst_ptr_size, - root_type, - 0u ); + dl_binary_writer writer; + dl_binary_writer_init( &writer, out_instance, out_instance_size, out_instance == 0x0, src_endian, out_endian, dst_ptr_size ); + + if( packed_instance == out_instance ) + dl_binary_writer_reserve( &writer, sizeof( dl_data_header ) ); + else + dl_binary_writer_write_zero( &writer, sizeof( dl_data_header ) ); if(out_instance != 0x0) { @@ -942,15 +950,65 @@ static dl_error_t dl_internal_convert_instance( dl_ctx_t dl_ctx, new_header->id = DL_INSTANCE_ID; new_header->version = DL_INSTANCE_VERSION; new_header->root_instance_type = type; - new_header->instance_size = uint32_t(*out_size); + new_header->instance_size = uint32_t( out_instance_size - sizeof( dl_data_header ) ); new_header->is_64_bit_ptr = out_ptr_size == 4 ? 0 : 1; - if(DL_ENDIAN_HOST != out_endian) - dl_swap_header(new_header); + uintptr_t offset_shift = out_ptr_size * 4; + if( out_instance_size >= ( 1ULL << offset_shift ) ) + new_header->not_using_ptr_chain_patching = 1; + + if( DL_ENDIAN_HOST != out_endian ) + dl_swap_header( new_header ); } + + SConvertContext conv_ctx( src_endian, out_endian, src_ptr_size, dst_ptr_size, dl_ctx->alloc ); + // While converting we always do slow patching, so neutralize the patch offsets + if( header.version == DL_INSTANCE_VERSION && !header.not_using_ptr_chain_patching ) + { + uint32_t offset_to_next_pointer_to_patch = header.first_pointer_to_patch; + uint8_t* patch_mem = packed_instance; + if( header.is_64_bit_ptr ) + { + uint64_t patch_mask = src_endian == DL_ENDIAN_HOST ? 0xFFFFFFFFULL : 0xFFFFFFFF00000000ULL; + while( offset_to_next_pointer_to_patch != 0 ) // 0 is the patch terminator + { + patch_mem += offset_to_next_pointer_to_patch; + if( patch_mem > packed_instance + sizeof(header) + header.instance_size ) + return DL_ERROR_MALFORMED_DATA; + uint64_t offsets = *(uint64_t*)patch_mem; + if( src_endian != DL_ENDIAN_HOST ) + offsets = dl_swap_endian_uint64( offsets ); + offset_to_next_pointer_to_patch = uint32_t(offsets >> 32); + *(uint64_t*)patch_mem = *(uint64_t*)patch_mem & patch_mask; + } + } + else + { + uint32_t patch_mask = src_endian == DL_ENDIAN_HOST ? 0xFFFFU : 0xFFFF0000U; + while( offset_to_next_pointer_to_patch != 0 ) // 0 is the patch terminator + { + patch_mem += offset_to_next_pointer_to_patch; + if (patch_mem > packed_instance + sizeof(header) + header.instance_size) + return DL_ERROR_MALFORMED_DATA; + uint32_t offsets = *(uint32_t*)patch_mem; + if( src_endian != DL_ENDIAN_HOST ) + offsets = dl_swap_endian_uint32( offsets ); + offset_to_next_pointer_to_patch = offsets >> 16; + *(uint32_t*)patch_mem = *(uint32_t*)patch_mem & patch_mask; + } + } - *out_size += sizeof(dl_data_header); - return err; + dl_data_header* packed_header = (dl_data_header*)packed_instance; + packed_header->not_using_ptr_chain_patching = 1; + packed_header->first_pointer_to_patch = 0; + } + return dl_internal_convert_no_header( dl_ctx, + packed_instance + sizeof(dl_data_header), + packed_instance + (header.version == DL_INSTANCE_VERSION ? 0 : sizeof(dl_data_header)), + conv_ctx, + &writer, + out_size, + root_type ); } #ifdef __cplusplus diff --git a/src/dl_patch_ptr.cpp b/src/dl_patch_ptr.cpp index f8222ee..610aa7d 100644 --- a/src/dl_patch_ptr.cpp +++ b/src/dl_patch_ptr.cpp @@ -3,21 +3,22 @@ struct dl_patched_ptrs { - CArrayStatic addresses; + CArrayStatic addresses; - explicit dl_patched_ptrs(dl_allocator alloc) - : addresses(alloc) - {} + explicit dl_patched_ptrs( dl_allocator alloc ) + : addresses( alloc ) + { + } void add( const uint8_t* addr ) { DL_ASSERT( !patched( addr ) ); - addresses.Add(addr); + addresses.Add( addr ); } bool patched( const uint8_t* addr ) { - for (size_t i = 0; i < addresses.Len(); ++i) + for( size_t i = 0; i < addresses.Len(); ++i ) if( addr == addresses[i] ) return true; return false; @@ -28,55 +29,53 @@ static uintptr_t dl_internal_patch_ptr( uint8_t* ptrptr, uintptr_t patch_distanc { union { uint8_t* src; uintptr_t* ptr; }; src = ptrptr; - if ( *ptr == DL_NULL_PTR_OFFSET[DL_PTR_SIZE_HOST] ) - *ptr = 0x0; - else + if( *ptr != 0 ) *ptr = *ptr + patch_distance; return *ptr; } -void dl_internal_patch_struct( dl_ctx_t ctx, - const dl_type_desc* type, - uint8_t* struct_data, - uintptr_t base_address, - uintptr_t patch_distance, - dl_patched_ptrs* patched_ptrs ); +static void dl_internal_patch_struct( dl_ctx_t ctx, + const dl_type_desc* type, + uint8_t* struct_data, + uintptr_t base_address, + uintptr_t patch_distance, + dl_patched_ptrs* patched_payloads ); static void dl_internal_patch_ptr_instance( dl_ctx_t ctx, - const dl_type_desc* sub_type, + const dl_type_desc* sub_type, uint8_t* ptr_data, uintptr_t base_address, uintptr_t patch_distance, - dl_patched_ptrs* patched_ptrs ) + dl_patched_ptrs* patched_payloads ) { uintptr_t offset = dl_internal_patch_ptr( ptr_data, patch_distance ); if( offset == 0x0 ) return; - uint8_t* ptr = (uint8_t*)base_address + offset; - if( patched_ptrs->patched( ptr ) ) + uintptr_t ptr = (uintptr_t)base_address + offset; + if( patched_payloads->patched( ptr ) ) return; - patched_ptrs->add( ptr ); - dl_internal_patch_struct( ctx, sub_type, ptr, base_address, patch_distance, patched_ptrs ); + patched_payloads->add( ptr ); + dl_internal_patch_struct( ctx, sub_type, (uint8_t*)ptr, base_address, patch_distance, patched_payloads ); } static void dl_internal_patch_str_array( uint8_t* array_data, uint32_t count, uintptr_t patch_distance ) { for( uint32_t index = 0; index < count; ++index ) - dl_internal_patch_ptr( array_data + index * sizeof(char*), patch_distance ); + dl_internal_patch_ptr( array_data + index * sizeof( char* ), patch_distance ); } static void dl_internal_patch_ptr_array( dl_ctx_t ctx, - uint8_t* array_data, + uint8_t* array_data, uint32_t count, const dl_type_desc* sub_type, uintptr_t base_address, uintptr_t patch_distance, - dl_patched_ptrs* patched_ptrs ) + dl_patched_ptrs* patched_payloads ) { for( uint32_t index = 0; index < count; ++index ) - dl_internal_patch_ptr_instance( ctx, sub_type, array_data + index * sizeof(void*), base_address, patch_distance, patched_ptrs ); + dl_internal_patch_ptr_instance( ctx, sub_type, array_data + index * sizeof(void*), base_address, patch_distance, patched_payloads ); } static void dl_internal_patch_struct_array( dl_ctx_t ctx, @@ -85,22 +84,22 @@ static void dl_internal_patch_struct_array( dl_ctx_t ctx, uint32_t count, uintptr_t base_address, uintptr_t patch_distance, - dl_patched_ptrs* patched_ptrs ) + dl_patched_ptrs* patched_payloads ) { uint32_t size = dl_internal_align_up( type->size[DL_PTR_SIZE_HOST], type->alignment[DL_PTR_SIZE_HOST] ); for( uint32_t index = 0; index < count; ++index ) { uint8_t* struct_data = array_data + index * size; - dl_internal_patch_struct( ctx, type, struct_data, base_address, patch_distance, patched_ptrs ); + dl_internal_patch_struct( ctx, type, struct_data, base_address, patch_distance, patched_payloads ); } } static void dl_internal_patch_member( dl_ctx_t ctx, - const dl_member_desc* member, - uint8_t* member_data, - uintptr_t base_address, - uintptr_t patch_distance, - dl_patched_ptrs* patched_ptrs ) + const dl_member_desc* member, + uint8_t* member_data, + uintptr_t base_address, + uintptr_t patch_distance, + dl_patched_ptrs* patched_payloads ) { dl_type_atom_t atom_type = member->AtomType(); dl_type_storage_t storage_type = member->StorageType(); @@ -120,7 +119,7 @@ static void dl_internal_patch_member( dl_ctx_t ctx, member_data, base_address, patch_distance, - patched_ptrs ); + patched_payloads ); break; case DL_TYPE_STORAGE_STRUCT: dl_internal_patch_struct( ctx, @@ -128,7 +127,7 @@ static void dl_internal_patch_member( dl_ctx_t ctx, member_data, base_address, patch_distance, - patched_ptrs ); + patched_payloads ); break; default: break; @@ -150,7 +149,7 @@ static void dl_internal_patch_member( dl_ctx_t ctx, dl_internal_find_type( ctx, member->type_id ), base_address, patch_distance, - patched_ptrs ); + patched_payloads ); break; case DL_TYPE_STORAGE_STRUCT: dl_internal_patch_struct_array( ctx, @@ -159,7 +158,7 @@ static void dl_internal_patch_member( dl_ctx_t ctx, member->inline_array_cnt(), base_address, patch_distance, - patched_ptrs ); + patched_payloads ); break; default: break; @@ -171,8 +170,7 @@ static void dl_internal_patch_member( dl_ctx_t ctx, { uintptr_t offset = dl_internal_patch_ptr( member_data, patch_distance ); - union { uint8_t* src; uint32_t ptr; }; - src = member_data + sizeof( void* ); + uint8_t* src = member_data + sizeof( void* ); uint32_t count = *(uint32_t*)src; if( count != 0 ) @@ -190,7 +188,7 @@ static void dl_internal_patch_member( dl_ctx_t ctx, dl_internal_find_type( ctx, member->type_id ), base_address, patch_distance, - patched_ptrs ); + patched_payloads ); break; case DL_TYPE_STORAGE_STRUCT: dl_internal_patch_struct_array( ctx, @@ -199,7 +197,7 @@ static void dl_internal_patch_member( dl_ctx_t ctx, count, base_address, patch_distance, - patched_ptrs ); + patched_payloads ); break; default: break; @@ -217,7 +215,7 @@ static void dl_internal_patch_union( dl_ctx_t ctx, uint8_t* union_data, uintptr_t base_address, uintptr_t patch_distance, - dl_patched_ptrs* patched_ptrs ) + dl_patched_ptrs* patched_payloads ) { DL_ASSERT(type->flags & DL_TYPE_FLAG_IS_UNION); size_t type_offset = dl_internal_union_type_offset( ctx, type, DL_PTR_SIZE_HOST ); @@ -226,28 +224,28 @@ static void dl_internal_patch_union( dl_ctx_t ctx, uint32_t union_type = *((uint32_t*)(union_data + type_offset)); const dl_member_desc* member = dl_internal_union_type_to_member(ctx, type, union_type); DL_ASSERT(member->offset[DL_PTR_SIZE_HOST] == 0); - dl_internal_patch_member( ctx, member, union_data, base_address, patch_distance, patched_ptrs ); + dl_internal_patch_member( ctx, member, union_data, base_address, patch_distance, patched_payloads ); } -void dl_internal_patch_struct( dl_ctx_t ctx, - const dl_type_desc* type, - uint8_t* struct_data, - uintptr_t base_address, - uintptr_t patch_distance, - dl_patched_ptrs* patched_ptrs ) +static void dl_internal_patch_struct( dl_ctx_t ctx, + const dl_type_desc* type, + uint8_t* struct_data, + uintptr_t base_address, + uintptr_t patch_distance, + dl_patched_ptrs* patched_payloads ) { if( type->flags & DL_TYPE_FLAG_HAS_SUBDATA ) { if( type->flags & DL_TYPE_FLAG_IS_UNION ) { - dl_internal_patch_union(ctx, type, struct_data, base_address, patch_distance, patched_ptrs); + dl_internal_patch_union( ctx, type, struct_data, base_address, patch_distance, patched_payloads ); } else { for( uint32_t member_index = 0; member_index < type->member_count; ++member_index ) { const dl_member_desc* member = dl_get_type_member( ctx, type, member_index ); - dl_internal_patch_member( ctx, member, struct_data + member->offset[DL_PTR_SIZE_HOST], base_address, patch_distance, patched_ptrs ); + dl_internal_patch_member( ctx, member, struct_data + member->offset[DL_PTR_SIZE_HOST], base_address, patch_distance, patched_payloads ); } } } @@ -257,7 +255,7 @@ void dl_internal_patch_member( dl_ctx_t ctx, const dl_member_desc* member, uint8_t* member_data, uintptr_t base_address, - uintptr_t patch_distance ) + uintptr_t patch_distance) { dl_patched_ptrs patched(ctx->alloc); dl_internal_patch_member( ctx, member, member_data, base_address, patch_distance, &patched ); @@ -270,11 +268,11 @@ void dl_internal_patch_instance( dl_ctx_t ctx, uintptr_t patch_distance ) { dl_patched_ptrs patched(ctx->alloc); - patched.add( instance ); + patched.add( (uintptr_t)instance ); if( type->flags & DL_TYPE_FLAG_IS_UNION ) { - dl_internal_patch_union(ctx, type, instance, base_address, patch_distance, &patched); + dl_internal_patch_union( ctx, type, instance, base_address, patch_distance, &patched ); } else { diff --git a/src/dl_patch_ptr.h b/src/dl_patch_ptr.h index fc6c721..0cfabf7 100644 --- a/src/dl_patch_ptr.h +++ b/src/dl_patch_ptr.h @@ -27,10 +27,10 @@ void dl_internal_patch_instance( dl_ctx_t ctx, * @param base_address base address to patch the pointers against. * @param patch_distance distance in bytes to patch all pointers. */ -void dl_internal_patch_member( dl_ctx_t ctx, - const dl_member_desc* member, - uint8_t* member_data, - uintptr_t base_address, - uintptr_t patch_distance ); +void dl_internal_patch_member( dl_ctx_t ctx, + const dl_member_desc* member, + uint8_t* member_data, + uintptr_t base_address, + uintptr_t patch_distance ); #endif // DL_PATCH_PTR_H_INCLUDED diff --git a/src/dl_store.h b/src/dl_store.h new file mode 100644 index 0000000..3465f23 --- /dev/null +++ b/src/dl_store.h @@ -0,0 +1,74 @@ +/* copyright (c) 2010 Fredrik Kihlander, see LICENSE for more info */ + +#ifndef DL_DL_STORE_H_INCLUDED +#define DL_DL_STORE_H_INCLUDED + +#pragma pack(push, 1) + +struct CDLBinStoreContext +{ + struct SString; + + CDLBinStoreContext( struct dl_binary_writer& writer_, dl_allocator alloc, CArrayStatic& ptrs_, CArrayStatic& strings_ ) + : writer( writer_ ) + , written_ptrs( alloc ) + , ptrs( ptrs_ ) + , strings( strings_ ) + { + } + + uintptr_t FindWrittenPtr( const void* ptr ) + { + if( ptr == 0 ) + return (uintptr_t)0; + + size_t len = written_ptrs.Len(); + for( size_t i = 0; i < len; ++i ) + if( written_ptrs[i].ptr == ptr ) + return written_ptrs[i].pos; + + return (uintptr_t)0; + } + + void AddWrittenPtr( const void* ptr, uintptr_t pos ) + { + written_ptrs.Add( { pos, ptr } ); + } + + uint32_t GetStringOffset( const char* str, int length, uint32_t hash ) + { + for( size_t i = 0; i < strings.Len(); ++i ) + if( strings[i].hash == hash && strings[i].str.len == length && memcmp( str, strings[i].str.str, length ) == 0 ) + return strings[i].offset; + + return 0; + } + + struct dl_binary_writer& writer; + + // Used to know where all pointer payloads are, so two identical pointers will refer to the same offset + struct SWrittenPtr + { + uintptr_t pos; + const void* ptr; + }; + CArrayStatic written_ptrs; + + // Used to know where all pointers are, to speed up pointer patching + CArrayStatic& ptrs; + + // Used to merge identical strings + struct SString + { + dl_substr str; + uint32_t hash; + uint32_t offset; + }; + CArrayStatic& strings; +}; + +dl_error_t dl_internal_store_member( dl_ctx_t dl_ctx, const dl_member_desc* member, const uint8_t* instance, CDLBinStoreContext* store_ctx ); + +#pragma pack( pop ) + +#endif // DL_DL_STORE_H_INCLUDED diff --git a/src/dl_txt_pack.cpp b/src/dl_txt_pack.cpp index 867b1e8..c84701c 100755 --- a/src/dl_txt_pack.cpp +++ b/src/dl_txt_pack.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -59,14 +60,14 @@ inline double dl_strtod(const char* str, char** endptr) } #endif if( tolower(str[1]) == 'a' && - tolower(str[2]) == 'x' && + tolower(str[2]) == 'x' && !isalnum(str[3]) ) { *endptr = (char*)str + 3; return DBL_MAX * sign; } if( tolower(str[1]) == 'i' && - tolower(str[2]) == 'n' && + tolower(str[2]) == 'n' && !isalnum(str[3])) { *endptr = (char*)str + 3; @@ -113,14 +114,14 @@ inline float dl_strtof(const char* str, char** endptr) } #endif if( tolower(str[1]) == 'a' && - tolower(str[2]) == 'x' && + tolower(str[2]) == 'x' && !isalnum(str[3]) ) { *endptr = (char*)str + 3; return FLT_MAX * sign; } if( tolower(str[1]) == 'i' && - tolower(str[2]) == 'n' && + tolower(str[2]) == 'n' && !isalnum(str[3])) { *endptr = (char*)str + 3; @@ -133,21 +134,48 @@ inline float dl_strtof(const char* str, char** endptr) struct dl_txt_pack_ctx { explicit dl_txt_pack_ctx(dl_allocator alloc) - : subdata(alloc) + : subdata(alloc) + , strings(alloc) + , ptrs(alloc) + { + } + + uint32_t GetStringOffset( dl_substr str, uint32_t hash ) { + for( size_t i = 0; i < strings.Len(); ++i ) + if( strings[i].hash == hash && strings[i].str.len == str.len && memcmp( strings[i].str.str, str.str, str.len ) == 0 ) + return strings[i].offset; + + return 0; } dl_txt_read_ctx read_ctx; dl_binary_writer* writer; + + // Where the "__subdata" section starts const char* subdata_pos; + + // Used to know where all pointer payloads are, so two identical pointers will refer to the same offset struct SSubData { dl_substr name; uint32_t name_hash; const dl_type_desc* type; size_t patch_pos; - }; + }; CArrayStatic subdata; + + // Used to merge identical strings + struct SString + { + dl_substr name; + uint32_t name_hash; + uint32_t offset; + }; + CArrayStatic<::CDLBinStoreContext::SString, 128> strings; + + // Used to know where all pointers are, to speed up pointer patching + CArrayStatic ptrs; }; static void dl_txt_pack_eat_and_write_int8( dl_ctx_t dl_ctx, dl_txt_pack_ctx* packctx ) @@ -225,7 +253,7 @@ static bool dl_txt_pack_eat_and_write_null(dl_txt_pack_ctx* packctx) dl_txt_eat_white( &packctx->read_ctx ); if( strncmp( packctx->read_ctx.iter, "null", 4 ) == 0 ) { - dl_binary_writer_write_ptr( packctx->writer, (uintptr_t)-1 ); + dl_binary_writer_write_ptr( packctx->writer, (uintptr_t) 0 ); packctx->read_ctx.iter += 4; return true; } @@ -242,16 +270,20 @@ static void dl_txt_pack_eat_and_write_string( dl_ctx_t dl_ctx, dl_txt_pack_ctx* if( str.str == 0x0 ) dl_txt_read_failed( dl_ctx, &packctx->read_ctx, DL_ERROR_MALFORMED_DATA, "expected a value of type 'string' or 'null'" ); - size_t curr = dl_binary_writer_tell( packctx->writer ); - dl_binary_writer_seek_end( packctx->writer ); - size_t strpos = dl_binary_writer_tell( packctx->writer ); - for( int i = 0; i < str.len; ++i ) + uint32_t hash = dl_internal_hash_buffer( (const uint8_t*)str.str, str.len ); + size_t strpos = packctx->GetStringOffset( str, hash ); + if( strpos == 0 ) { - if( str.str[i] == '\\' ) + size_t curr = dl_binary_writer_tell( packctx->writer ); + dl_binary_writer_seek_end( packctx->writer ); + strpos = dl_binary_writer_tell( packctx->writer ); + for( int i = 0; i < str.len; ++i ) { - ++i; - switch( str.str[i] ) + if( str.str[i] == '\\' ) { + ++i; + switch( str.str[i] ) + { case '\'': case '\"': case '\\': @@ -265,14 +297,18 @@ static void dl_txt_pack_eat_and_write_string( dl_ctx_t dl_ctx, dl_txt_pack_ctx* break; default: DL_ASSERT( false && "unhandled escape!" ); + } } + else + dl_binary_writer_write_uint8( packctx->writer, (uint8_t)str.str[i] ); } - else - dl_binary_writer_write_uint8( packctx->writer, (uint8_t)str.str[i] ); + dl_binary_writer_write_uint8( packctx->writer, '\0' ); + dl_binary_writer_seek_set( packctx->writer, curr ); + packctx->strings.Add( { str, hash, (uint32_t) strpos } ); } - dl_binary_writer_write_uint8( packctx->writer, '\0' ); - dl_binary_writer_seek_set( packctx->writer, curr ); - dl_binary_writer_write( packctx->writer, &strpos, sizeof(size_t) ); + if( !packctx->writer->dummy ) + packctx->ptrs.Add( dl_binary_writer_tell( packctx->writer ) ); + dl_binary_writer_write( packctx->writer, &strpos, sizeof( size_t ) ); } static void dl_txt_pack_eat_and_write_enum( dl_ctx_t dl_ctx, dl_txt_pack_ctx* packctx, const dl_enum_desc* edesc ) @@ -302,16 +338,20 @@ static void dl_txt_pack_eat_and_write_enum( dl_ctx_t dl_ctx, dl_txt_pack_ctx* pa packctx->read_ctx.iter = ename.str + ename.len + 1; } -static void dl_txt_pack_eat_and_write_ptr( dl_ctx_t dl_ctx, dl_txt_pack_ctx* packctx, const dl_type_desc* type, size_t patch_pos ) +static bool dl_txt_pack_eat_and_write_ptr( dl_ctx_t dl_ctx, dl_txt_pack_ctx* packctx, const dl_type_desc* type, size_t patch_pos ) { if( dl_txt_pack_eat_and_write_null(packctx) ) - return; + return false; dl_substr ptr = dl_txt_eat_string( &packctx->read_ctx ); if( ptr.str == 0x0 ) dl_txt_read_failed( dl_ctx, &packctx->read_ctx, DL_ERROR_TXT_INVALID_MEMBER_TYPE, "expected string" ); + if( !packctx->writer->dummy ) + packctx->ptrs.Add( patch_pos ); + packctx->subdata.Add({ ptr, dl_internal_hash_buffer((const uint8_t*)ptr.str, ptr.len), type, patch_pos }); + return true; } static void dl_txt_pack_validate_c_symbol_key( dl_ctx_t dl_ctx, dl_txt_pack_ctx* packctx, dl_substr symbol ) @@ -523,7 +563,7 @@ static void dl_txt_pack_array_item_size_align( dl_ctx_t dl_ctx, } } -const char* dl_txt_skip_array( const char* iter, const char* end ) +static const char* dl_txt_skip_array( const char* iter, const char* end ) { iter = dl_txt_skip_white( iter, end ); if( *iter != '[' ) @@ -708,15 +748,16 @@ static uint32_t dl_txt_pack_find_array_length( dl_ctx_t dl_ctx, dl_txt_pack_ctx* return (uint32_t)-1; } -static void dl_txt_pack_write_default_value( dl_ctx_t dl_ctx, - dl_txt_pack_ctx* packctx, - const dl_member_desc* member, - size_t member_pos ) +static dl_error_t dl_txt_pack_write_default_value( dl_ctx_t dl_ctx, + dl_txt_pack_ctx* packctx, + const dl_member_desc* member, + size_t member_pos ) { uint8_t* member_default_value = dl_ctx->default_data + member->default_value_offset; uint32_t member_size = member->size[DL_PTR_SIZE_HOST]; + dl_error_t err = DL_ERROR_OK; // ... handle bitfields differently since they take up sub-parts of bytes. if(member->AtomType() == DL_TYPE_ATOM_BITFIELD) { @@ -768,22 +809,19 @@ static void dl_txt_pack_write_default_value( dl_ctx_t dl_ctx, else { dl_binary_writer_seek_set( packctx->writer, member_pos ); - dl_binary_writer_write( packctx->writer, member_default_value, member->size[DL_PTR_SIZE_HOST] ); - } - - if( member_size != member->default_value_size ) - { - uint8_t* subdata = member_default_value + member_size; - // ... sub ptrs, copy and patch ... - dl_binary_writer_seek_end( packctx->writer ); - uintptr_t subdata_pos = dl_binary_writer_tell( packctx->writer ); - - dl_binary_writer_write( packctx->writer, subdata, member->default_value_size - member_size ); - - uint8_t* member_data = packctx->writer->data + member_pos; - if( !packctx->writer->dummy ) - dl_internal_patch_member( dl_ctx, member, member_data, (uintptr_t)packctx->writer->data, subdata_pos - member_size - (dl_ctx->default_data_patched ? (uintptr_t)member_default_value : 0) ); + if( member_size == member->default_value_size ) + dl_binary_writer_write( packctx->writer, member_default_value, member->size[DL_PTR_SIZE_HOST] ); + else + { + CDLBinStoreContext store_ctx( *packctx->writer, dl_ctx->alloc, packctx->ptrs, packctx->strings ); + uint8_t* default_data = (uint8_t*)dl_alloc( &dl_ctx->alloc, member->default_value_size ); + memcpy( default_data, member_default_value, member->default_value_size ); + dl_internal_patch_member( dl_ctx, member, default_data, (uintptr_t)0, (uintptr_t)default_data - sizeof( dl_data_header ) ); + err = dl_internal_store_member( dl_ctx, member, default_data, &store_ctx ); + dl_free( &dl_ctx->alloc, default_data ); + } } + return err; } static dl_error_t dl_txt_pack_member( dl_ctx_t dl_ctx, dl_txt_pack_ctx* packctx, size_t instance_pos, const dl_member_desc* member ) @@ -837,7 +875,7 @@ static dl_error_t dl_txt_pack_member( dl_ctx_t dl_ctx, dl_txt_pack_ctx* packctx, uint32_t array_length = dl_txt_pack_find_array_length( dl_ctx, packctx, member ); if( array_length == 0 ) { - dl_binary_writer_write_pint( packctx->writer, (size_t)-1 ); + dl_binary_writer_write_pint( packctx->writer, (size_t)0 ); dl_binary_writer_write_uint32( packctx->writer, 0 ); } else @@ -847,6 +885,9 @@ static dl_error_t dl_txt_pack_member( dl_ctx_t dl_ctx, dl_txt_pack_ctx* packctx, size_t array_pos = dl_binary_writer_needed_size( packctx->writer ); array_pos = dl_internal_align_up( array_pos, element_align ); + if( !packctx->writer->dummy ) + packctx->ptrs.Add( dl_binary_writer_tell( packctx->writer ) ); + dl_binary_writer_write_pint( packctx->writer, array_pos ); dl_binary_writer_write_uint32( packctx->writer, array_length ); dl_binary_writer_seek_end( packctx->writer ); @@ -893,7 +934,9 @@ static dl_error_t dl_txt_pack_member( dl_ctx_t dl_ctx, dl_txt_pack_ctx* packctx, for(uint32_t sub_member_i = 0; sub_member_i < sub_type->member_count; ++sub_member_i) { const dl_member_desc* sub_member = dl_get_type_member(dl_ctx, sub_type, sub_member_i); - dl_txt_pack_write_default_value(dl_ctx, packctx, sub_member, current_member_array_position + sub_member->offset[DL_PTR_SIZE_HOST]); + dl_error_t err = dl_txt_pack_write_default_value(dl_ctx, packctx, sub_member, current_member_array_position + sub_member->offset[DL_PTR_SIZE_HOST]); + if( err != DL_ERROR_OK ) + return err; } current_member_array_position += sub_type->size[DL_PTR_SIZE_HOST]; } @@ -1119,7 +1162,8 @@ static dl_error_t dl_txt_pack_eat_and_write_struct( dl_ctx_t dl_ctx, dl_txt_pack dl_txt_read_failed( dl_ctx, &packctx->read_ctx, DL_ERROR_TXT_MISSING_MEMBER, "member %s.%s is not set and has no default value", dl_internal_type_name( dl_ctx, type ), dl_internal_member_name( dl_ctx, member ) ); size_t member_pos = instance_pos + member->offset[DL_PTR_SIZE_HOST]; - dl_txt_pack_write_default_value(dl_ctx, packctx, member, member_pos); + dl_error_t err = dl_txt_pack_write_default_value(dl_ctx, packctx, member, member_pos); + if( DL_ERROR_OK != err ) return err; } } return DL_ERROR_OK; @@ -1162,7 +1206,7 @@ static dl_error_t dl_txt_pack_finalize_subdata( dl_ctx_t dl_ctx, dl_txt_pack_ctx packctx->read_ctx.iter = packctx->subdata_pos; CArrayStatic subinstances(dl_ctx->alloc); - subinstances.Add({ { "__root", 6 }, 0, dl_internal_hash_string("__root") }); + subinstances.Add({ { "__root", 6 }, sizeof(dl_data_header), dl_internal_hash_string("__root") }); dl_txt_eat_char( dl_ctx, &packctx->read_ctx, '{' ); @@ -1287,12 +1331,12 @@ static const dl_type_desc* dl_txt_pack_inner( dl_ctx_t dl_ctx, dl_txt_pack_ctx* return 0x0; } -dl_error_t dl_txt_pack( dl_ctx_t dl_ctx, const char* txt_instance, unsigned char* out_buffer, size_t out_buffer_size, size_t* produced_bytes ) +dl_error_t dl_txt_pack_internal( dl_ctx_t dl_ctx, const char* txt_instance, unsigned char* out_buffer, size_t out_buffer_size, size_t* produced_bytes, bool use_fast_ptr_patch ) { dl_binary_writer writer; dl_binary_writer_init( &writer, - out_buffer + sizeof(dl_data_header), - out_buffer_size - sizeof(dl_data_header), + out_buffer, + out_buffer_size, out_buffer_size == 0, DL_ENDIAN_HOST, DL_ENDIAN_HOST, @@ -1305,24 +1349,43 @@ dl_error_t dl_txt_pack( dl_ctx_t dl_ctx, const char* txt_instance, unsigned char packctx.subdata_pos = 0x0; packctx.read_ctx.err = DL_ERROR_OK; + dl_binary_writer_write_zero( &writer, sizeof( dl_data_header ) ); const dl_type_desc* root_type = dl_txt_pack_inner( dl_ctx, &packctx ); if( packctx.read_ctx.err == DL_ERROR_OK ) { // write header if( out_buffer_size > 0 ) { - dl_data_header header; - memset(&header, 0x0, sizeof(dl_data_header)); - header.id = DL_INSTANCE_ID; - header.version = DL_INSTANCE_VERSION; - header.root_instance_type = dl_internal_typeid_of( dl_ctx, root_type ); - header.instance_size = (uint32_t)dl_binary_writer_needed_size( &writer ); - header.is_64_bit_ptr = sizeof(void*) == 8 ? 1 : 0; - memcpy( out_buffer, &header, sizeof(dl_data_header) ); + CArrayStatic& pointers = packctx.ptrs; + + dl_data_header& header = *(dl_data_header*)writer.data; + header.id = DL_INSTANCE_ID; + header.version = DL_INSTANCE_VERSION; + header.root_instance_type = dl_internal_typeid_of( dl_ctx, root_type ); + header.instance_size = uint32_t(dl_binary_writer_needed_size( &writer ) - sizeof( dl_data_header )); + header.is_64_bit_ptr = sizeof( void* ) == 8 ? 1 : 0; + header.first_pointer_to_patch = pointers.Len() ? (uint32_t)pointers[0] : 0; + + uintptr_t offset_shift = sizeof( uintptr_t ) * 4; + if( dl_binary_writer_needed_size( &writer ) >= ( 1ULL << offset_shift ) || !use_fast_ptr_patch ) + header.not_using_ptr_chain_patching = 1; + else + { + std::sort( pointers.m_Ptr, pointers.m_Ptr + pointers.m_nElements ); + if( !packctx.writer->dummy && pointers.Len() ) + { + pointers.Add( pointers[pointers.Len() - 1] ); // Adding last pointer again so the offset to next pointer becomes 0 which terminates patching + for( size_t i = 0; i < pointers.Len() - 1; ++i ) + { + uintptr_t offset = *(uintptr_t*)&packctx.writer->data[pointers[i]]; + *(uintptr_t*)&packctx.writer->data[pointers[i]] = offset | ( ( (uintptr_t)( pointers[i + 1] - pointers[i] ) ) << offset_shift ); + } + } + } } if( produced_bytes ) - *produced_bytes = (unsigned int)dl_binary_writer_needed_size( &writer ) + sizeof(dl_data_header); + *produced_bytes = (unsigned int)dl_binary_writer_needed_size( &writer ); } else { @@ -1331,6 +1394,12 @@ dl_error_t dl_txt_pack( dl_ctx_t dl_ctx, const char* txt_instance, unsigned char return packctx.read_ctx.err; } +dl_error_t dl_txt_pack(dl_ctx_t dl_ctx, const char* txt_instance, unsigned char* out_buffer, size_t out_buffer_size, size_t* produced_bytes) +{ + bool use_fast_ptr_patch = true; + return dl_txt_pack_internal( dl_ctx, txt_instance, out_buffer, out_buffer_size, produced_bytes, use_fast_ptr_patch ); +} + dl_error_t dl_txt_pack_calc_size( dl_ctx_t dl_ctx, const char* txt_instance, size_t* out_instance_size ) { return dl_txt_pack( dl_ctx, txt_instance, 0x0, 0, out_instance_size ); diff --git a/src/dl_txt_unpack.cpp b/src/dl_txt_unpack.cpp index 869451a..a06ad58 100755 --- a/src/dl_txt_unpack.cpp +++ b/src/dl_txt_unpack.cpp @@ -17,7 +17,7 @@ struct dl_txt_unpack_ctx int indent; struct SPtr { - uintptr_t offset; + const uint8_t* ptr; dl_typeid_t tid; }; CArrayStatic ptrs; @@ -54,12 +54,12 @@ static void dl_txt_unpack_write_string( dl_binary_writer* writer, const char* st dl_binary_writer_write_uint8( writer, '\"' ); } -static void dl_txt_unpack_write_string_or_null( dl_binary_writer* writer, dl_txt_unpack_ctx* unpack_ctx, uintptr_t offset ) +static void dl_txt_unpack_write_string_or_null( dl_binary_writer* writer, const char* str ) { - if( offset == (uintptr_t)-1 ) + if( str == 0 ) dl_binary_writer_write( writer, "null", 4 ); else - dl_txt_unpack_write_string( writer, (const char*)&unpack_ctx->packed_instance[offset] ); + dl_txt_unpack_write_string( writer, str ); } static void dl_txt_unpack_int8( dl_binary_writer* writer, int8_t data ) @@ -150,20 +150,20 @@ static void dl_txt_unpack_enum( dl_ctx_t dl_ctx, dl_binary_writer* writer, const DL_ASSERT_MSG(false, "failed to find enum value " DL_PINT_FMT_STR, value); } -static void dl_txt_unpack_ptr( dl_binary_writer* writer, uintptr_t offset ) +static void dl_txt_unpack_ptr( dl_binary_writer* writer, dl_txt_unpack_ctx* unpack_ctx, const uint8_t* ptr ) { - if( offset == (uintptr_t)-1 ) + if( ptr == 0 ) { dl_binary_writer_write( writer, "null", 4 ); } - else if( offset == 0 ) + else if( ptr == unpack_ctx->packed_instance ) { dl_binary_writer_write( writer, "\"__root\"", 8 ); } else { char buffer[256]; - dl_internal_str_format( buffer, sizeof(buffer), "ptr_" DL_UINT64_FMT_STR, (uint64_t)offset ); + dl_internal_str_format( buffer, sizeof( buffer ), "ptr_" DL_UINT64_FMT_STR, (uint64_t)( ptr - unpack_ctx->packed_instance ) ); dl_txt_unpack_write_string( writer, buffer ); } } @@ -297,10 +297,10 @@ static dl_error_t dl_txt_unpack_array( dl_ctx_t dl_ctx, uintptr_t* mem = (uintptr_t*)array_data; for( uint32_t i = 0; i < array_count - 1; ++i ) { - dl_txt_unpack_write_string_or_null( writer, unpack_ctx, mem[i] ); + dl_txt_unpack_write_string_or_null( writer, (const char*)mem[i] ); dl_binary_writer_write( writer, ", ", 2 ); } - dl_txt_unpack_write_string_or_null( writer, unpack_ctx, mem[array_count - 1] ); + dl_txt_unpack_write_string_or_null( writer, (const char*)mem[array_count - 1] ); } break; case DL_TYPE_STORAGE_PTR: @@ -308,10 +308,10 @@ static dl_error_t dl_txt_unpack_array( dl_ctx_t dl_ctx, uintptr_t* mem = (uintptr_t*)array_data; for( uint32_t i = 0; i < array_count - 1; ++i ) { - dl_txt_unpack_ptr( writer, mem[i] ); + dl_txt_unpack_ptr( writer, unpack_ctx, (const uint8_t*)mem[i] ); dl_binary_writer_write( writer, ", ", 2 ); } - dl_txt_unpack_ptr( writer, mem[array_count - 1] ); + dl_txt_unpack_ptr( writer, unpack_ctx, (const uint8_t*)mem[array_count - 1] ); unpack_ctx->has_ptrs = true; break; } @@ -488,10 +488,10 @@ static dl_error_t dl_txt_unpack_member( dl_ctx_t dl_ctx, dl_txt_unpack_ctx* unpa case DL_TYPE_STORAGE_ENUM_UINT32: dl_txt_unpack_enum ( dl_ctx, writer, dl_internal_find_enum( dl_ctx, member->type_id ), (uint64_t)*(uint32_t*) member_data ); break; case DL_TYPE_STORAGE_ENUM_INT64: dl_txt_unpack_enum ( dl_ctx, writer, dl_internal_find_enum( dl_ctx, member->type_id ), (uint64_t)*( int64_t*) member_data ); break; case DL_TYPE_STORAGE_ENUM_UINT64: dl_txt_unpack_enum ( dl_ctx, writer, dl_internal_find_enum( dl_ctx, member->type_id ), (uint64_t)*(uint64_t*) member_data ); break; - case DL_TYPE_STORAGE_STR: dl_txt_unpack_write_string_or_null( writer, unpack_ctx, *(uintptr_t*)member_data ); break; + case DL_TYPE_STORAGE_STR: dl_txt_unpack_write_string_or_null( writer, *(const char**)member_data ); break; case DL_TYPE_STORAGE_PTR: { - dl_txt_unpack_ptr( writer, *(uintptr_t*)member_data ); + dl_txt_unpack_ptr( writer, unpack_ctx, ( const uint8_t* ) * (uintptr_t*)member_data ); unpack_ctx->has_ptrs = true; } break; @@ -503,12 +503,12 @@ static dl_error_t dl_txt_unpack_member( dl_ctx_t dl_ctx, dl_txt_unpack_ctx* unpa break; case DL_TYPE_ATOM_ARRAY: { - uintptr_t offset = *(uintptr_t*)member_data; - uint32_t count = *(uint32_t*)(member_data + sizeof(uintptr_t)); - if( offset == (uintptr_t)-1 ) + const uint8_t* array = *(const uint8_t**) member_data; + uint32_t array_count = *(uint32_t*)( member_data + sizeof( uintptr_t ) ); + if( array_count == 0 ) dl_binary_writer_write( writer, "[]", 2 ); else - return dl_txt_unpack_array( dl_ctx, unpack_ctx, writer, member->StorageType(), &unpack_ctx->packed_instance[offset], count, member->type_id ); + return dl_txt_unpack_array( dl_ctx, unpack_ctx, writer, member->StorageType(), array, array_count, member->type_id ); } break; case DL_TYPE_ATOM_INLINE_ARRAY: @@ -546,31 +546,27 @@ static dl_error_t dl_txt_unpack_write_subdata_ptr( dl_ctx_t dl_ctx, const uint8_t* ptrptr, const dl_type_desc* sub_type ) { - uintptr_t offset = *(uintptr_t*)( ptrptr ); - if( offset == (uintptr_t)-1 ) - return DL_ERROR_OK; - if( offset == 0 ) + const uint8_t* ptr = *(const uint8_t**)( ptrptr ); + if( ptr == 0 ) return DL_ERROR_OK; for (size_t i = 0; i < unpack_ctx->ptrs.Len(); ++i) - if( unpack_ctx->ptrs[i].offset == offset ) + if( unpack_ctx->ptrs[i].ptr == ptr ) return DL_ERROR_OK; - unpack_ctx->ptrs.Add( { offset, 0 } ); + unpack_ctx->ptrs.Add( { ptr, 0 } ); dl_txt_unpack_write_indent( writer, unpack_ctx ); - dl_txt_unpack_ptr( writer, offset ); + dl_txt_unpack_ptr( writer, unpack_ctx, ptr ); dl_binary_writer_write( writer, " : ", 3 ); - DL_ASSERT_MSG(offset < unpack_ctx->packed_instance_size, "Trying to read from offset %d in a buffer of size %d bytes", offset, unpack_ctx->packed_instance_size); - - dl_error_t err = dl_txt_unpack_struct( dl_ctx, unpack_ctx, writer, sub_type, &unpack_ctx->packed_instance[offset] ); + dl_error_t err = dl_txt_unpack_struct( dl_ctx, unpack_ctx, writer, sub_type, ptr ); if( DL_ERROR_OK != err ) return err; // TODO: extra , at last elem =/ dl_binary_writer_write( writer, ",\n", 2 ); - return dl_txt_unpack_write_subdata( dl_ctx, unpack_ctx, writer, sub_type, &unpack_ctx->packed_instance[offset] ); + return dl_txt_unpack_write_subdata( dl_ctx, unpack_ctx, writer, sub_type, ptr ); } static dl_error_t dl_txt_unpack_write_subdata_ptr_array( dl_ctx_t dl_ctx, @@ -665,9 +661,8 @@ static dl_error_t dl_txt_unpack_write_member_subdata( dl_ctx_t dl_ctx, dl_txt_un const dl_type_desc* subtype = dl_internal_find_type( dl_ctx, member->type_id ); if( subtype->flags & DL_TYPE_FLAG_HAS_SUBDATA ) { - uintptr_t array_offset = *(uintptr_t*)(member_data); - uint32_t array_count = *(uint32_t*)(member_data + sizeof(uintptr_t)); - const uint8_t* array = unpack_ctx->packed_instance + array_offset; + const uint8_t* array = *(const uint8_t**) member_data; + uint32_t array_count = *(uint32_t*)(member_data + sizeof(uintptr_t)); for( uint32_t i = 0; i < array_count; ++i ) { dl_error_t err = dl_txt_unpack_write_subdata( dl_ctx, unpack_ctx, writer, subtype, array + i * subtype->size[DL_PTR_SIZE_HOST] ); @@ -678,9 +673,8 @@ static dl_error_t dl_txt_unpack_write_member_subdata( dl_ctx_t dl_ctx, dl_txt_un break; case DL_TYPE_STORAGE_PTR: { - uintptr_t array_offset = *(uintptr_t*)(member_data); - uint32_t array_count = *(uint32_t*)(member_data + sizeof(uintptr_t) ); - const uint8_t* array = unpack_ctx->packed_instance + array_offset; + const uint8_t* array = *(const uint8_t**)member_data; + uint32_t array_count = *(uint32_t*)(member_data + sizeof(uintptr_t) ); return dl_txt_unpack_write_subdata_ptr_array( dl_ctx, unpack_ctx, writer, @@ -709,8 +703,6 @@ static dl_error_t dl_txt_unpack_write_subdata( dl_ctx_t dl_ctx, dl_txt_unpack_ct // TODO: check if type is not set at all ... size_t type_offset = dl_internal_union_type_offset(dl_ctx, type, DL_PTR_SIZE_HOST); - DL_ASSERT_MSG(size_t((struct_data + type_offset) - unpack_ctx->packed_instance) < unpack_ctx->packed_instance_size, "Trying to read from offset %d in a buffer of size %d bytes", size_t((struct_data + type_offset) - unpack_ctx->packed_instance), unpack_ctx->packed_instance_size); - // find member index from union type ... uint32_t union_type = *((uint32_t*)(struct_data + type_offset)); const dl_member_desc* member = dl_internal_union_type_to_member(dl_ctx, type, union_type); @@ -736,8 +728,6 @@ static dl_error_t dl_txt_unpack_struct( dl_ctx_t dl_ctx, dl_txt_unpack_ctx* unpa // TODO: check if type is not set at all ... size_t type_offset = dl_internal_union_type_offset( dl_ctx, type, DL_PTR_SIZE_HOST ); - DL_ASSERT_MSG(size_t((struct_data + type_offset) - unpack_ctx->packed_instance) < unpack_ctx->packed_instance_size, "Trying to read from offset %d in a buffer of size %d bytes", size_t((struct_data + type_offset) - unpack_ctx->packed_instance), unpack_ctx->packed_instance_size); - // find member index from union type ... uint32_t union_type = *((uint32_t*)(struct_data + type_offset)); const dl_member_desc* member = dl_internal_union_type_to_member(dl_ctx, type, union_type); @@ -808,19 +798,10 @@ static dl_error_t dl_txt_unpack_root( dl_ctx_t dl_ctx, dl_txt_unpack_ctx* unpack return DL_ERROR_OK; } -dl_error_t dl_txt_unpack( dl_ctx_t dl_ctx, dl_typeid_t type, - const unsigned char* packed_instance, size_t packed_instance_size, - char* out_txt_instance, size_t out_txt_instance_size, - size_t* produced_bytes ) +dl_error_t dl_txt_unpack_loaded( dl_ctx_t dl_ctx, dl_typeid_t type, + const void* loaded_packed_instance, char* out_txt_instance, + size_t out_txt_instance_size, size_t* produced_bytes ) { - dl_data_header* header = (dl_data_header*)packed_instance; - - if( packed_instance_size < sizeof(dl_data_header) ) return DL_ERROR_MALFORMED_DATA; - if( header->id == DL_INSTANCE_ID_SWAPED ) return DL_ERROR_ENDIAN_MISMATCH; - if( header->id != DL_INSTANCE_ID ) return DL_ERROR_MALFORMED_DATA; - if( header->version != DL_INSTANCE_VERSION) return DL_ERROR_VERSION_MISMATCH; - if( header->root_instance_type != type ) return DL_ERROR_TYPE_MISMATCH; - dl_binary_writer writer; dl_binary_writer_init( &writer, (uint8_t*)out_txt_instance, @@ -830,21 +811,44 @@ dl_error_t dl_txt_unpack( dl_ctx_t dl_ctx, dl_typeid_t typ DL_ENDIAN_HOST, DL_PTR_SIZE_HOST ); - dl_txt_unpack_ctx unpackctx(dl_ctx->alloc); - unpackctx.packed_instance = packed_instance + sizeof(dl_data_header); - unpackctx.packed_instance_size = packed_instance_size; - unpackctx.indent = 0; - unpackctx.has_ptrs = false; + dl_txt_unpack_ctx unpackctx( dl_ctx->alloc ); + unpackctx.packed_instance = reinterpret_cast(loaded_packed_instance); + unpackctx.indent = 0; + unpackctx.has_ptrs = false; + + unpackctx.ptrs.Add( { unpackctx.packed_instance, type } ); - dl_error_t err = dl_txt_unpack_root( dl_ctx, &unpackctx, &writer, header->root_instance_type ); + dl_error_t err = dl_txt_unpack_root( dl_ctx, &unpackctx, &writer, type ); if( produced_bytes ) *produced_bytes = writer.needed_size; return err; } +dl_error_t dl_txt_unpack_loaded_calc_size( dl_ctx_t dl_ctx, dl_typeid_t type, const void* packed_instance, size_t* out_txt_instance_size ) +{ + return dl_txt_unpack_loaded( dl_ctx, type, packed_instance, 0x0, 0, out_txt_instance_size ); +} + +dl_error_t dl_txt_unpack( dl_ctx_t dl_ctx, dl_typeid_t type, + unsigned char* packed_instance, size_t packed_instance_size, + char* out_txt_instance, size_t out_txt_instance_size, + size_t* produced_bytes ) +{ + void* loaded_instance; + size_t consumed; + dl_error_t err = dl_instance_load_inplace( dl_ctx, type, (uint8_t*)packed_instance, packed_instance_size, &loaded_instance, &consumed ); + if( err != DL_ERROR_OK ) + return err; + err = dl_txt_unpack_loaded( dl_ctx, type, (const void*)loaded_instance, out_txt_instance, out_txt_instance_size, produced_bytes ); + if( err != DL_ERROR_OK ) + return err; + err = dl_instance_store( dl_ctx, type, loaded_instance, (uint8_t*)packed_instance, packed_instance_size, &consumed ); + return err; +} + dl_error_t dl_txt_unpack_calc_size( dl_ctx_t dl_ctx, dl_typeid_t type, - const unsigned char* packed_instance, size_t packed_instance_size, + unsigned char* packed_instance, size_t packed_instance_size, size_t* out_txt_instance_size ) { return dl_txt_unpack( dl_ctx, type, packed_instance, packed_instance_size, 0x0, 0, out_txt_instance_size ); diff --git a/src/dl_typelib_read_txt.cpp b/src/dl_typelib_read_txt.cpp index 00e3f83..fea1104 100644 --- a/src/dl_typelib_read_txt.cpp +++ b/src/dl_typelib_read_txt.cpp @@ -200,6 +200,7 @@ static const dl_builtin_type* dl_find_builtin_type( const char* name ) } dl_type_t dl_make_type( dl_type_atom_t atom, dl_type_storage_t storage ); +dl_error_t dl_txt_pack_internal( dl_ctx_t dl_ctx, const char* txt_instance, unsigned char* out_buffer, size_t out_buffer_size, size_t* produced_bytes, bool use_fast_ptr_patch ); static void dl_load_txt_build_default_data( dl_ctx_t ctx, dl_txt_read_ctx* read_state, unsigned int member_index ) { @@ -242,7 +243,8 @@ static void dl_load_txt_build_default_data( dl_ctx_t ctx, dl_txt_read_ctx* read_ uint8_t* pack_buffer = (uint8_t*)dl_alloc( &ctx->alloc, prod_bytes ); - err = dl_txt_pack( ctx, def_buffer, pack_buffer, prod_bytes, 0x0 ); + bool use_fast_ptr_patch = false; + err = dl_txt_pack_internal( ctx, def_buffer, pack_buffer, prod_bytes, 0x0, use_fast_ptr_patch ); if( err != DL_ERROR_OK ) dl_txt_read_failed( ctx, read_state, DL_ERROR_INVALID_DEFAULT_VALUE, "failed to pack default-value for member \"%s\" with error \"%s\"", dl_internal_member_name( ctx, member ), diff --git a/src/dl_types.h b/src/dl_types.h index fd97ff2..4ae3d74 100755 --- a/src/dl_types.h +++ b/src/dl_types.h @@ -48,7 +48,7 @@ #endif static const uint32_t DL_UNUSED DL_TYPELIB_VERSION = 5; // format version for type-libraries. -static const uint32_t DL_UNUSED DL_INSTANCE_VERSION = 1; // format version for instances. +static const uint32_t DL_UNUSED DL_INSTANCE_VERSION = 2; // format version for instances. static const uint32_t DL_UNUSED DL_INSTANCE_VERSION_SWAPED = dl_swap_endian_uint32( DL_INSTANCE_VERSION ); static const uint32_t DL_UNUSED DL_TYPELIB_ID = ('D'<< 24) | ('L' << 16) | ('T' << 8) | 'L'; static const uint32_t DL_UNUSED DL_TYPELIB_ID_SWAPED = dl_swap_endian_uint32( DL_TYPELIB_ID ); @@ -89,8 +89,8 @@ typedef enum static const uintptr_t DL_NULL_PTR_OFFSET[2] = { - (uintptr_t)0xFFFFFFFF, // DL_PTR_SIZE_32BIT - (uintptr_t)-1 // DL_PTR_SIZE_64BIT + (uintptr_t)0, // DL_PTR_SIZE_32BIT + (uintptr_t)0 // DL_PTR_SIZE_64BIT }; struct dl_typelib_header @@ -117,7 +117,9 @@ struct dl_data_header dl_typeid_t root_instance_type; uint32_t instance_size; uint8_t is_64_bit_ptr; // currently uses uint8 instead of bitfield to be compiler-compliant. - uint8_t pad[7]; + uint8_t not_using_ptr_chain_patching; // currently uses uint8 instead of bitfield to be compiler-compliant. + uint8_t pad[2]; + uint32_t first_pointer_to_patch; }; enum dl_ptr_size_t diff --git a/tests/dl_tests_error.cpp b/tests/dl_tests_error.cpp index 37ec6f8..89987d4 100644 --- a/tests/dl_tests_error.cpp +++ b/tests/dl_tests_error.cpp @@ -113,11 +113,11 @@ TEST_F(DLError, version_mismatch_returned) conv.instance = packed; unsigned int* instance_version; instance_version = conv.instance_version + 1; - EXPECT_EQ(1u, *instance_version); + EXPECT_EQ(2u, *instance_version); *instance_version = 0xFFFFFFFF; conv.instance = swaped; instance_version = conv.instance_version + 1; - EXPECT_EQ(0x01000000u, *instance_version); + EXPECT_EQ(0x02000000u, *instance_version); *instance_version = 0xFFFFFFFF; // test all functions in... diff --git a/tests/dl_tests_ptr.cpp b/tests/dl_tests_ptr.cpp index 383b9b8..7e245e8 100644 --- a/tests/dl_tests_ptr.cpp +++ b/tests/dl_tests_ptr.cpp @@ -215,7 +215,7 @@ TYPED_TEST(DLBase, ptr_chain_long) ptrs[i] = { (uint32_t) i, &ptrs[i + 1] }; ptrs[DL_ARRAY_LENGTH(ptrs) - 1] = { DL_ARRAY_LENGTH(ptrs) - 1, &ptrs[0] }; - PtrChain loaded[1024]; + PtrChain loaded[1030]; this->do_the_round_about(PtrChain::TYPE_ID, &ptrs, &loaded, sizeof(loaded)); From 1fef82f0e11a2463742981b36b0f2e5f49b26c4d Mon Sep 17 00:00:00 2001 From: Christian Nilsendahl Date: Fri, 4 Nov 2022 10:16:13 +0100 Subject: [PATCH 4/8] Fixed so default values are functional through reflection Fixed pointer patching of default values Fixed merging of identical strings when packing json data --- src/dl.cpp | 84 ++++++++----------------------------- src/dl_convert.cpp | 3 +- src/dl_patch_ptr.cpp | 26 ++++++------ src/dl_patch_ptr.h | 4 +- src/dl_reflect.cpp | 2 +- src/dl_txt_pack.cpp | 4 +- src/dl_typelib_read_bin.cpp | 21 +++++----- tests/dl_tests_array.cpp | 15 +++++++ tests/dl_tests_txt.cpp | 48 ++++++++++++++++++++- tests/unittest.tld | 3 +- 10 files changed, 109 insertions(+), 101 deletions(-) diff --git a/src/dl.cpp b/src/dl.cpp index 5b4f4b5..0232247 100644 --- a/src/dl.cpp +++ b/src/dl.cpp @@ -1,4 +1,5 @@ #include "dl_types.h" +#include "dl_store.h" #include "dl_swap.h" #include "dl_binary_writer.h" #include "dl_patch_ptr.h" @@ -139,63 +140,6 @@ dl_error_t DL_DLL_EXPORT dl_instance_load_inplace( dl_ctx_t dl_ctx, return DL_ERROR_OK; } -struct CDLBinStoreContext -{ - CDLBinStoreContext( uint8_t* out_data, size_t out_data_size, bool is_dummy, dl_allocator alloc ) - : written_ptrs(alloc) - , ptrs(alloc) - , strings(alloc) - { - dl_binary_writer_init( &writer, out_data, out_data_size, is_dummy, DL_ENDIAN_HOST, DL_ENDIAN_HOST, DL_PTR_SIZE_HOST ); - } - - uintptr_t FindWrittenPtr( void* ptr ) - { - if( ptr == 0 ) - return (uintptr_t)0; - - size_t len = written_ptrs.Len(); - for( size_t i = 0; i < len; ++i ) - if( written_ptrs[i].ptr == ptr ) - return written_ptrs[i].pos; - - return (uintptr_t)0; - } - - void AddWrittenPtr( const void* ptr, uintptr_t pos ) - { - written_ptrs.Add( { pos, ptr } ); - } - - uint32_t GetStringOffset(const char* str, uint32_t length, uint32_t hash) - { - for (size_t i = 0; i < strings.Len(); ++i) - if (strings[i].hash == hash && strings[i].length == length && strcmp(str, strings[i].str) == 0) - return strings[i].offset; - - return 0; - } - - dl_binary_writer writer; - - struct SWrittenPtr - { - uintptr_t pos; - const void* ptr; - }; - CArrayStatic written_ptrs; - CArrayStatic ptrs; - - struct SString - { - const char* str; - uint32_t length; - uint32_t hash; - uint32_t offset; - }; - CArrayStatic strings; -}; - static void dl_internal_store_string( const uint8_t* instance, CDLBinStoreContext* store_ctx ) { char* str = *(char**)instance; @@ -205,7 +149,7 @@ static void dl_internal_store_string( const uint8_t* instance, CDLBinStoreContex return; } uint32_t hash = dl_internal_hash_string(str); - uint32_t length = (uint32_t) strlen(str); + int length = (int) strlen(str); uintptr_t offset = store_ctx->GetStringOffset(str, length, hash); if (offset == 0) // Merge identical strings { @@ -221,9 +165,9 @@ static void dl_internal_store_string( const uint8_t* instance, CDLBinStoreContex dl_binary_writer_write( &store_ctx->writer, &offset, sizeof(uintptr_t) ); } -static dl_error_t dl_internal_instance_store( dl_ctx_t dl_ctx, const dl_type_desc* type, uint8_t* instance, CDLBinStoreContext* store_ctx ); +static dl_error_t dl_internal_instance_store( dl_ctx_t dl_ctx, const dl_type_desc* type, const uint8_t* instance, CDLBinStoreContext* store_ctx ); -static dl_error_t dl_internal_store_ptr( dl_ctx_t dl_ctx, uint8_t* instance, const dl_type_desc* sub_type, CDLBinStoreContext* store_ctx ) +static dl_error_t dl_internal_store_ptr( dl_ctx_t dl_ctx, const uint8_t* instance, const dl_type_desc* sub_type, CDLBinStoreContext* store_ctx ) { uint8_t* data = *(uint8_t**)instance; uintptr_t offset = store_ctx->FindWrittenPtr( data ); @@ -262,7 +206,7 @@ static dl_error_t dl_internal_store_ptr( dl_ctx_t dl_ctx, uint8_t* instance, con return DL_ERROR_OK; } -static dl_error_t dl_internal_store_array( dl_ctx_t dl_ctx, dl_type_storage_t storage_type, const dl_type_desc* sub_type, uint8_t* instance, uint32_t count, uintptr_t size, CDLBinStoreContext* store_ctx ) +static dl_error_t dl_internal_store_array( dl_ctx_t dl_ctx, dl_type_storage_t storage_type, const dl_type_desc* sub_type, const uint8_t* instance, uint32_t count, uintptr_t size, CDLBinStoreContext* store_ctx ) { switch( storage_type ) { @@ -301,7 +245,7 @@ static dl_error_t dl_internal_store_array( dl_ctx_t dl_ctx, dl_type_storage_t st return DL_ERROR_OK; } -static dl_error_t dl_internal_store_member( dl_ctx_t dl_ctx, const dl_member_desc* member, uint8_t* instance, CDLBinStoreContext* store_ctx ) +dl_error_t dl_internal_store_member( dl_ctx_t dl_ctx, const dl_member_desc* member, const uint8_t* instance, CDLBinStoreContext* store_ctx ) { dl_type_atom_t atom_type = member->AtomType(); dl_type_storage_t storage_type = member->StorageType(); @@ -374,8 +318,8 @@ static dl_error_t dl_internal_store_member( dl_ctx_t dl_ctx, const dl_member_des uintptr_t size = 0; const dl_type_desc* sub_type = 0x0; - uint8_t* data_ptr = instance; - uint32_t count = *(uint32_t*)( data_ptr + sizeof(void*) ); + const uint8_t* data_ptr = instance; + uint32_t count = *(const uint32_t*)( data_ptr + sizeof(void*) ); uintptr_t offset = 0; @@ -414,7 +358,7 @@ static dl_error_t dl_internal_store_member( dl_ctx_t dl_ctx, const dl_member_des dl_binary_writer_seek_set( &store_ctx->writer, pos ); if( !store_ctx->writer.dummy ) - store_ctx->ptrs.Add( dl_binary_writer_tell( &store_ctx->writer ) ); + store_ctx->ptrs.Add( dl_binary_writer_tell( &store_ctx->writer ) ); } // make room for ptr @@ -422,6 +366,8 @@ static dl_error_t dl_internal_store_member( dl_ctx_t dl_ctx, const dl_member_des // write count dl_binary_writer_write( &store_ctx->writer, &count, sizeof(uint32_t) ); + if (sizeof(uintptr_t) == 8) + dl_binary_writer_write_zero( &store_ctx->writer, sizeof(uint32_t) ); } return DL_ERROR_OK; @@ -437,7 +383,7 @@ static dl_error_t dl_internal_store_member( dl_ctx_t dl_ctx, const dl_member_des return DL_ERROR_OK; } -static dl_error_t dl_internal_instance_store( dl_ctx_t dl_ctx, const dl_type_desc* type, uint8_t* instance, CDLBinStoreContext* store_ctx ) +static dl_error_t dl_internal_instance_store( dl_ctx_t dl_ctx, const dl_type_desc* type, const uint8_t* instance, CDLBinStoreContext* store_ctx ) { bool last_was_bitfield = false; @@ -496,7 +442,11 @@ dl_error_t dl_instance_store( dl_ctx_t dl_ctx, dl_typeid_t type_id, return DL_ERROR_TYPE_NOT_FOUND; bool store_ctx_is_dummy = out_buffer_size == 0; - CDLBinStoreContext store_context( out_buffer, out_buffer_size, store_ctx_is_dummy, dl_ctx->alloc ); + dl_binary_writer writer; + dl_binary_writer_init( &writer, out_buffer, out_buffer_size, store_ctx_is_dummy, DL_ENDIAN_HOST, DL_ENDIAN_HOST, DL_PTR_SIZE_HOST ); + CArrayStatic ptrs( dl_ctx->alloc ); + CArrayStatic strings( dl_ctx->alloc ); + CDLBinStoreContext store_context( writer, dl_ctx->alloc, ptrs, strings ); dl_data_header* header = (dl_data_header*)out_buffer; if( out_buffer_size > 0 ) diff --git a/src/dl_convert.cpp b/src/dl_convert.cpp index 91dc129..bd423d5 100644 --- a/src/dl_convert.cpp +++ b/src/dl_convert.cpp @@ -295,7 +295,6 @@ static dl_error_t dl_internal_convert_collect_instances_from_member( dl_ctx_t dl_internal_find_type(ctx, member->type_id), base_data, convert_ctx ); - break; case DL_TYPE_STORAGE_STR: dl_internal_convert_collect_instances_from_str_array( member_data, member->inline_array_cnt(), @@ -513,7 +512,7 @@ static dl_error_t dl_internal_convert_write_member( dl_ctx_t ctx, case DL_TYPE_STORAGE_PTR: { uintptr_t offset = dl_internal_read_ptr_data(member_data, conv_ctx.src_endian, conv_ctx.src_ptr_size); - dl_internal_convert_save_patch_pos( &conv_ctx, writer, dl_binary_writer_tell( writer), offset ); + dl_internal_convert_save_patch_pos( &conv_ctx, writer, dl_binary_writer_tell(writer), offset ); } break; default: diff --git a/src/dl_patch_ptr.cpp b/src/dl_patch_ptr.cpp index 610aa7d..ce231b5 100644 --- a/src/dl_patch_ptr.cpp +++ b/src/dl_patch_ptr.cpp @@ -3,7 +3,7 @@ struct dl_patched_ptrs { - CArrayStatic addresses; + CArrayStatic addresses; explicit dl_patched_ptrs( dl_allocator alloc ) : addresses( alloc ) @@ -37,14 +37,14 @@ static uintptr_t dl_internal_patch_ptr( uint8_t* ptrptr, uintptr_t patch_distanc static void dl_internal_patch_struct( dl_ctx_t ctx, const dl_type_desc* type, uint8_t* struct_data, - uintptr_t base_address, + uint8_t* base_address, uintptr_t patch_distance, dl_patched_ptrs* patched_payloads ); static void dl_internal_patch_ptr_instance( dl_ctx_t ctx, const dl_type_desc* sub_type, uint8_t* ptr_data, - uintptr_t base_address, + uint8_t* base_address, uintptr_t patch_distance, dl_patched_ptrs* patched_payloads ) { @@ -52,12 +52,12 @@ static void dl_internal_patch_ptr_instance( dl_ctx_t ctx, if( offset == 0x0 ) return; - uintptr_t ptr = (uintptr_t)base_address + offset; + uint8_t* ptr = base_address + offset; if( patched_payloads->patched( ptr ) ) return; patched_payloads->add( ptr ); - dl_internal_patch_struct( ctx, sub_type, (uint8_t*)ptr, base_address, patch_distance, patched_payloads ); + dl_internal_patch_struct( ctx, sub_type, ptr, base_address, patch_distance, patched_payloads ); } static void dl_internal_patch_str_array( uint8_t* array_data, uint32_t count, uintptr_t patch_distance ) @@ -70,7 +70,7 @@ static void dl_internal_patch_ptr_array( dl_ctx_t ctx, uint8_t* array_data, uint32_t count, const dl_type_desc* sub_type, - uintptr_t base_address, + uint8_t* base_address, uintptr_t patch_distance, dl_patched_ptrs* patched_payloads ) { @@ -82,7 +82,7 @@ static void dl_internal_patch_struct_array( dl_ctx_t ctx, const dl_type_desc* type, uint8_t* array_data, uint32_t count, - uintptr_t base_address, + uint8_t* base_address, uintptr_t patch_distance, dl_patched_ptrs* patched_payloads ) { @@ -97,7 +97,7 @@ static void dl_internal_patch_struct_array( dl_ctx_t ctx, static void dl_internal_patch_member( dl_ctx_t ctx, const dl_member_desc* member, uint8_t* member_data, - uintptr_t base_address, + uint8_t* base_address, uintptr_t patch_distance, dl_patched_ptrs* patched_payloads ) { @@ -213,7 +213,7 @@ static void dl_internal_patch_member( dl_ctx_t ctx, static void dl_internal_patch_union( dl_ctx_t ctx, const dl_type_desc* type, uint8_t* union_data, - uintptr_t base_address, + uint8_t* base_address, uintptr_t patch_distance, dl_patched_ptrs* patched_payloads ) { @@ -230,7 +230,7 @@ static void dl_internal_patch_union( dl_ctx_t ctx, static void dl_internal_patch_struct( dl_ctx_t ctx, const dl_type_desc* type, uint8_t* struct_data, - uintptr_t base_address, + uint8_t* base_address, uintptr_t patch_distance, dl_patched_ptrs* patched_payloads ) { @@ -254,7 +254,7 @@ static void dl_internal_patch_struct( dl_ctx_t ctx, void dl_internal_patch_member( dl_ctx_t ctx, const dl_member_desc* member, uint8_t* member_data, - uintptr_t base_address, + uint8_t* base_address, uintptr_t patch_distance) { dl_patched_ptrs patched(ctx->alloc); @@ -264,11 +264,11 @@ void dl_internal_patch_member( dl_ctx_t ctx, void dl_internal_patch_instance( dl_ctx_t ctx, const dl_type_desc* type, uint8_t* instance, - uintptr_t base_address, + uint8_t* base_address, uintptr_t patch_distance ) { dl_patched_ptrs patched(ctx->alloc); - patched.add( (uintptr_t)instance ); + patched.add( instance ); if( type->flags & DL_TYPE_FLAG_IS_UNION ) { diff --git a/src/dl_patch_ptr.h b/src/dl_patch_ptr.h index 0cfabf7..eeb6989 100644 --- a/src/dl_patch_ptr.h +++ b/src/dl_patch_ptr.h @@ -15,7 +15,7 @@ void dl_internal_patch_instance( dl_ctx_t ctx, const dl_type_desc* type, uint8_t* instance, - uintptr_t base_address, + uint8_t* base_address, uintptr_t patch_distance ); /** @@ -30,7 +30,7 @@ void dl_internal_patch_instance( dl_ctx_t ctx, void dl_internal_patch_member( dl_ctx_t ctx, const dl_member_desc* member, uint8_t* member_data, - uintptr_t base_address, + uint8_t* base_address, uintptr_t patch_distance ); #endif // DL_PATCH_PTR_H_INCLUDED diff --git a/src/dl_reflect.cpp b/src/dl_reflect.cpp index a3df1cd..0224e96 100755 --- a/src/dl_reflect.cpp +++ b/src/dl_reflect.cpp @@ -131,7 +131,7 @@ dl_error_t DL_DLL_EXPORT dl_reflect_get_type_members( dl_ctx_t dl_ctx, dl_typeid if( member_desc.default_value_size && member_desc.default_value_size != member_desc.size[DL_PTR_SIZE_HOST] ) { uint8_t* default_data = dl_ctx->default_data + member_desc.default_value_offset; - dl_internal_patch_member( dl_ctx, &member_desc, default_data, 0, (uintptr_t)default_data ); + dl_internal_patch_member( dl_ctx, &member_desc, default_data, 0, (uintptr_t)default_data - sizeof(dl_data_header) ); } } dl_ctx->default_data_patched = true; // This is not thread safe. It would be better to avoid moving around default data and keep it patched like meta data diff --git a/src/dl_txt_pack.cpp b/src/dl_txt_pack.cpp index c84701c..aa34b50 100755 --- a/src/dl_txt_pack.cpp +++ b/src/dl_txt_pack.cpp @@ -1,6 +1,7 @@ /* copyright (c) 2010 Fredrik Kihlander, see LICENSE for more info */ #include "dl_types.h" +#include "dl_store.h" #include
#include "dl_binary_writer.h" #include "dl_patch_ptr.h" @@ -9,7 +10,6 @@ #include #include #include -#include #include #include @@ -1364,7 +1364,6 @@ dl_error_t dl_txt_pack_internal( dl_ctx_t dl_ctx, const char* txt_instance, unsi header.root_instance_type = dl_internal_typeid_of( dl_ctx, root_type ); header.instance_size = uint32_t(dl_binary_writer_needed_size( &writer ) - sizeof( dl_data_header )); header.is_64_bit_ptr = sizeof( void* ) == 8 ? 1 : 0; - header.first_pointer_to_patch = pointers.Len() ? (uint32_t)pointers[0] : 0; uintptr_t offset_shift = sizeof( uintptr_t ) * 4; if( dl_binary_writer_needed_size( &writer ) >= ( 1ULL << offset_shift ) || !use_fast_ptr_patch ) @@ -1380,6 +1379,7 @@ dl_error_t dl_txt_pack_internal( dl_ctx_t dl_ctx, const char* txt_instance, unsi uintptr_t offset = *(uintptr_t*)&packctx.writer->data[pointers[i]]; *(uintptr_t*)&packctx.writer->data[pointers[i]] = offset | ( ( (uintptr_t)( pointers[i + 1] - pointers[i] ) ) << offset_shift ); } + header.first_pointer_to_patch = (uint32_t)pointers[0]; } } } diff --git a/src/dl_typelib_read_bin.cpp b/src/dl_typelib_read_bin.cpp index ef46d09..e8020c5 100644 --- a/src/dl_typelib_read_bin.cpp +++ b/src/dl_typelib_read_bin.cpp @@ -97,7 +97,7 @@ dl_error_t dl_context_load_type_library( dl_ctx_t dl_ctx, const unsigned char* l if( member_desc.default_value_size && member_desc.default_value_size != member_desc.size[DL_PTR_SIZE_HOST] ) { uint8_t* default_data = dl_ctx->default_data + member_desc.default_value_offset; - dl_internal_patch_member( dl_ctx, &member_desc, default_data, 0, static_cast(-(ptrdiff_t)default_data) ); + dl_internal_patch_member( dl_ctx, &member_desc, default_data, 0, static_cast( sizeof(dl_data_header) - (ptrdiff_t)default_data ) ); } } dl_ctx->default_data_patched = false; // This is not thread safe. It would be better to avoid moving around default data and keep it patched like meta data @@ -146,16 +146,15 @@ dl_error_t dl_context_load_type_library( dl_ctx_t dl_ctx, const unsigned char* l } } - memcpy( dl_ctx->type_ids + dl_ctx->type_count, lib_data + types_lookup_offset, sizeof( dl_typeid_t ) * header.type_count ); - memcpy( dl_ctx->enum_ids + dl_ctx->enum_count, lib_data + enums_lookup_offset, sizeof( dl_typeid_t ) * header.enum_count ); - memcpy( dl_ctx->type_descs + dl_ctx->type_count, lib_data + types_offset, sizeof( dl_type_desc ) * header.type_count ); - memcpy( dl_ctx->enum_descs + dl_ctx->enum_count, lib_data + enums_offset, sizeof( dl_enum_desc ) * header.enum_count ); - memcpy( dl_ctx->member_descs + dl_ctx->member_count, lib_data + members_offset, sizeof( dl_member_desc ) * header.member_count ); - memcpy( dl_ctx->enum_value_descs + dl_ctx->enum_value_count, lib_data + enum_values_offset, sizeof( dl_enum_value_desc ) * header.enum_value_count ); - memcpy( dl_ctx->enum_alias_descs + dl_ctx->enum_alias_count, lib_data + enum_aliases_offset, sizeof( dl_enum_alias_desc ) * header.enum_alias_count ); + memcpy( dl_ctx->type_ids + dl_ctx->type_count, lib_data + types_lookup_offset, sizeof( dl_typeid_t ) * header.type_count ); + memcpy( dl_ctx->enum_ids + dl_ctx->enum_count, lib_data + enums_lookup_offset, sizeof( dl_typeid_t ) * header.enum_count ); + memcpy( dl_ctx->type_descs + dl_ctx->type_count, lib_data + types_offset, sizeof( dl_type_desc ) * header.type_count ); + memcpy( dl_ctx->enum_descs + dl_ctx->enum_count, lib_data + enums_offset, sizeof( dl_enum_desc ) * header.enum_count ); + memcpy( dl_ctx->member_descs + dl_ctx->member_count, lib_data + members_offset, sizeof( dl_member_desc ) * header.member_count ); + memcpy( dl_ctx->enum_value_descs + dl_ctx->enum_value_count, lib_data + enum_values_offset, sizeof( dl_enum_value_desc ) * header.enum_value_count ); + memcpy( dl_ctx->enum_alias_descs + dl_ctx->enum_alias_count, lib_data + enum_aliases_offset, sizeof( dl_enum_alias_desc ) * header.enum_alias_count ); memcpy( dl_ctx->typedata_strings + dl_ctx->typedata_strings_size, lib_data + typedata_strings_offset, header.typeinfo_strings_size ); - if(header.c_includes_size) - memcpy( dl_ctx->c_includes + dl_ctx->c_includes_size, lib_data + c_includes_offset, header.c_includes_size ); + memcpy( dl_ctx->c_includes + dl_ctx->c_includes_size, lib_data + c_includes_offset, header.c_includes_size ); if( DL_ENDIAN_HOST == DL_ENDIAN_BIG ) { @@ -243,7 +242,7 @@ dl_error_t dl_context_load_type_library( dl_ctx_t dl_ctx, const unsigned char* l void* loaded_instance; size_t consumed; dl_error_t err = dl_instance_load_inplace( dl_ctx, type_id, (uint8_t*)dl_ctx->metadatas[dl_ctx->metadatas_count + i], instance_size + sizeof( dl_data_header ), &loaded_instance, &consumed ); - DL_ASSERT( DL_ERROR_OK == err ); (void)err; + if( DL_ERROR_OK != err ) return err; DL_ASSERT( instance_size + sizeof( dl_data_header ) == consumed ); dl_ctx->metadata_infos[dl_ctx->metadatas_count + i] = loaded_instance; dl_ctx->metadata_typeinfos[dl_ctx->metadatas_count + i] = type_id; diff --git a/tests/dl_tests_array.cpp b/tests/dl_tests_array.cpp index 30f5dac..eb5a779 100644 --- a/tests/dl_tests_array.cpp +++ b/tests/dl_tests_array.cpp @@ -76,6 +76,13 @@ TYPED_TEST(DLBase, array_string_merge) this->do_the_round_about(StringArray::TYPE_ID, &original, &loaded[0], sizeof(loaded) / 2); + size_t merged_from_txt_size = 0; // Two long but identical strings, should be merged + EXPECT_DL_ERR_OK(dl_txt_unpack_loaded_calc_size(this->Ctx, original.TYPE_ID, (unsigned char*)&original, &merged_from_txt_size)); + char* txt_buffer = (char*)malloc( merged_from_txt_size ); + EXPECT_DL_ERR_OK(dl_txt_unpack_loaded(this->Ctx, original.TYPE_ID, (unsigned char*)&original, txt_buffer, merged_from_txt_size, &merged_from_txt_size)); + EXPECT_DL_ERR_OK(dl_txt_pack_calc_size(this->Ctx, txt_buffer, &merged_from_txt_size)); + free( txt_buffer ); + EXPECT_STREQ(original.Strings[0], loaded[0].Strings[0]); EXPECT_STREQ(original.Strings[1], loaded[0].Strings[1]); @@ -84,6 +91,13 @@ TYPED_TEST(DLBase, array_string_merge) this->do_the_round_about(StringArray::TYPE_ID, &original, &loaded[10], sizeof(loaded) / 2); + size_t unique_from_txt_size = 0; // Two shorter and different strings, should not be merged + EXPECT_DL_ERR_OK(dl_txt_unpack_loaded_calc_size(this->Ctx, original.TYPE_ID, (unsigned char*)&original, &unique_from_txt_size)); + txt_buffer = (char*)malloc( unique_from_txt_size ); + EXPECT_DL_ERR_OK(dl_txt_unpack_loaded(this->Ctx, original.TYPE_ID, (unsigned char*)&original, txt_buffer, unique_from_txt_size, &unique_from_txt_size)); + EXPECT_DL_ERR_OK(dl_txt_pack_calc_size(this->Ctx, txt_buffer, &unique_from_txt_size)); + free( txt_buffer ); + EXPECT_STREQ(original.Strings[0], loaded[10].Strings[0]); EXPECT_STREQ(original.Strings[1], loaded[10].Strings[1]); @@ -92,6 +106,7 @@ TYPED_TEST(DLBase, array_string_merge) size_t unique_store_size = 0; // Two shorter and different strings, should not be merged EXPECT_DL_ERR_OK(dl_instance_calc_size(this->Ctx, loaded[10].TYPE_ID, &loaded[10], &unique_store_size)); EXPECT_LT(merged_store_size, unique_store_size); // Check that long merged strings take less memory than short unique strings + EXPECT_LT(merged_from_txt_size, unique_from_txt_size); // Check that long merged strings take less memory than short unique strings } TYPED_TEST(DLBase, array_string_null) diff --git a/tests/dl_tests_txt.cpp b/tests/dl_tests_txt.cpp index 6793944..e76d9a5 100755 --- a/tests/dl_tests_txt.cpp +++ b/tests/dl_tests_txt.cpp @@ -194,6 +194,23 @@ TEST_F(DLText, default_value_with_data_in_struct) EXPECT_STREQ("who", loaded[0].Str); } +TEST_F(DLText, default_value_with_data_in_struct_merge_strings) +{ + // default-values should be set correctly! + + const char* text_data = STRINGIFY( { "DefaultWithOtherDataBefore" : { "t1" : "who" } } ); + + unsigned char out_data_text[1024]; + DefaultWithOtherDataBefore loaded[10]; // this is so ugly! + + EXPECT_DL_ERR_OK(dl_txt_pack(Ctx, text_data, out_data_text, sizeof(out_data_text), 0x0)); + EXPECT_DL_ERR_OK(dl_instance_load(Ctx, DefaultWithOtherDataBefore::TYPE_ID, loaded, sizeof(loaded), out_data_text, sizeof(out_data_text), 0x0)); + + EXPECT_STREQ("who", loaded[0].t1); + EXPECT_STREQ("who", loaded[0].Str); + EXPECT_EQ(loaded[0].Str, loaded[0].t1); +} + TEST_F(DLText, default_value_ptr) { // default-values should be set correctly! @@ -425,15 +442,42 @@ TEST_F(DLText, default_value_array_string) EXPECT_DL_ERR_OK(dl_txt_pack(Ctx, text_data, out_data_text, sizeof(out_data_text), 0x0)); EXPECT_DL_ERR_OK(dl_instance_load(Ctx, DefaultArrayStr::TYPE_ID, loaded, sizeof(loaded), out_data_text, sizeof(out_data_text), 0x0)); - EXPECT_EQ(4u, loaded[0].Arr.count); + EXPECT_EQ(5u, loaded[0].Arr.count); EXPECT_STREQ("cow", loaded[0].Arr[0]); EXPECT_STREQ("bells", loaded[0].Arr[1]); EXPECT_STREQ("are", loaded[0].Arr[2]); EXPECT_STREQ("cool", loaded[0].Arr[3]); + EXPECT_STREQ("bells", loaded[0].Arr[4]); + EXPECT_EQ(loaded[0].Arr[1], loaded[0].Arr[4]); +} + +TEST_F( DLText, default_value_double_array_string ) +{ + // default-values should be set correctly! + + const char* text_data = STRINGIFY( { "ArrayDefaultArrayStr" : { "Arr1" : { "Arr" : ["are"] } } } ); + + unsigned char out_data_text[1024]; + ArrayDefaultArrayStr loaded[10]; + + EXPECT_DL_ERR_OK( dl_txt_pack( Ctx, text_data, out_data_text, sizeof( out_data_text ), 0x0 ) ); + EXPECT_DL_ERR_OK( dl_instance_load( Ctx, ArrayDefaultArrayStr::TYPE_ID, loaded, sizeof( loaded ), out_data_text, sizeof( out_data_text ), 0x0 ) ); + + EXPECT_EQ( 1u, loaded[0].Arr1.Arr.count ); + EXPECT_EQ( 5u, loaded[0].Arr2.Arr.count ); + + EXPECT_STREQ( "are", loaded[0].Arr1.Arr[0] ); + EXPECT_STREQ( "cow", loaded[0].Arr2.Arr[0] ); + EXPECT_STREQ( "bells", loaded[0].Arr2.Arr[1] ); + EXPECT_STREQ( "are", loaded[0].Arr2.Arr[2] ); + EXPECT_STREQ( "cool", loaded[0].Arr2.Arr[3] ); + EXPECT_STREQ( "bells", loaded[0].Arr2.Arr[4] ); + EXPECT_EQ( loaded[0].Arr1.Arr[0], loaded[0].Arr2.Arr[2] ); + EXPECT_EQ( loaded[0].Arr2.Arr[1], loaded[0].Arr2.Arr[4] ); } -TEST_F(DLText, default_value_array_array) +TEST_F( DLText, default_value_array_array ) { // default-values should be set correctly! diff --git a/tests/unittest.tld b/tests/unittest.tld index d409bae..c342fff 100755 --- a/tests/unittest.tld +++ b/tests/unittest.tld @@ -277,7 +277,8 @@ ] }, ///* hard one "DefaultArrayPod" : { "members" : [ { "name" : "Arr", "type" : "uint32[]", "default" : [ 1, 3, 3, 7 ] } ] }, - "DefaultArrayStr" : { "members" : [ { "name" : "Arr", "type" : "string[]", "default" : [ "cow", "bells", "are", "cool" ] } ] }, + "DefaultArrayStr" : { "members" : [ { "name" : "Arr", "type" : "string[]", "default" : [ "cow", "bells", "are", "cool", "bells" ] } ] }, + "ArrayDefaultArrayStr" :{ "members" : [ { "name" : "Arr1", "type" : "DefaultArrayStr" }, { "name" : "Arr2", "type" : "DefaultArrayStr", "default" : { "Arr" : [ "cow", "bells", "are", "cool", "bells" ] } } ] }, "DefaultArrayEnum" : { "members" : [ { "name" : "Arr", "type" : "TestEnum1[]", "default" : [ "TESTENUM1_VALUE3", "TESTENUM1_VALUE1", "TESTENUM1_VALUE2", From 15af0c87bc7967a71a64863220b51867cc68bf46 Mon Sep 17 00:00:00 2001 From: Christian Nilsendahl Date: Fri, 4 Nov 2022 10:44:42 +0100 Subject: [PATCH 5/8] Removed pragma pack --- src/dl_store.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/dl_store.h b/src/dl_store.h index 3465f23..5fdb4cf 100644 --- a/src/dl_store.h +++ b/src/dl_store.h @@ -3,8 +3,6 @@ #ifndef DL_DL_STORE_H_INCLUDED #define DL_DL_STORE_H_INCLUDED -#pragma pack(push, 1) - struct CDLBinStoreContext { struct SString; @@ -69,6 +67,4 @@ struct CDLBinStoreContext dl_error_t dl_internal_store_member( dl_ctx_t dl_ctx, const dl_member_desc* member, const uint8_t* instance, CDLBinStoreContext* store_ctx ); -#pragma pack( pop ) - #endif // DL_DL_STORE_H_INCLUDED From 39f4844c03195f2603ab37e59744d7aa4b915b43 Mon Sep 17 00:00:00 2001 From: Christian Nilsendahl Date: Fri, 4 Nov 2022 10:47:41 +0100 Subject: [PATCH 6/8] Const correctness --- src/dl_store.h | 4 ++-- src/dl_types.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/dl_store.h b/src/dl_store.h index 5fdb4cf..b7e7a0e 100644 --- a/src/dl_store.h +++ b/src/dl_store.h @@ -15,7 +15,7 @@ struct CDLBinStoreContext { } - uintptr_t FindWrittenPtr( const void* ptr ) + uintptr_t FindWrittenPtr( const void* ptr ) const { if( ptr == 0 ) return (uintptr_t)0; @@ -33,7 +33,7 @@ struct CDLBinStoreContext written_ptrs.Add( { pos, ptr } ); } - uint32_t GetStringOffset( const char* str, int length, uint32_t hash ) + uint32_t GetStringOffset( const char* str, int length, uint32_t hash ) const { for( size_t i = 0; i < strings.Len(); ++i ) if( strings[i].hash == hash && strings[i].str.len == length && memcmp( str, strings[i].str.str, length ) == 0 ) diff --git a/src/dl_types.h b/src/dl_types.h index 4ae3d74..f90907d 100755 --- a/src/dl_types.h +++ b/src/dl_types.h @@ -385,7 +385,7 @@ class CArrayStatic } } - inline size_t Len() + inline size_t Len() const { return m_nElements; } From d1c50602795f575fd1c8cce2cedef8d851691173 Mon Sep 17 00:00:00 2001 From: Christian Nilsendahl Date: Tue, 27 Feb 2024 09:47:35 +0100 Subject: [PATCH 7/8] Massaged code to reduce warnings --- src/dl.cpp | 6 +++--- src/dl_store.h | 2 +- src/dl_txt_pack.cpp | 4 ++-- src/dl_typelib_read_txt.cpp | 8 -------- src/dl_types.h | 10 +++++++++- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/src/dl.cpp b/src/dl.cpp index c0d6dda..fbbe599 100644 --- a/src/dl.cpp +++ b/src/dl.cpp @@ -152,7 +152,7 @@ static void dl_internal_store_string( const uint8_t* instance, CDLBinStoreContex return; } uint32_t hash = dl_internal_hash_string(str); - int length = (int) strlen(str); + uint32_t length = (uint32_t) strlen(str); uintptr_t offset = store_ctx->GetStringOffset(str, length, hash); if (offset == 0) // Merge identical strings { @@ -160,7 +160,7 @@ static void dl_internal_store_string( const uint8_t* instance, CDLBinStoreContex dl_binary_writer_seek_end(&store_ctx->writer); offset = dl_binary_writer_tell(&store_ctx->writer); dl_binary_writer_write(&store_ctx->writer, str, length + 1); - store_ctx->strings.Add( {str, length, hash, (uint32_t) offset} ); + store_ctx->strings.Add( {{str, length}, hash, (uint32_t) offset} ); dl_binary_writer_seek_set(&store_ctx->writer, pos); } if( !store_ctx->writer.dummy ) @@ -369,7 +369,7 @@ dl_error_t dl_internal_store_member( dl_ctx_t dl_ctx, const dl_member_desc* memb // write count dl_binary_writer_write( &store_ctx->writer, &count, sizeof(uint32_t) ); - if (sizeof(uintptr_t) == 8) + if DL_CONSTANT_EXPRESSION(sizeof(uintptr_t) == 8) dl_binary_writer_write_zero( &store_ctx->writer, sizeof(uint32_t) ); } return DL_ERROR_OK; diff --git a/src/dl_store.h b/src/dl_store.h index b7e7a0e..01520fb 100644 --- a/src/dl_store.h +++ b/src/dl_store.h @@ -33,7 +33,7 @@ struct CDLBinStoreContext written_ptrs.Add( { pos, ptr } ); } - uint32_t GetStringOffset( const char* str, int length, uint32_t hash ) const + uint32_t GetStringOffset( const char* str, uint32_t length, uint32_t hash ) const { for( size_t i = 0; i < strings.Len(); ++i ) if( strings[i].hash == hash && strings[i].str.len == length && memcmp( str, strings[i].str.str, length ) == 0 ) diff --git a/src/dl_txt_pack.cpp b/src/dl_txt_pack.cpp index 4fbd753..91bf58a 100755 --- a/src/dl_txt_pack.cpp +++ b/src/dl_txt_pack.cpp @@ -282,7 +282,7 @@ static void dl_txt_pack_eat_and_write_string( dl_ctx_t dl_ctx, dl_txt_pack_ctx* size_t curr = dl_binary_writer_tell( packctx->writer ); dl_binary_writer_seek_end( packctx->writer ); strpos = dl_binary_writer_tell( packctx->writer ); - for( int i = 0; i < str.len; ++i ) + for( uint32_t i = 0; i < str.len; ++i ) { if( str.str[i] == '\\' ) { @@ -361,7 +361,7 @@ static bool dl_txt_pack_eat_and_write_ptr( dl_ctx_t dl_ctx, dl_txt_pack_ctx* pac static void dl_txt_pack_validate_c_symbol_key( dl_ctx_t dl_ctx, dl_txt_pack_ctx* packctx, dl_substr symbol ) { - for(int i = 0; i < symbol.len; ++i) + for(uint32_t i = 0; i < symbol.len; ++i) { if(!isalnum(symbol.str[i]) || symbol.str[i] == '_') dl_txt_read_failed( dl_ctx, &packctx->read_ctx, DL_ERROR_TXT_PARSE_ERROR, "found a non-valid key \"%.*s\" in data, did you miss a string-terminator? (\" or \')", symbol.len, symbol.str ); diff --git a/src/dl_typelib_read_txt.cpp b/src/dl_typelib_read_txt.cpp index 4186f64..88ccc59 100644 --- a/src/dl_typelib_read_txt.cpp +++ b/src/dl_typelib_read_txt.cpp @@ -9,14 +9,6 @@ #include // strtoul #include -#if (__cplusplus >= 201703L) // C++17 -# define DL_CONSTANT_EXPRESSION(expr) constexpr(expr) -#elif defined(_MSC_VER) -# define DL_CONSTANT_EXPRESSION(expr) (0, (expr)) -#else -# define DL_CONSTANT_EXPRESSION(expr) (expr) -#endif - template static inline T* dl_grow_array( dl_allocator* alloc, T* ptr, size_t* cap, size_t min_inc ) { diff --git a/src/dl_types.h b/src/dl_types.h index a41ce28..1a2be0e 100755 --- a/src/dl_types.h +++ b/src/dl_types.h @@ -27,6 +27,14 @@ #define DL_ARRAY_LENGTH(Array) (sizeof(Array)/sizeof(Array[0])) +#if (__cplusplus >= 201703L) // C++17 +# define DL_CONSTANT_EXPRESSION(expr) constexpr(expr) +#elif defined(_MSC_VER) +# define DL_CONSTANT_EXPRESSION(expr) (0, (expr)) +#else +# define DL_CONSTANT_EXPRESSION(expr) (expr) +#endif + #if defined( __LP64__ ) && !defined(__APPLE__) #define DL_INT64_FMT_STR "%ld" #define DL_UINT64_FMT_STR "%lu" @@ -329,7 +337,7 @@ struct dl_context struct dl_substr { const char* str; - int len; + uint32_t len; }; #ifdef _MSC_VER From 13e60e457329b1b032fb41104c9321afe857259c Mon Sep 17 00:00:00 2001 From: Christian Nilsendahl Date: Tue, 27 Feb 2024 09:56:34 +0100 Subject: [PATCH 8/8] More warning fixes --- src/dl_txt_pack.cpp | 4 ++-- src/dl_txt_read.h | 2 +- src/dl_typelib_read_bin.cpp | 2 +- src/dl_typelib_read_txt.cpp | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/dl_txt_pack.cpp b/src/dl_txt_pack.cpp index 91bf58a..f17ad53 100755 --- a/src/dl_txt_pack.cpp +++ b/src/dl_txt_pack.cpp @@ -821,7 +821,7 @@ static dl_error_t dl_txt_pack_write_default_value( dl_ctx_t dl_ctx, CDLBinStoreContext store_ctx( *packctx->writer, dl_ctx->alloc, packctx->ptrs, packctx->strings ); uint8_t* default_data = (uint8_t*)dl_alloc( &dl_ctx->alloc, member->default_value_size ); memcpy( default_data, member_default_value, member->default_value_size ); - dl_internal_patch_member( dl_ctx, member, default_data, (uintptr_t)0, (uintptr_t)default_data - sizeof( dl_data_header ) ); + dl_internal_patch_member( dl_ctx, member, default_data, nullptr, (uintptr_t)default_data - sizeof( dl_data_header ) ); err = dl_internal_store_member( dl_ctx, member, default_data, &store_ctx ); dl_free( &dl_ctx->alloc, default_data ); } @@ -1054,7 +1054,7 @@ static const dl_substr dl_txt_eat_object_key( dl_txt_read_ctx* readctx ) if(key_start != key_end) { res.str = key_start; - res.len = (int)(key_end - key_start); + res.len = (uint32_t)(key_end - key_start); readctx->iter = res.str + res.len; return res; } diff --git a/src/dl_txt_read.h b/src/dl_txt_read.h index 8f1d8b3..86467e4 100644 --- a/src/dl_txt_read.h +++ b/src/dl_txt_read.h @@ -98,7 +98,7 @@ static dl_substr dl_txt_eat_string_quote( dl_txt_read_ctx* readctx, char quote ) if( *key_end == quote ) { res.str = key_start; - res.len = (int)(key_end - key_start); + res.len = (uint32_t)(key_end - key_start); readctx->iter = res.str + res.len + 1; return res; } diff --git a/src/dl_typelib_read_bin.cpp b/src/dl_typelib_read_bin.cpp index 5f50b0e..cb5a992 100644 --- a/src/dl_typelib_read_bin.cpp +++ b/src/dl_typelib_read_bin.cpp @@ -97,7 +97,7 @@ dl_error_t dl_context_load_type_library( dl_ctx_t dl_ctx, const unsigned char* l if( member_desc.default_value_size && member_desc.default_value_size != member_desc.size[DL_PTR_SIZE_HOST] ) { uint8_t* default_data = dl_ctx->default_data + member_desc.default_value_offset; - dl_internal_patch_member( dl_ctx, &member_desc, default_data, 0, static_cast( sizeof(dl_data_header) - (ptrdiff_t)default_data ) ); + dl_internal_patch_member( dl_ctx, &member_desc, default_data, 0, static_cast( sizeof(dl_data_header) - (size_t)default_data ) ); } } dl_ctx->default_data_patched = false; // This is not thread safe. It would be better to avoid moving around default data and keep it patched like meta data diff --git a/src/dl_typelib_read_txt.cpp b/src/dl_typelib_read_txt.cpp index 88ccc59..1a28832 100644 --- a/src/dl_typelib_read_txt.cpp +++ b/src/dl_typelib_read_txt.cpp @@ -1131,7 +1131,7 @@ static void dl_context_load_txt_type_library_read_member( dl_ctx_t ctx, dl_txt_r while(*end != ',' && *end != '}') ++end; } default_val.str = start; - default_val.len = (int)(end - start); + default_val.len = (uint32_t)(end - start); read_state->iter = end; } else if( strncmp( "const", key.str, 5) == 0)