Added support for includes

README.md

@@ -119,13 +119,22 @@ Type-libs in text-format follows this format.
     "c_includes" : ["../../tests/dl_test_included.h", 
                     "<some/system/header.h>"],
 
+	// "tld_includes" is a list of other type libraries which this type library uses types from. If the types only are used as
+	// metadata then including the type library is enough, but if the types are used for members then the type lib c header
+    // also needs to be added to c_includes above. These tld_includes can be either binary or text files.
+	// Note that the mapping from "include path" to "what it means" is interpreted outside of DL's library, thus it doesn't
+	// have to be filesystem paths, it can be any string. By default the dltlc tool treats them as file paths.
+	"tld_includes" : [
+		"to_include.bin",
+	],
+
     // A typelibrary can have any number of "enums" sections. In each of these sections enum-types are specified.
     // An enum is just the same thing as in c/c++ a collection of integer-values.
     "enums" : {
         // each key in "enums" declare an enum with that name, in this case an enum called "my_enum"
         "my_enum" : {
             // an enum can be declared "extern", declaring a type extern will make it NOT be generated in .h-files and
-            // the actual definition is expected to be found in "other ways" ( from other includes most likely ).
+            // the actual definition is expected to be found in "other ways" ( from other c_includes most likely ).
             // dl will however generate static_assert():s to check that all values are set correctly and exist.
             // Defaults to `false` if not set
             "extern" : true,

bam.lua

@@ -3,27 +3,35 @@ BUILD_PATH = "local"
 EXTERNALS_PATH = 'external'
 GTEST_PATH = PathJoin( EXTERNALS_PATH, 'gtest' )
 
-function dl_type_lib( tlc_file, dltlc, dl_tests )
+function dl_type_lib( tlc_file, dltlc, dl_tests, dependency )
 	local output_path = PathJoin( BUILD_PATH, 'generated' )
 	local out_file = PathJoin( output_path, PathFilename( PathBase( tlc_file ) ) )
-	local out_header    = out_file .. ".h"
-	local out_lib       = out_file .. ".bin"
-	local out_lib_h     = out_file .. ".bin.h"
-	local out_lib_txt_h = out_file .. ".txt.h"
+	local type_lib_outputs = {}
+	type_lib_outputs.out_header    = out_file .. ".h"
+	type_lib_outputs.out_lib       = out_file .. ".bin"
+	type_lib_outputs.out_lib_h     = out_file .. ".bin.h"
+	type_lib_outputs.out_lib_txt_h = out_file .. ".txt.h"
 
 	local BIN2HEX  = _bam_exe .. " -e tool/bin2hex.lua"
 
 	if family == "windows" then
 		dltlc = string.gsub( dltlc, "/", "\\" )
 	end
 
-	AddJob( out_lib,       "tlc " .. out_lib,    dltlc   .. " -o "    .. out_lib    .. " "     .. tlc_file,  tlc_file )
-	AddJob( out_lib_h,     "tlc " .. out_lib_h,  BIN2HEX .. " dst="   .. out_lib_h  .. " src=" .. out_lib,   out_lib )
-	AddJob( out_lib_txt_h, "tlc " .. out_lib_h,  BIN2HEX .. " dst="   .. out_lib_txt_h  .. " src=" .. tlc_file, tlc_file )
-	AddJob( out_header,    "tlc " .. out_header, dltlc   .. " -c -o " .. out_header .. " "     .. tlc_file,  tlc_file )
+	AddJob( type_lib_outputs.out_lib,       "tlc " .. type_lib_outputs.out_lib,    dltlc   .. " -o "    .. type_lib_outputs.out_lib        .. " "     .. tlc_file, tlc_file )
+	AddJob( type_lib_outputs.out_lib_h,     "tlc " .. type_lib_outputs.out_lib_h,  BIN2HEX .. " dst="   .. type_lib_outputs.out_lib_h      .. " src=" .. type_lib_outputs.out_lib, type_lib_outputs.out_lib )
+	AddJob( type_lib_outputs.out_lib_txt_h, "tlc " .. type_lib_outputs.out_lib_h,  BIN2HEX .. " dst="   .. type_lib_outputs.out_lib_txt_h  .. " src=" .. tlc_file, tlc_file )
+	AddJob( type_lib_outputs.out_header,    "tlc " .. type_lib_outputs.out_header, dltlc   .. " -c -o " .. type_lib_outputs.out_header     .. " "     .. tlc_file, tlc_file )
 
 	AddDependency( tlc_file, dltlc )
-	AddDependency( dl_tests, out_lib_h )
+	if dependency then
+		AddDependency( tlc_file, dependency )
+	end
+	AddDependency( dl_tests, type_lib_outputs.out_lib_h )
+	AddDependency( dl_tests, type_lib_outputs.out_header )
+	AddDependency( dl_tests, type_lib_outputs.out_lib_txt_h )
+
+	return type_lib_outputs
 end
 
 function DefaultSettings( platform, config, compiler )
@@ -269,13 +277,14 @@ dltlc    = Link( build_settings, "dltlc",    Compile( dl_settings, CollectRecurs
 dl_tests = Link( test_settings,  "dl_tests", Compile( test_settings, Collect("tests/*.cpp") ), dl_lib, gtest_lib )
 dlbench  = Link( test_settings,  "dlbench",  Compile( test_settings, Collect("benchmark/*.cpp") ), dl_lib )
 
-tl1 = dl_type_lib( "tests/sized_enums.tld", dltlc, dl_tests )
-tl2 = dl_type_lib( "tests/unittest.tld",    dltlc, dl_tests )
-tl3 = dl_type_lib( "tests/unittest2.tld",   dltlc, dl_tests )
-tl4 = dl_type_lib( "tests/small.tld",       dltlc, dl_tests )
+tl1 = dl_type_lib( "tests/to_include.tld",  dltlc, dl_tests )
+tl2 = dl_type_lib( "tests/sized_enums.tld", dltlc, dl_tests )
+tl3 = dl_type_lib( "tests/unittest.tld",    dltlc, dl_tests, tl1.out_lib )
+tl4 = dl_type_lib( "tests/unittest2.tld",   dltlc, dl_tests, "tests/to_include.tld" )
+tl5 = dl_type_lib( "tests/small.tld",       dltlc, dl_tests )
 tlbench = dl_type_lib( "benchmark/dlbench.tld", dltlc, dl_tests )
 
-dl_test_valid_c = Compile( dl_settings, Collect( "tests/*.c" ), tl1, tl2, tl3, tl4, tlbench )
+dl_test_valid_c = Compile( dl_settings, Collect( "tests/*.c" ) )
 
 local    test_args = ""
 local py_test_args = ""

include/dl/dl.h

@@ -87,6 +87,7 @@ typedef enum DL_NODISCARD
 	DL_ERROR_TXT_MULTIPLE_MEMBERS_IN_UNION_SET,
 
 	DL_ERROR_TYPELIB_MISSING_MEMBERS_IN_TYPE,
+	DL_ERROR_TYPELIB_MISSING_INCLUDE_HANDLER,
 
 	DL_ERROR_UTIL_FILE_NOT_FOUND,
 	DL_ERROR_UTIL_FILE_TYPE_MISMATCH,

include/dl/dl_typelib.h

@@ -19,19 +19,49 @@ extern "C" {
 
 #include <dl/dl.h>
 
+/*
+	Function pointer type: dl_include_handler
+	    Is responsible for locating and loading a type-library into dl_ctx.
+
+	Parameters:
+	    handler_ctx        - context passed in to dl_context_load_txt_type_library()
+	    dl_ctx             - dl-context to load typelib into.
+	    include_identifier - the text given in the include, commonly a relative path but could be any string
+
+	Return:
+		DL_ERROR_OK on success. 
+ */
+typedef dl_error_t ( *dl_include_handler )( void* handler_ctx, dl_ctx_t dl_ctx, const char* include_identifier );
+
+/*
+	Struct: dl_load_txt_type_library_params_t
+		Passed with configuration parameters to dl_context_load_txt_type_library.
+		This struct is open to change in later versions of dl.
+
+	Members:
+		include_handler - a callback function which tries to load the given type library, can be nullptr
+		handler_ctx     - this will be passed straight down to the include_handler callback
+*/
+typedef struct dl_load_type_library_params_t
+{
+	dl_include_handler include_handler;
+	void* handler_ctx;
+} dl_load_type_library_params_t;
+
 /*
  	Function: dl_context_load_txt_type_library
 		Load type-library from text representation into dl_ctx.
 
 	Parameters:
-		dl_ctx        - dl-context to load typelib into..
+		dl_ctx        - dl-context to load typelib into.
 		lib_data      - pointer to loaded txt-type library to load.
 		lib_data_size - size of string pointed to by lib_data.
+		load_params   - optional parameters which controls how the type library is loaded.
 
 	Note:
 		This function do not have the same rules of memory allocation and might allocate memory behind the scenes.
  */
-dl_error_t DL_DLL_EXPORT dl_context_load_txt_type_library( dl_ctx_t dl_ctx, const char* lib_data, size_t lib_data_size );
+dl_error_t DL_DLL_EXPORT dl_context_load_txt_type_library( dl_ctx_t dl_ctx, const char* lib_data, size_t lib_data_size, const dl_load_type_library_params_t* load_params );
 
 /*
 	Function: dl_context_write_type_library

src/dl.cpp

@@ -593,6 +593,7 @@ const char* dl_error_to_string( dl_error_t error )
 		DL_ERR_TO_STR(DL_ERROR_TXT_MULTIPLE_MEMBERS_IN_UNION_SET);
 
 		DL_ERR_TO_STR(DL_ERROR_TYPELIB_MISSING_MEMBERS_IN_TYPE);
+		DL_ERR_TO_STR(DL_ERROR_TYPELIB_MISSING_INCLUDE_HANDLER);
 
 		DL_ERR_TO_STR(DL_ERROR_UTIL_FILE_NOT_FOUND);
 		DL_ERR_TO_STR(DL_ERROR_UTIL_FILE_TYPE_MISMATCH);

src/dl_txt_read.h

@@ -5,13 +5,17 @@
 #include <setjmp.h>
 #include "dl_types.h"
 
+#include <dl/dl_typelib.h>
+
 struct dl_txt_read_ctx
 {
 	jmp_buf jumpbuf;
 	const char* start;
 	const char* end;
 	const char* iter;
 	dl_error_t err;
+	dl_include_handler include_handler;
+	void* include_handler_ctx;
 };
 
 

src/dl_typelib_read_bin.cpp

@@ -159,18 +159,26 @@ dl_error_t dl_context_load_type_library( dl_ctx_t dl_ctx, const unsigned char* l
 		if(dl_ctx->type_descs[ dl_ctx->type_count + i ].comment != UINT32_MAX)
 			dl_ctx->type_descs[ dl_ctx->type_count + i ].comment += td_str_offset;
 		dl_ctx->type_descs[ dl_ctx->type_count + i ].member_start += dl_ctx->member_count;
-		if( dl_ctx->type_descs[ dl_ctx->type_count + i ].metadata_start )
-			dl_ctx->type_descs[ dl_ctx->type_count + i ].metadata_start += dl_ctx->metadatas_count;
+		dl_ctx->type_descs[ dl_ctx->type_count + i ].metadata_start += dl_ctx->metadatas_count;
+
+		if( dl_ctx->performing_include )
+		{
+			dl_ctx->type_descs[ dl_ctx->type_count + i ].flags |= (uint32_t)DL_TYPE_FLAG_IS_EXTERNAL;
+			dl_ctx->type_descs[ dl_ctx->type_count + i ].flags &= ~(uint32_t)DL_TYPE_FLAG_VERIFY_EXTERNAL_SIZE_ALIGN;
+		}
 	}
 
 	for( unsigned int i = 0; i < header.member_count; ++i )
 	{
 		dl_ctx->member_descs[ dl_ctx->member_count + i ].name += td_str_offset;
-		if( dl_ctx->member_descs[dl_ctx->member_count + i].metadata_start )
-			dl_ctx->member_descs[dl_ctx->member_count + i].metadata_start += dl_ctx->metadatas_count;
+		if( dl_ctx->member_descs[ dl_ctx->member_count + i ].metadata_start )
+			dl_ctx->member_descs[ dl_ctx->member_count + i ].metadata_start += dl_ctx->metadatas_count;
 
-		if( dl_ctx->member_descs[dl_ctx->member_count + i].default_value_offset != UINT32_MAX )
-			dl_ctx->member_descs[dl_ctx->member_count + i].default_value_offset += (uint32_t)dl_ctx->default_data_size;
+		if( dl_ctx->member_descs[ dl_ctx->member_count + i ].default_value_offset != UINT32_MAX )
+			dl_ctx->member_descs[ dl_ctx->member_count + i ].default_value_offset += (uint32_t)dl_ctx->default_data_size;
+
+		if( dl_ctx->performing_include )
+			dl_ctx->member_descs[ dl_ctx->member_count + i ].flags &= ~(uint32_t)DL_MEMBER_FLAG_VERIFY_EXTERNAL_SIZE_OFFSET;
 	}
 
 	for( unsigned int i = 0; i < header.enum_count; ++i )
@@ -182,6 +190,11 @@ dl_error_t dl_context_load_type_library( dl_ctx_t dl_ctx, const unsigned char* l
 		dl_ctx->enum_descs[ dl_ctx->enum_count + i ].alias_start += dl_ctx->enum_alias_count;
 		if( dl_ctx->enum_descs[ dl_ctx->enum_count + i ].metadata_start )
 			dl_ctx->enum_descs[ dl_ctx->enum_count + i ].metadata_start += dl_ctx->metadatas_count;
+		if( dl_ctx->performing_include )
+		{
+			dl_ctx->enum_descs[ dl_ctx->enum_count + i ].flags |= (uint32_t)DL_TYPE_FLAG_IS_EXTERNAL;
+			dl_ctx->enum_descs[ dl_ctx->enum_count + i ].flags &= ~(uint32_t)DL_TYPE_FLAG_VERIFY_EXTERNAL_SIZE_ALIGN;
+		}
 	}
 
 	for( unsigned int i = 0; i < header.enum_alias_count; ++i )