From 9c1a0547bb7e24a7e25feb6dcbddd4c40f7ff990 Mon Sep 17 00:00:00 2001 From: Christopher Hall Date: Thu, 3 Dec 2015 13:18:44 +0800 Subject: [PATCH 01/24] FreeBSD compatibility Add cgo items for FreeBSD clang compiler. Since FreeBSD already has a libucl pkg there is no need to extract libucl sources. And to avoid having gnumake installed add a BSDmakefile that just checks that lib ucl was installed by: sudo pkg install libucl Minor item: add cast to fix warning. Signed-off-by: Christopher Hall --- BSDmakefile | 6 ++++++ go-libucl.h | 2 +- libucl.go | 4 ++++ 3 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 BSDmakefile diff --git a/BSDmakefile b/BSDmakefile new file mode 100644 index 0000000..7f6e260 --- /dev/null +++ b/BSDmakefile @@ -0,0 +1,6 @@ +# BSDmakefile + +.PHONY: all +all: + pkg info -s libucl >/dev/null || (echo "first: sudo pkg install libucl" && false) + go test diff --git a/go-libucl.h b/go-libucl.h index 2ca1d1d..77c4f95 100644 --- a/go-libucl.h +++ b/go-libucl.h @@ -24,7 +24,7 @@ static inline bool _go_macro_handler(const unsigned char *data, size_t len, void // Returns the ucl_macro_handler that we have, since we can't get this // type from cgo. static inline ucl_macro_handler _go_macro_handler_func() { - return &_go_macro_handler; + return (ucl_macro_handler)&_go_macro_handler; } // This just converts an int to a void*, because Go doesn't let us do that diff --git a/libucl.go b/libucl.go index 6542301..6afa547 100644 --- a/libucl.go +++ b/libucl.go @@ -2,4 +2,8 @@ package libucl // #cgo CFLAGS: -Ivendor/libucl/include -Wno-int-to-void-pointer-cast // #cgo LDFLAGS: -Lvendor/libucl -lucl +// +// #cgo freebsd CFLAGS: -I/usr/local/include -Wno-int-to-void-pointer-cast +// #cgo freebsd LDFLAGS: -L/usr/local/lib -lucl +// import "C" From 3ac2803785192cbe35714567a3587bee00995732 Mon Sep 17 00:00:00 2001 From: Christopher Hall Date: Thu, 3 Dec 2015 13:29:19 +0800 Subject: [PATCH 02/24] Support for signed/unsigned 64 bit integer types Add a couple of simple int64/uint64 tests. Signed-off-by: Christopher Hall --- decoder.go | 22 ++++++++++++++++++++++ decoder_test.go | 40 ++++++++++++++++++++++++++++------------ object.go | 4 ++++ 3 files changed, 54 insertions(+), 12 deletions(-) diff --git a/decoder.go b/decoder.go index 6454891..1bec18b 100644 --- a/decoder.go +++ b/decoder.go @@ -24,6 +24,12 @@ func decode(name string, o *Object, result reflect.Value) error { return decodeIntoInterface(name, o, result) case reflect.Int: return decodeIntoInt(name, o, result) + case reflect.Int64: + return decodeIntoInt(name, o, result) + case reflect.Uint: + return decodeIntoUint(name, o, result) + case reflect.Uint64: + return decodeIntoUint(name, o, result) case reflect.Map: return decodeIntoMap(name, o, result) case reflect.Ptr: @@ -73,6 +79,22 @@ func decodeIntoInt(name string, o *Object, result reflect.Value) error { return nil } +func decodeIntoUint(name string, o *Object, result reflect.Value) error { + switch o.Type() { + case ObjectTypeString: + i, err := strconv.ParseUint(o.ToString(), 0, result.Type().Bits()) + if err == nil { + result.SetUint(i) + } else { + return fmt.Errorf("cannot parse '%s' as int: %s", name, err) + } + default: + result.SetUint(o.ToUint()) + } + + return nil +} + func decodeIntoInterface(name string, o *Object, result reflect.Value) error { var set reflect.Value redecode := true diff --git a/decoder_test.go b/decoder_test.go index 15a03cb..dc3a6e1 100644 --- a/decoder_test.go +++ b/decoder_test.go @@ -7,16 +7,26 @@ import ( func TestObjectDecode_basic(t *testing.T) { type Basic struct { - Bool bool - BoolStr string - Str string - Num int - NumStr int + Bool bool + BoolStr string + Str string + Num int + NumStr int + Card uint + CardStr uint + Num64 int64 + Num64Str int64 + Card64 uint64 + Card64Str uint64 } obj := testParseString(t, ` - bool = true; str = bar; num = 7; numstr = "42"; - boolstr = true; + bool = true; boolstr = true; + str = bar; + num = 7; numstr = "42"; + card = 29; cardstr = "16384"; + num64 = 77; num64str = "4242"; + card64 = 2929; card64str = "524288"; `) defer obj.Close() @@ -26,11 +36,17 @@ func TestObjectDecode_basic(t *testing.T) { } expected := Basic{ - Bool: true, - BoolStr: "true", - Str: "bar", - Num: 7, - NumStr: 42, + Bool: true, + BoolStr: "true", + Str: "bar", + Num: 7, + NumStr: 42, + Card: 29, + CardStr: 16384, + Num64: 77, + Num64Str: 4242, + Card64: 2929, + Card64Str: 524288, } if !reflect.DeepEqual(result, expected) { diff --git a/object.go b/object.go index 7513be1..0b2f256 100644 --- a/object.go +++ b/object.go @@ -154,6 +154,10 @@ func (o *Object) ToInt() int64 { return int64(C.ucl_object_toint(o.object)) } +func (o *Object) ToUint() uint64 { + return uint64(C.ucl_object_toint(o.object)) +} + func (o *Object) ToFloat() float64 { return float64(C.ucl_object_todouble(o.object)) } From 6e53e8a4e8d32dfef12ec9fba127552a86d72fc1 Mon Sep 17 00:00:00 2001 From: Christopher Hall Date: Wed, 9 Dec 2015 15:05:07 +0800 Subject: [PATCH 03/24] fix the cmake command Signed-off-by: Christopher Hall --- Makefile | 4 ++-- decoder_test.go | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index e2efabb..ccfb08c 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,8 @@ libucl: vendor/libucl/$(LIBUCL_NAME) vendor/libucl/libucl.a: vendor/libucl cd vendor/libucl && \ - cmake cmake/ && \ - make + cmake . && \ + make vendor/libucl/libucl.dll: vendor/libucl cd vendor/libucl && \ diff --git a/decoder_test.go b/decoder_test.go index dc3a6e1..02b50a4 100644 --- a/decoder_test.go +++ b/decoder_test.go @@ -178,7 +178,7 @@ func TestObjectDecode_mapObject(t *testing.T) { expected := map[string]interface{}{ "foo": "bar", "bar": []map[string]interface{}{ - map[string]interface{}{ + { "baz": "what", }, }, @@ -205,10 +205,10 @@ func TestObjectDecode_mapObjectMultiple(t *testing.T) { expected := map[string]interface{}{ "foo": "bar", "bar": []map[string]interface{}{ - map[string]interface{}{ + { "baz": "what", }, - map[string]interface{}{ + { "port": 3000, }, }, @@ -242,7 +242,7 @@ func TestObjectDecode_mapReuseVal(t *testing.T) { expected := Result{ Struct: map[string]Struct{ - "foo": Struct{ + "foo": { Foo: "bar", Bar: "baz", }, @@ -421,7 +421,7 @@ func TestObjectDecode_mapStructNamed(t *testing.T) { } expected := map[string]Nested{ - "foo": Nested{ + "foo": { Name: "foo", Foo: "bar", }, @@ -457,7 +457,7 @@ func TestObjectDecode_mapStructObject(t *testing.T) { defer result.Value["foo"].Object.Close() expected := map[string]Nested{ - "foo": Nested{ + "foo": { Foo: "bar", Object: fooObj, }, From 01271ff9257e8759a9ed3d6371089c805c529966 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Mon, 8 Feb 2016 14:00:17 -0500 Subject: [PATCH 04/24] Switch to pkg-config for CFLAGS and LDFLAGS This allows for a more portable version of the wrapper, as it doesn't require libucl to sit in vendor/libucl, and instead sit elsewhere on the filesystem, which is a more realistic approach for an OS with a package manager. --- libucl.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libucl.go b/libucl.go index 6542301..5db9f11 100644 --- a/libucl.go +++ b/libucl.go @@ -1,5 +1,4 @@ package libucl -// #cgo CFLAGS: -Ivendor/libucl/include -Wno-int-to-void-pointer-cast -// #cgo LDFLAGS: -Lvendor/libucl -lucl +// #cgo pkg-config: libucl import "C" From 84cdc895d1c97f877c4e13a1538cbf71eb414f91 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Tue, 9 Feb 2016 09:54:39 -0500 Subject: [PATCH 05/24] Removed Makefiles for building libucl This is part of a move away from building libucl just for use in go, and using the OS's version instead. --- BSDmakefile | 6 ------ Makefile | 35 ----------------------------------- README.md | 17 ++++++----------- 3 files changed, 6 insertions(+), 52 deletions(-) delete mode 100644 BSDmakefile delete mode 100644 Makefile diff --git a/BSDmakefile b/BSDmakefile deleted file mode 100644 index 7f6e260..0000000 --- a/BSDmakefile +++ /dev/null @@ -1,6 +0,0 @@ -# BSDmakefile - -.PHONY: all -all: - pkg info -s libucl >/dev/null || (echo "first: sudo pkg install libucl" && false) - go test diff --git a/Makefile b/Makefile deleted file mode 100644 index e2efabb..0000000 --- a/Makefile +++ /dev/null @@ -1,35 +0,0 @@ -LIBUCL_NAME=libucl.a - -# If we're on Windows, we need to change some variables so things compile -# properly. -ifeq ($(OS), Windows_NT) - LIBUCL_NAME=libucl.dll -endif - -export CGO_CFLAGS CGO_LDFLAGS PATH - -all: libucl - go test - -libucl: vendor/libucl/$(LIBUCL_NAME) - -vendor/libucl/libucl.a: vendor/libucl - cd vendor/libucl && \ - cmake cmake/ && \ - make - -vendor/libucl/libucl.dll: vendor/libucl - cd vendor/libucl && \ - $(MAKE) -f Makefile.w32 && \ - cp .obj/libucl.dll . && \ - cp libucl.dll $(CURDIR) - -vendor/libucl: - rm -rf vendor/libucl - mkdir -p vendor/libucl - git clone https://github.com/vstakhov/libucl.git vendor/libucl - -clean: - rm -rf vendor - -.PHONY: all clean libucl test diff --git a/README.md b/README.md index 7fc356e..b1ddc4c 100644 --- a/README.md +++ b/README.md @@ -10,19 +10,14 @@ Windows. is not guaranteed. Additionally, it is not feature complete yet, though it is certainly usable for real purposes (we do!). -## Installation +## Prerequisites +* libucl (This is a wrapper for this library) +* pkg-config (cgo uses this for locate where libucl is) -Because we vendor the source of libucl, you can go ahead and get it directly. -We'll keep up to date with libucl. The package name is `libucl`. +## Installation ``` -$ go get github.com/mitchellh/go-libucl +$ go get github.com/draringi/go-libucl ``` -Documentation is available on GoDoc: http://godoc.org/github.com/mitchellh/go-libucl - -### Compiling Libucl - -Libucl should compile easily and cleanly on POSIX systems. - -On Windows, msys should be used. msys-regex needs to be compiled. +Documentation is available on GoDoc: http://godoc.org/github.com/draringi/go-libucl From a3dbaf4bc7854289fdc5a67a811a0843d42992dc Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Thu, 11 Feb 2016 14:41:57 -0500 Subject: [PATCH 06/24] Add information about the fork to the readme file --- README.md | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b1ddc4c..b420e4c 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,15 @@ # Libucl Library for Go +This version of go-libucl is forked from the [mitchellh version](https://github.com/mitchellh/go-libucl), +with the goal of having a version with a focus on using the OS's copy of libucl, in a portable manner. +As such, it uses pkg-config to determine the location of libucl. +In addition, patches from [hwx](https://github.com/bitmark-inc/go-libucl) have been +pulled in with 64-bit integer support. + go-libucl is a [libucl](https://github.com/vstakhov/libucl) library for [Go](http://golang.org). Rather than re-implement libucl in Go, this library uses cgo to bind directly to libucl. This allows the libucl project to be -the central source of knowledge. This project works on Mac OS X, Linux, and -Windows. +the central source of knowledge. This project has been tested on Linux and FreeBSD. **Warning:** This library is still under development and API compatibility is not guaranteed. Additionally, it is not feature complete yet, though From 3e04cbb4bd7c86d6de546cb68a5f4388dd8a9bbe Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Thu, 11 Feb 2016 15:05:00 -0500 Subject: [PATCH 07/24] Modernize gitignore and travis tests Changed gitignore to github's default golang gitignore, to better fit the project. Changed travis config to pull in libucl and install it before running tests. Changed travis config to use most recent version of go. --- .gitignore | 27 ++++++++++++++++++++++++--- .travis-ucl.sh | 8 ++++++++ .travis.yml | 6 ++++-- 3 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 .travis-ucl.sh diff --git a/.gitignore b/.gitignore index 4eeca0d..daf913b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,24 @@ -.vagrant/ -vendor/ -/libucl.dll +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof diff --git a/.travis-ucl.sh b/.travis-ucl.sh new file mode 100644 index 0000000..0dc3fa5 --- /dev/null +++ b/.travis-ucl.sh @@ -0,0 +1,8 @@ +#!/bin/sh +UCL_VERSION=0.7.3 +set -ex +mkdir /tmp/libucl +cd /tmp/libucl +wget https://github.com/vstakhov/libucl/archive/$UCL_VERSION.tar.gz +tar xzf $UCL_VERSION.tar.xz +cd libucl-$UCL_VERSION && ./autogen.sh && ./configure --prefix=/usr --enable-urls && make && sudo make install diff --git a/.travis.yml b/.travis.yml index dfc743f..9c03eb2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,8 @@ +sudo: required language: go go: - - 1.2.1 + - tip -script: make test +before_install: + - ./.travis-ucl.sh From 269fd58ad5a296c845ec8a9451b17225bb4caff2 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Thu, 11 Feb 2016 15:12:04 -0500 Subject: [PATCH 08/24] Fix permissions on travis ucl build script --- .travis-ucl.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .travis-ucl.sh diff --git a/.travis-ucl.sh b/.travis-ucl.sh old mode 100644 new mode 100755 From 007a2dea78c63bd159c0e56168f81759c924345d Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Thu, 11 Feb 2016 15:20:19 -0500 Subject: [PATCH 09/24] Fix typo in travit-ucl script In addition, remove build folder when done. --- .travis-ucl.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis-ucl.sh b/.travis-ucl.sh index 0dc3fa5..3d19bad 100755 --- a/.travis-ucl.sh +++ b/.travis-ucl.sh @@ -4,5 +4,6 @@ set -ex mkdir /tmp/libucl cd /tmp/libucl wget https://github.com/vstakhov/libucl/archive/$UCL_VERSION.tar.gz -tar xzf $UCL_VERSION.tar.xz +tar xzf $UCL_VERSION.tar.gz cd libucl-$UCL_VERSION && ./autogen.sh && ./configure --prefix=/usr --enable-urls && make && sudo make install +rm -fr /tmp/libucl From 59fe18d0593761d735201784bf1fecc47d8333f1 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Thu, 11 Feb 2016 15:28:37 -0500 Subject: [PATCH 10/24] Add build status image to readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index b420e4c..24ad273 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # Libucl Library for Go +[![Build Status](https://travis-ci.org/draringi/go-libucl.svg?branch=master)](https://travis-ci.org/draringi/go-libucl) This version of go-libucl is forked from the [mitchellh version](https://github.com/mitchellh/go-libucl), with the goal of having a version with a focus on using the OS's copy of libucl, in a portable manner. From 9fead9e0d8abe6ddda2a979e4e2e8cc6d459a090 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Thu, 11 Feb 2016 18:20:16 -0500 Subject: [PATCH 11/24] Updates to documentation --- decoder.go | 6 ++---- libucl.go | 2 ++ object.go | 22 +++++++++++++++------- parser.go | 17 ++++++++--------- 4 files changed, 27 insertions(+), 20 deletions(-) diff --git a/decoder.go b/decoder.go index 1bec18b..afa7102 100644 --- a/decoder.go +++ b/decoder.go @@ -43,8 +43,6 @@ func decode(name string, o *Object, result reflect.Value) error { default: return fmt.Errorf("%s: unsupported type: %s", name, result.Kind()) } - - return nil } func decodeIntoBool(name string, o *Object, result reflect.Value) error { @@ -347,8 +345,8 @@ func decodeIntoStruct(name string, o *Object, result reflect.Value) error { usedKeys := make(map[string]struct{}) decodedFields := make([]string, 0, len(fields)) - decodedFieldsVal := make([]reflect.Value, 0) - unusedKeysVal := make([]reflect.Value, 0) + var decodedFieldsVal []reflect.Value + var unusedKeysVal []reflect.Value for fieldType, field := range fields { if !field.IsValid() { // This should never happen diff --git a/libucl.go b/libucl.go index 5db9f11..bce1d4c 100644 --- a/libucl.go +++ b/libucl.go @@ -1,3 +1,5 @@ +// Package libucl provides golang bindings to libucl, a configuration library for +// UCL, the Universal Configuration Language. package libucl // #cgo pkg-config: libucl diff --git a/object.go b/object.go index 0b2f256..f51c804 100644 --- a/object.go +++ b/object.go @@ -43,7 +43,7 @@ const ( EmitYAML ) -// Free the memory associated with the object. This must be called when +// Close frees the memory associated with the object. This must be called when // you're done using it. func (o *Object) Close() error { C.ucl_object_unref(o.object) @@ -69,6 +69,7 @@ func (o *Object) Delete(key string) { C.ucl_object_delete_key(o.object, ckey) } +// Get returns the element with matching key. func (o *Object) Get(key string) *Object { ckey := C.CString(key) defer C.free(unsafe.Pointer(ckey)) @@ -83,7 +84,7 @@ func (o *Object) Get(key string) *Object { return result } -// Iterate over the objects in this object. +// Iterate returns an iterator that iterates over the objects in this object. // // The iterator must be closed when it is finished. // @@ -99,7 +100,7 @@ func (o *Object) Iterate(expand bool) *ObjectIter { } } -// Returns the key of this value/object as a string, or the empty +// Key returns the key of this value/object as a string, or the empty // string if the object doesn't have a key. func (o *Object) Key() string { return C.GoString(C.ucl_object_key(o.object)) @@ -118,10 +119,10 @@ func (o *Object) Len() uint { iter := o.Iterate(false) defer iter.Close() - var count uint = 0 + var count uint for obj := iter.Next(); obj != nil; obj = iter.Next() { obj.Close() - count += 1 + count++ } return count @@ -130,14 +131,14 @@ func (o *Object) Len() uint { return uint(o.object.len) } -// Increments the ref count associated with this. You have to call +// Ref increments the ref count associated with this. You have to call // close an additional time to free the memory. func (o *Object) Ref() error { C.ucl_object_ref(o.object) return nil } -// Returns the type that this object represents. +// Type returns the type that this object represents. func (o *Object) Type() ObjectType { return ObjectType(C.ucl_object_type(o.object)) } @@ -146,30 +147,37 @@ func (o *Object) Type() ObjectType { // Conversion Functions //------------------------------------------------------------------------ +// ToBool converts a UCL Object to a boolean value func (o *Object) ToBool() bool { return bool(C.ucl_object_toboolean(o.object)) } +// ToInt converts a UCL Object to a signed integer value func (o *Object) ToInt() int64 { return int64(C.ucl_object_toint(o.object)) } +// ToUint converts a UCL Object to an unsigned integer value func (o *Object) ToUint() uint64 { return uint64(C.ucl_object_toint(o.object)) } +// ToFloat converts a UCL Object to an floating point value func (o *Object) ToFloat() float64 { return float64(C.ucl_object_todouble(o.object)) } +// ToString converts a UCL Object to a string func (o *Object) ToString() string { return C.GoString(C.ucl_object_tostring(o.object)) } +// Close frees the object iterator func (o *ObjectIter) Close() { C.ucl_object_unref(o.object) } +// Next returns the next iterative UCL Object func (o *ObjectIter) Next() *Object { obj := C.ucl_iterate_object(o.object, &o.iter, C._Bool(o.expand)) if obj == nil { diff --git a/parser.go b/parser.go index bdf3e02..a20af46 100644 --- a/parser.go +++ b/parser.go @@ -13,16 +13,15 @@ import "C" type MacroFunc func(string) // ParserFlag are flags that can be used to initialize a parser. -// -// ParserKeyLowercase will lowercase all keys. -// -// ParserKeyZeroCopy will attempt to do a zero-copy parse if possible. type ParserFlag int const ( + // ParserKeyLowercase will lowercase all keys. ParserKeyLowercase ParserFlag = C.UCL_PARSER_KEY_LOWERCASE - ParserZeroCopy = C.UCL_PARSER_ZEROCOPY - ParserNoTime = C.UCL_PARSER_NO_TIME + // ParserZeroCopy will attempt to do a zero-copy parse if possible. + ParserZeroCopy = C.UCL_PARSER_ZEROCOPY + // ParserNoTime will treat time values as strings. + ParserNoTime = C.UCL_PARSER_NO_TIME ) // Keeps track of all the macros internally @@ -80,8 +79,8 @@ func (p *Parser) AddFile(path string) error { return nil } -// Closes the parser. Once it is closed it can no longer be used. You -// should always close the parser once you're done with it to clean up +// Close frees the parser. Once it is freed it can no longer be used. You +// should always free the parser once you're done with it to clean up // any unused memory. func (p *Parser) Close() { C.ucl_parser_free(p.parser) @@ -95,7 +94,7 @@ func (p *Parser) Close() { } } -// Retrieves the root-level object for a configuration. +// Object retrieves the root-level object for a configuration. func (p *Parser) Object() *Object { obj := C.ucl_parser_get_object(p.parser) if obj == nil { From 917379e77f6035251840b45c2a4316d930e0a38a Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Thu, 11 Feb 2016 18:36:52 -0500 Subject: [PATCH 12/24] More documentation updates, and readme updates as well --- README.md | 4 +++- object.go | 17 ++++++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 24ad273..994b4a1 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,10 @@ # Libucl Library for Go [![Build Status](https://travis-ci.org/draringi/go-libucl.svg?branch=master)](https://travis-ci.org/draringi/go-libucl) +[![GoDoc](https://godoc.org/github.com/draringi/go-libucl?status.svg)](https://godoc.org/github.com/draringi/go-libucl) This version of go-libucl is forked from the [mitchellh version](https://github.com/mitchellh/go-libucl), -with the goal of having a version with a focus on using the OS's copy of libucl, in a portable manner. +with the goal of having a version with a focus on using the OS's copy of libucl, in a portable manner, +as well as improve the Documentation quality. As such, it uses pkg-config to determine the location of libucl. In addition, patches from [hwx](https://github.com/bitmark-inc/go-libucl) have been pulled in with 64-bit integer support. diff --git a/object.go b/object.go index f51c804..b6a3532 100644 --- a/object.go +++ b/object.go @@ -21,25 +21,40 @@ type ObjectIter struct { type ObjectType int const ( + // ObjectTypeObject signifies a UCL Object (key/value pair) ObjectTypeObject ObjectType = iota + // ObjectTypeArray signifies a UCL array ObjectTypeArray + // ObjectTypeInt signifies an integer number ObjectTypeInt + // ObjectTypeFloat signifies a floating-point nmber ObjectTypeFloat + // ObjectTypeString signifies a string ObjectTypeString + // ObjectTypeBoolean signifies a boolean value (true/false) ObjectTypeBoolean + // ObjectTypeTime signifies time in seconds stored as a floating-point number ObjectTypeTime + // ObjectTypeUserData signifies an opaque user-provided pointer, typically + // used in macros ObjectTypeUserData + // ObjectTypeNull signifies a null/non-existant value ObjectTypeNull ) // Emitter is a type of built-in emitter that can be used to convert -// an object to another config format. +// an object to another config format. All Emitters except EmitConfig +// are considered lossy, and information such as implicit arrays can be lost. type Emitter int const ( + // EmitJSON is the canonic json notation (with spaces indented structure) EmitJSON Emitter = iota + // EmitJSONCompact is compact json notation (without spaces or newlines) EmitJSONCompact + // EmitConfig is UCL (nginx-like) EmitConfig + // EmitYAML is yaml inlined notation EmitYAML ) From 39dbadbb0d7fd920e6ef71a77a8a44bbedb7023a Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Sat, 13 Feb 2016 21:50:46 -0500 Subject: [PATCH 13/24] Added missing parser flag UCL_PARSER_NO_IMPLICIT_ARRAYS and missing string flags The corresponding go ParserFlag for C's UCL_PARSER_NO_IMPLICIT_ARRAYS is ParserNoImplicitArrays The string flags are for a function in the C code for creating objects from strings. A wrapper function in go is coming soon. TODO: * Add go wrapper for ucl_object_fromstring_common and related functions * Add tests for new wrappers * Further checks for missing functionality --- object.go | 25 +++++++++++++++++++++++++ parser.go | 10 ++++++---- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/object.go b/object.go index b6a3532..fee981c 100644 --- a/object.go +++ b/object.go @@ -204,3 +204,28 @@ func (o *ObjectIter) Next() *Object { return &Object{object: obj} } + +// StringFlag are flags used in the conversion of strings into UCL objects +type StringFlag int + +const ( + // StringEscape tells the converter to JSON escape the inputed string + StringEscape StringFlag = C.UCL_STRING_ESCAPE + // StringTrim tells the converter to trim leading and trailing whitespaces + StringTrim StringFlag = C.UCL_STRING_TRIM + // StringParseBoolean tells the converter to parse the inputted string as a boolean + StringParseBoolean StringFlag = C.UCL_STRING_PARSE_BOOLEAN + // StringParseInt tells the converter to parse the inputted string as an integer + StringParseInt StringFlag = C.UCL_STRING_PARSE_INT + // StringParseDouble tells the converter to parse the inputted string as a floating-point number + StringParseDouble StringFlag = C.UCL_STRING_PARSE_DOUBLE + // StringParseTime tells the converter to parse the inputted string as a time value, and treat as a floating-point number. + StringParseTime StringFlag = C.UCL_STRING_PARSE_TIME + // StringParseNumber tells the converter to parse the inputted string as a number (integer, floating-point or time) + StringParseNumber StringFlag = C.UCL_STRING_PARSE_TIME + // StringParse tells the converter to parse the inputted string + StringParse StringFlag = C.UCL_STRING_PARSE + // StringParseBytes tells the converter to parse the inputted string as being in bytes notation + // (e.g. 10k = 10*1024, not 10*1000) + StringParseBytes StringFlag = C.UCL_STRING_PARSE_BYTES +) diff --git a/parser.go b/parser.go index a20af46..1e12b99 100644 --- a/parser.go +++ b/parser.go @@ -19,14 +19,16 @@ const ( // ParserKeyLowercase will lowercase all keys. ParserKeyLowercase ParserFlag = C.UCL_PARSER_KEY_LOWERCASE // ParserZeroCopy will attempt to do a zero-copy parse if possible. - ParserZeroCopy = C.UCL_PARSER_ZEROCOPY + ParserZeroCopy ParserFlag = C.UCL_PARSER_ZEROCOPY // ParserNoTime will treat time values as strings. - ParserNoTime = C.UCL_PARSER_NO_TIME + ParserNoTime ParserFlag = C.UCL_PARSER_NO_TIME + // ParserNoImplicitArrays forces the creation explicit arrays instead of implicit ones + ParserNoImplicitArrays ParserFlag = C.UCL_PARSER_NO_IMPLICIT_ARRAYS ) // Keeps track of all the macros internally -var macros map[int]MacroFunc = nil -var macrosIdx int = 0 +var macros map[int]MacroFunc +var macrosIdx int var macrosLock sync.Mutex // Parser is responsible for parsing libucl data. From a2a3fc4bb4b423fe8a71c70d7bd1beee367a90ef Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Sun, 14 Feb 2016 12:00:28 -0500 Subject: [PATCH 14/24] Added UCL Object construction functions These functions wrap around libucl's ucl_object_from* functions. Additional tests were added to the test suite in object_test.go to test that these new functions operate correctly. --- object.go | 44 ++++++++++++++++++++--- object_test.go | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 133 insertions(+), 5 deletions(-) diff --git a/object.go b/object.go index fee981c..e919b33 100644 --- a/object.go +++ b/object.go @@ -217,15 +217,49 @@ const ( StringParseBoolean StringFlag = C.UCL_STRING_PARSE_BOOLEAN // StringParseInt tells the converter to parse the inputted string as an integer StringParseInt StringFlag = C.UCL_STRING_PARSE_INT - // StringParseDouble tells the converter to parse the inputted string as a floating-point number + // StringParseDouble tells the converter to parse the inputted string as a + // floating-point number StringParseDouble StringFlag = C.UCL_STRING_PARSE_DOUBLE - // StringParseTime tells the converter to parse the inputted string as a time value, and treat as a floating-point number. + // StringParseTime tells the converter to parse the inputted string as a time value, + // and treat as a floating-point number. StringParseTime StringFlag = C.UCL_STRING_PARSE_TIME - // StringParseNumber tells the converter to parse the inputted string as a number (integer, floating-point or time) + // StringParseNumber tells the converter to parse the inputted string as a number + // (integer, floating-point or time) StringParseNumber StringFlag = C.UCL_STRING_PARSE_TIME // StringParse tells the converter to parse the inputted string StringParse StringFlag = C.UCL_STRING_PARSE - // StringParseBytes tells the converter to parse the inputted string as being in bytes notation - // (e.g. 10k = 10*1024, not 10*1000) + // StringParseBytes tells the converter to parse the inputted string as being in + // bytes notation (e.g. 10k = 10*1024, not 10*1000) StringParseBytes StringFlag = C.UCL_STRING_PARSE_BYTES ) + +// NewObject creates a new UCL Object from a string, JSON escaping it in the process +func NewObject(data string) *Object { + obj := C.ucl_object_fromlstring(C.CString(data), C.size_t(len(data))) + return &Object{object: obj} +} + +// NewFormattedObject creates a new UCL Object from a string, according to the instructions +// given in the flags +func NewFormattedObject(data string, flags StringFlag) *Object { + obj := C.ucl_object_fromstring_common(C.CString(data), C.size_t(len(data)), uint32(flags)) + return &Object{object: obj} +} + +// NewIntegerObject creates a new UCL Object from a 64-bit integer +func NewIntegerObject(data int64) *Object { + obj := C.ucl_object_fromint(C.int64_t(data)) + return &Object{object: obj} +} + +// NewDoubleObject creates a new UCL Object from a 64-bit floating-point number +func NewDoubleObject(data float64) *Object { + obj := C.ucl_object_fromdouble(C.double(data)) + return &Object{object: obj} +} + +// NewBoolObject creates a new UCL Object from a boolean +func NewBoolObject(data bool) *Object { + obj := C.ucl_object_frombool(C.bool(data)) + return &Object{object: obj} +} diff --git a/object_test.go b/object_test.go index df4456f..edb02fb 100644 --- a/object_test.go +++ b/object_test.go @@ -234,3 +234,97 @@ func TestObjectToFloat_negativeOneThird(t *testing.T) { t.Fatalf("bad: %#v, expected: %v", v.ToFloat(), g) } } + +func TestIntToObject(t *testing.T) { + var testValue int64 = 42 + obj := NewIntegerObject(testValue) + defer obj.Close() + v := obj.ToInt() + if v != testValue { + t.Fatalf("bad: %d, expected: %d", v, testValue) + } +} + +func TestBoolToObject(t *testing.T) { + objTrue := NewBoolObject(true) + defer objTrue.Close() + if !objTrue.ToBool() { + t.Fatalf("bad: false, expected: true") + } + objFalse := NewBoolObject(false) + defer objFalse.Close() + if objFalse.ToBool() { + t.Fatalf("bad: true, expected: false") + } +} + +func TestFloatToObject(t *testing.T) { + testValue := -1.0 / 3.0 + obj := NewDoubleObject(testValue) + defer obj.Close() + if obj.ToFloat() != testValue { + t.Fatalf("bad: %#v, expected: %v", obj.ToFloat(), testValue) + } +} + +func TestSimpleStringToObject(t *testing.T) { + s := "Hello World!" + obj := NewObject(s) + defer obj.Close() + if obj.ToString() != s { + t.Fatalf("bad: \"%s\", expected: \"%s\"", obj.ToString(), s) + } +} + +func TestJSONEscapeStringToObject(t *testing.T) { + inputString := "complex\tstring\nfrom\"hell\"" + escapedString := "complex\\tstring\\nfrom\\\"hell\\\"" + obj := NewObject(inputString) + defer obj.Close() + if obj.ToString() != escapedString { + t.Fatalf("bad: \"%s\", expected: \"%s\"", obj.ToString(), escapedString) + } +} + +func TestStringIntToObject(t *testing.T) { + var i int64 = 42 + s := "42" + obj := NewFormattedObject(s, StringParseInt) + defer obj.Close() + if obj.ToInt() != i { + t.Fatalf("bad: %d, expected: %d", obj.ToInt(), i) + } +} + +func TestStringFloatToObject(t *testing.T) { + expectedValue := -1.0 / 3.0 + testString := "-0.3333333333333333148296162562473909929394721984863281" + obj := NewFormattedObject(testString, StringParseDouble) + defer obj.Close() + if obj.ToFloat() != expectedValue { + t.Fatalf("bad: %#v, expected: %v", obj.ToFloat(), expectedValue) + } +} + +func TestStringBoolToObject(t *testing.T) { + objFalse := NewFormattedObject("false", StringParseBoolean) + defer objFalse.Close() + if objFalse.ToBool() { + t.Fatal("bad: true, expected: false") + } + objTrue := NewFormattedObject("true", StringParseBoolean) + defer objTrue.Close() + if !objTrue.ToBool() { + t.Fatal("bad: false, expected: true") + } +} + +func TestStringTrimToObject(t *testing.T) { + rawString := " Hello World\n\t\n\t\n\t " + expectedResult := "Hello World" + obj := NewFormattedObject(rawString, StringTrim) + defer obj.Close() + if obj.ToString() != expectedResult { + t.Fatalf("bad: \"%s\", expected: \"%s\"", obj.ToString(), expectedResult) + } +} From 60350a139fb1b5d7657d459a298fdf12f9989953 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Sun, 14 Feb 2016 13:56:07 -0500 Subject: [PATCH 15/24] Added validation funtionality Validation functionality is provided via libucl's ucl_object_validate. A test has been written for a helper function, but no test has been written yet for the Validate method, as I have yet to get around to learning the json-schema format. --- go-libucl.h | 3 ++ validator.go | 73 +++++++++++++++++++++++++++++++++++++++++++++++ validator_test.go | 14 +++++++++ 3 files changed, 90 insertions(+) create mode 100644 validator.go create mode 100644 validator_test.go diff --git a/go-libucl.h b/go-libucl.h index 77c4f95..9f115c8 100644 --- a/go-libucl.h +++ b/go-libucl.h @@ -3,6 +3,7 @@ #include #include +#include static inline char *_go_uchar_to_char(const unsigned char *c) { return (char *)c; @@ -33,4 +34,6 @@ static inline void *_go_macro_index(int idx) { return (void *)idx; } +typedef struct ucl_schema_error ucl_schema_error_t; + #endif /* _GOLIBUCL_H_INCLUDED */ diff --git a/validator.go b/validator.go new file mode 100644 index 0000000..ec4e00d --- /dev/null +++ b/validator.go @@ -0,0 +1,73 @@ +package libucl + +import "errors" + +// #include "go-libucl.h" +import "C" + +// SchemaErrorCode is a progamatic way to ficgure out what went wrong during validation +type SchemaErrorCode int + +const ( + // SchemaOK means nothing went wrong (no error) + SchemaOK SchemaErrorCode = iota + // SchemaTypeMismatch means the type of object is wrong + SchemaTypeMismatch + // SchemaInvalidSchema means the provided schema is not valid according to json-schema draft 4 + SchemaInvalidSchema + // SchemaMissingProperty means at least one property of the object is missing + SchemaMissingProperty + // SchemaConstraint means a contraint was not met. + SchemaConstraint + // SchemaMissingDependency means a dependency was not met + SchemaMissingDependency + // SchemaGenericError is a generic error (matches UCL_SCHEMA_UNKNOWN in libucl) + SchemaGenericError +) + +// SchemaError contains information on an error found when validating an UCL Object +// against a provided json-schema style schema +type SchemaError struct { + code SchemaErrorCode + message string + object *Object +} + +// Validate validates the object againt a provided schema, which should conform to +// the 4th draft of the json-schema standard +func (o *Object) Validate(schema *Object) (SchemaError, error) { + var cError C.ucl_schema_error_t + var err error + var schemaError SchemaError + ok := C.ucl_object_validate(schema.object, o.object, &cError) + if !ok { + schemaError.code = SchemaErrorCode(cError.code) + schemaError.message = bufferToString(cError.msg) + schemaError.object = &Object{object: cError.obj} + err = errors.New(schemaError.message) + } + return schemaError, err +} + +// Convert a fixed char array to a go string, as gGo doesn't let us use +// [128]C.char as *C.char in C.GoString +func bufferToString(buffer [128]C.char) string { + var byteBuffer []byte + for _, c := range buffer { + if c == 0 { + break + } else { + byteBuffer = append(byteBuffer, byte(c)) + } + } + return string(byteBuffer) +} + +// Purely exists for the test, as cgo isn't supported inside tests +func byteToBufferAdapter(buffer [128]byte) string { + var tmp [128]C.char + for i, c := range buffer { + tmp[i] = C.char(c) + } + return bufferToString(tmp) +} diff --git a/validator_test.go b/validator_test.go new file mode 100644 index 0000000..2adb98f --- /dev/null +++ b/validator_test.go @@ -0,0 +1,14 @@ +package libucl + +import ( + "testing" +) + +func TestBufferToString(t *testing.T) { + expected := "Hello World!" + buffer := [128]byte{'H', 'e', 'l', 'l', 'o', ' ', 'W', 'o', 'r', 'l', 'd', '!', 0} + str := byteToBufferAdapter(buffer) + if str != expected { + t.Fatalf("bad: \"%s\", expected: \"%s\"", str, expected) + } +} From 1b751f92aef5ed14b272982ff6f396689ae96488 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Sun, 14 Feb 2016 14:25:05 -0500 Subject: [PATCH 16/24] Formatting changes to better show documentation on godoc.org --- object.go | 12 ++++++------ parser.go | 3 ++- validator.go | 3 ++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/object.go b/object.go index e919b33..cfcb54f 100644 --- a/object.go +++ b/object.go @@ -220,16 +220,16 @@ const ( // StringParseDouble tells the converter to parse the inputted string as a // floating-point number StringParseDouble StringFlag = C.UCL_STRING_PARSE_DOUBLE - // StringParseTime tells the converter to parse the inputted string as a time value, - // and treat as a floating-point number. + // StringParseTime tells the converter to parse the inputted string as a + // time value, and treat as a floating-point number. StringParseTime StringFlag = C.UCL_STRING_PARSE_TIME - // StringParseNumber tells the converter to parse the inputted string as a number - // (integer, floating-point or time) + // StringParseNumber tells the converter to parse the inputted string as a + // number (integer, floating-point or time) StringParseNumber StringFlag = C.UCL_STRING_PARSE_TIME // StringParse tells the converter to parse the inputted string StringParse StringFlag = C.UCL_STRING_PARSE - // StringParseBytes tells the converter to parse the inputted string as being in - // bytes notation (e.g. 10k = 10*1024, not 10*1000) + // StringParseBytes tells the converter to parse the inputted string as being + // in bytes notation (e.g. 10k = 10*1024, not 10*1000) StringParseBytes StringFlag = C.UCL_STRING_PARSE_BYTES ) diff --git a/parser.go b/parser.go index 1e12b99..97515ae 100644 --- a/parser.go +++ b/parser.go @@ -22,7 +22,8 @@ const ( ParserZeroCopy ParserFlag = C.UCL_PARSER_ZEROCOPY // ParserNoTime will treat time values as strings. ParserNoTime ParserFlag = C.UCL_PARSER_NO_TIME - // ParserNoImplicitArrays forces the creation explicit arrays instead of implicit ones + // ParserNoImplicitArrays forces the creation explicit arrays instead of + // implicit ones ParserNoImplicitArrays ParserFlag = C.UCL_PARSER_NO_IMPLICIT_ARRAYS ) diff --git a/validator.go b/validator.go index ec4e00d..742c9c5 100644 --- a/validator.go +++ b/validator.go @@ -13,7 +13,8 @@ const ( SchemaOK SchemaErrorCode = iota // SchemaTypeMismatch means the type of object is wrong SchemaTypeMismatch - // SchemaInvalidSchema means the provided schema is not valid according to json-schema draft 4 + // SchemaInvalidSchema means the provided schema is not valid according to + // json-schema draft 4 SchemaInvalidSchema // SchemaMissingProperty means at least one property of the object is missing SchemaMissingProperty From 95eb8f19b02b0beecc7f414b5b0eafcc92057c5c Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Sun, 14 Feb 2016 16:30:27 -0500 Subject: [PATCH 17/24] Added initial variable support This adds 3 methods to the Parser object: `RegisterVariable`, `SetFileVariables` and `AddFileAndSetVariables` (which combines `AddFile` with `SetFileVariables`) Unknown Variable Handler support to come This gets us part way towards resolving issue #5 --- parser.go | 39 +++++++++++++++++++++++++++++ parser_test.go | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+) diff --git a/parser.go b/parser.go index 97515ae..a9d59b0 100644 --- a/parser.go +++ b/parser.go @@ -150,3 +150,42 @@ func go_macro_call(id C.int, data *C.char, n C.int) C.bool { f(C.GoStringN(data, n)) return true } + +// SetFileVariables sets the standard file variables ($FILENAME and $CURDIR) based +// on the provided filepath. If the argument expand is true, the path will be expanded +// out to an absolute path +// +// For example, if the current directory is /etc/nginx, and you give a path of +// ../file.conf, with exand = false, $FILENAME = ../file.conf and $CURDIR = .., +// while with expand = true, $FILENAME = /etc/file.conf and $CURDIR = /etc +func (p *Parser) SetFileVariables(filepath string, expand bool) error { + cpath := C.CString(filepath) + defer C.free(unsafe.Pointer(cpath)) + ok := C.ucl_parser_set_filevars(p.parser, cpath, C.bool(expand)) + if !ok { + return errors.New("Unable to set file variables") + } + return nil +} + +// RegisterVariable adds a new variable to the parser, which can be accessed in +// the configuration file as $variable_name +func (p *Parser) RegisterVariable(variable, value string) { + cVariable := C.CString(variable) + defer C.free(unsafe.Pointer(cVariable)) + cValue := C.CString(value) + defer C.free(unsafe.Pointer(cValue)) + C.ucl_parser_register_variable(p.parser, cVariable, cValue) +} + +// AddFileAndSetVariables is a combination of AddFile and SetFileVariables. +// It is meant to be a simple way to do both actions in a single function call. +func (p *Parser) AddFileAndSetVariables(path string, expand bool) error { + err := p.AddFile(path) + if err != nil { + return err + } + + err = p.SetFileVariables(path, expand) + return err +} diff --git a/parser_test.go b/parser_test.go index ff59f52..4cf3c5b 100644 --- a/parser_test.go +++ b/parser_test.go @@ -2,6 +2,7 @@ package libucl import ( "io/ioutil" + "path" "testing" ) @@ -131,3 +132,69 @@ func TestParseString(t *testing.T) { t.Fatalf("bad: %d", obj.Len()) } } + +func TestRegisterVariable(t *testing.T) { + p := NewParser(0) + defer p.Close() + value := "bar" + p.RegisterVariable("FOO", value) + + err := p.AddString("foo = $FOO; baz = boo;") + if err != nil { + t.Fatalf("err: %s", err) + } + obj := p.Object() + if obj == nil { + t.Fatal("Configuration should produce object") + } + defer obj.Close() + + v := obj.Get("foo") + if v == nil { + t.Fatal("Key \"foo\" should exist") + } + defer v.Close() + if v.ToString() != value { + t.Fatalf("bad: \"%s\", expected: \"%s\"", v.ToString(), value) + } +} + +func TestFileVariables(t *testing.T) { + tf, err := ioutil.TempFile("", "libucl") + if err != nil { + t.Fatalf("err: %s", err) + } + tf.Write([]byte("file = $FILENAME; dir = $CURDIR")) + tf.Close() + + p := NewParser(0) + defer p.Close() + + if err := p.AddFileAndSetVariables(tf.Name(), false); err != nil { + t.Fatalf("err: %s", err) + } + + obj := p.Object() + if obj == nil { + t.Fatal("obj should not be nil") + } + defer obj.Close() + + filename := obj.Get("file") + if filename == nil { + t.Fatal("key \"file\" should exist") + } + defer filename.Close() + if filename.ToString() != tf.Name() { + t.Errorf("bad: %s, expected %s", filename.ToString(), tf.Name()) + } + + dir := obj.Get("dir") + if dir == nil { + t.Fatal("key \"dir\" should exist") + } + defer dir.Close() + if dir.ToString() != path.Dir(tf.Name()) { + t.Errorf("bad: %s, expected %s", dir.ToString(), path.Dir(tf.Name())) + } +} From e2cc02dfc7d4249977d80031061f39c2b3030926 Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Sun, 14 Feb 2016 16:47:37 -0500 Subject: [PATCH 18/24] Various fixes Change to .travis-ucl.sh is to build support for signatures into the test machine's copy of libucl. Change to decoder_test.go is to show what is expected in random failing test (see issue #6). Change to object.go is to remove memory leak, and make sure created C strings are freed. --- .travis-ucl.sh | 2 +- decoder_test.go | 2 +- object.go | 8 ++++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.travis-ucl.sh b/.travis-ucl.sh index 3d19bad..48e2101 100755 --- a/.travis-ucl.sh +++ b/.travis-ucl.sh @@ -5,5 +5,5 @@ mkdir /tmp/libucl cd /tmp/libucl wget https://github.com/vstakhov/libucl/archive/$UCL_VERSION.tar.gz tar xzf $UCL_VERSION.tar.gz -cd libucl-$UCL_VERSION && ./autogen.sh && ./configure --prefix=/usr --enable-urls && make && sudo make install +cd libucl-$UCL_VERSION && ./autogen.sh && ./configure --prefix=/usr --enable-urls --enable-signatures && make && sudo make install rm -fr /tmp/libucl diff --git a/decoder_test.go b/decoder_test.go index dc3a6e1..9def000 100644 --- a/decoder_test.go +++ b/decoder_test.go @@ -399,7 +399,7 @@ func TestObjectDecode_structKeys(t *testing.T) { Keys: []string{"Foo", "Bar"}, } if !reflect.DeepEqual(expected, result) { - t.Fatalf("bad: %#v", result) + t.Fatalf("bad: %#v, expected: %#v", result, expected) } } diff --git a/object.go b/object.go index cfcb54f..dee0a95 100644 --- a/object.go +++ b/object.go @@ -235,14 +235,18 @@ const ( // NewObject creates a new UCL Object from a string, JSON escaping it in the process func NewObject(data string) *Object { - obj := C.ucl_object_fromlstring(C.CString(data), C.size_t(len(data))) + cData := C.CString(data) + defer C.free(unsafe.Pointer(cData)) + obj := C.ucl_object_fromlstring(cData, C.size_t(len(data))) return &Object{object: obj} } // NewFormattedObject creates a new UCL Object from a string, according to the instructions // given in the flags func NewFormattedObject(data string, flags StringFlag) *Object { - obj := C.ucl_object_fromstring_common(C.CString(data), C.size_t(len(data)), uint32(flags)) + cData := C.CString(data) + defer C.free(unsafe.Pointer(cData)) + obj := C.ucl_object_fromstring_common(cData, C.size_t(len(data)), uint32(flags)) return &Object{object: obj} } From 28c472fc5e8837b75d5dd42565911647da13e03f Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Sun, 14 Feb 2016 17:00:44 -0500 Subject: [PATCH 19/24] Added simple sorter to fix random ordering issue in test TestObjectDecode_structKeys The sorter lists strings in decreasing alphabetical ordering (z->a) so Foo comes before Bar. This should fix the random ordering issue, fixing issue #6. --- decoder_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/decoder_test.go b/decoder_test.go index 9def000..3309fcd 100644 --- a/decoder_test.go +++ b/decoder_test.go @@ -2,9 +2,16 @@ package libucl import ( "reflect" + "sort" "testing" ) +type foobarSorter []string + +func (s foobarSorter) Len() int { return len(s) } +func (s foobarSorter) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +func (s foobarSorter) Less(i, j int) bool { return s[i] > s[j] } + func TestObjectDecode_basic(t *testing.T) { type Basic struct { Bool bool @@ -392,7 +399,7 @@ func TestObjectDecode_structKeys(t *testing.T) { if err := obj.Decode(&result); err != nil { t.Fatalf("err: %s", err) } - + sort.Sort(foobarSorter(result.Keys)) expected := Struct{ Foo: []string{"foo", "bar", "12"}, Bar: "baz", From 3253cb9a6cdafaedf671669dfe0d6262995669ad Mon Sep 17 00:00:00 2001 From: Michael Williams Date: Thu, 18 Feb 2016 16:19:30 -0500 Subject: [PATCH 20/24] Add ability to use native go file implementation to with libucl In addition, added libucl's error messages to some methods. --- go-libucl.h | 4 ++++ parser.go | 20 +++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/go-libucl.h b/go-libucl.h index 9f115c8..bc7e52b 100644 --- a/go-libucl.h +++ b/go-libucl.h @@ -9,6 +9,10 @@ static inline char *_go_uchar_to_char(const unsigned char *c) { return (char *)c; } +static inline unsigned char *_go_char_to_uchar(const char *c) { + return (unsigned char *)c; +} + //------------------------------------------------------------------- // Helpers: Macros //------------------------------------------------------------------- diff --git a/parser.go b/parser.go index a9d59b0..7ea84c3 100644 --- a/parser.go +++ b/parser.go @@ -2,6 +2,7 @@ package libucl import ( "errors" + "os" "sync" "unsafe" ) @@ -161,9 +162,10 @@ func go_macro_call(id C.int, data *C.char, n C.int) C.bool { func (p *Parser) SetFileVariables(filepath string, expand bool) error { cpath := C.CString(filepath) defer C.free(unsafe.Pointer(cpath)) - ok := C.ucl_parser_set_filevars(p.parser, cpath, C.bool(expand)) - if !ok { - return errors.New("Unable to set file variables") + result := C.ucl_parser_set_filevars(p.parser, cpath, C.bool(expand)) + if !result { + errstr := C.ucl_parser_get_error(p.parser) + return errors.New(C.GoString(errstr)) } return nil } @@ -189,3 +191,15 @@ func (p *Parser) AddFileAndSetVariables(path string, expand bool) error { err = p.SetFileVariables(path, expand) return err } + +// AddOpenFile reads in the configuration from a file already opened using os.Open +// or a related function. +func (p *Parser) AddOpenFile(f *os.File) error { + fd := f.Fd() + result := C.ucl_parser_add_fd(p.parser, C.int(fd)) + if !result { + errstr := C.ucl_parser_get_error(p.parser) + return errors.New(C.GoString(errstr)) + } + return nil +} From ab819872e7241dd4db3d9607231529c381b4493b Mon Sep 17 00:00:00 2001 From: Christopher Hall Date: Thu, 27 Apr 2017 16:12:43 +0800 Subject: [PATCH 21/24] convert to pkg-config Signed-off-by: Christopher Hall --- libucl.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/libucl.go b/libucl.go index 6afa547..fe434ab 100644 --- a/libucl.go +++ b/libucl.go @@ -1,9 +1,6 @@ package libucl -// #cgo CFLAGS: -Ivendor/libucl/include -Wno-int-to-void-pointer-cast -// #cgo LDFLAGS: -Lvendor/libucl -lucl -// -// #cgo freebsd CFLAGS: -I/usr/local/include -Wno-int-to-void-pointer-cast -// #cgo freebsd LDFLAGS: -L/usr/local/lib -lucl +// #cgo CFLAGS: -Wno-int-to-void-pointer-cast +// #cgo pkg-config: libucl // import "C" From 4489d4546aee0a3163de59ed8255a6810ffe1926 Mon Sep 17 00:00:00 2001 From: Christopher Hall Date: Fri, 11 Aug 2017 11:15:28 +0800 Subject: [PATCH 22/24] fix the macro call * original code always called the first macro as call back parameters list was incorrect * add support for macro arguments e.g. .macro(param=value) "body-text" the parameters are passed as ucl object so complex macros are possible Signed-off-by: Christopher Hall --- Makefile | 35 ----------------------------------- decoder.go | 34 ++++++++++++++++++++-------------- go-libucl.h | 6 +++--- libucl.go | 1 - parser.go | 17 ++++++++++++++--- parser_test.go | 18 ++++++++++++++---- 6 files changed, 51 insertions(+), 60 deletions(-) delete mode 100644 Makefile diff --git a/Makefile b/Makefile deleted file mode 100644 index ccfb08c..0000000 --- a/Makefile +++ /dev/null @@ -1,35 +0,0 @@ -LIBUCL_NAME=libucl.a - -# If we're on Windows, we need to change some variables so things compile -# properly. -ifeq ($(OS), Windows_NT) - LIBUCL_NAME=libucl.dll -endif - -export CGO_CFLAGS CGO_LDFLAGS PATH - -all: libucl - go test - -libucl: vendor/libucl/$(LIBUCL_NAME) - -vendor/libucl/libucl.a: vendor/libucl - cd vendor/libucl && \ - cmake . && \ - make - -vendor/libucl/libucl.dll: vendor/libucl - cd vendor/libucl && \ - $(MAKE) -f Makefile.w32 && \ - cp .obj/libucl.dll . && \ - cp libucl.dll $(CURDIR) - -vendor/libucl: - rm -rf vendor/libucl - mkdir -p vendor/libucl - git clone https://github.com/vstakhov/libucl.git vendor/libucl - -clean: - rm -rf vendor - -.PHONY: all clean libucl test diff --git a/decoder.go b/decoder.go index afa7102..af9164d 100644 --- a/decoder.go +++ b/decoder.go @@ -134,12 +134,13 @@ func decodeIntoInterface(name string, o *Object, result reflect.Value) error { for o := outer.Next(); o != nil; o = outer.Next() { m := make(map[string]interface{}) inner := o.Iterate(true) + inner_loop: for o2 := inner.Next(); o2 != nil; o2 = inner.Next() { var raw interface{} err = decode(name, o2, reflect.Indirect(reflect.ValueOf(&raw))) o2.Close() if err != nil { - break + break inner_loop } m[o2.Key()] = raw @@ -159,7 +160,7 @@ func decodeIntoInterface(name string, o *Object, result reflect.Value) error { set = reflect.Indirect(reflect.New(reflect.TypeOf(""))) default: return fmt.Errorf( - "%s: unsupported type to interface: %s", name, o.Type()) + "%s: unsupported type to interface: %v", name, o.Type()) } if redecode { @@ -289,7 +290,7 @@ func decodeIntoString(name string, o *Object, result reflect.Value) error { case ObjectTypeInt: result.SetString(strconv.FormatInt(o.ToInt(), 10)) default: - return fmt.Errorf("%s: unsupported type to string: %s", name, objType) + return fmt.Errorf("%s: unsupported type to string: %v", name, objType) } return nil @@ -310,6 +311,7 @@ func decodeIntoStruct(name string, o *Object, result reflect.Value) error { structs = structs[1:] structType := structVal.Type() + struct_loop: for i := 0; i < structType.NumField(); i++ { fieldType := structType.Field(i) @@ -325,16 +327,17 @@ func decodeIntoStruct(name string, o *Object, result reflect.Value) error { // if specified in the tag. squash := false tagParts := strings.Split(fieldType.Tag.Get(tagName), ",") + tag_loop: for _, tag := range tagParts[1:] { if tag == "squash" { squash = true - break + break tag_loop } } if squash { structs = append(structs, result.FieldByName(fieldType.Name)) - continue + continue struct_loop } } @@ -347,6 +350,7 @@ func decodeIntoStruct(name string, o *Object, result reflect.Value) error { decodedFields := make([]string, 0, len(fields)) var decodedFieldsVal []reflect.Value var unusedKeysVal []reflect.Value +field_loop: for fieldType, field := range fields { if !field.IsValid() { // This should never happen @@ -356,7 +360,7 @@ func decodeIntoStruct(name string, o *Object, result reflect.Value) error { // If we can't set the field, then it is unexported or something, // and we just continue onwards. if !field.CanSet() { - continue + continue field_loop } fieldName := fieldType.Name @@ -367,20 +371,20 @@ func decodeIntoStruct(name string, o *Object, result reflect.Value) error { switch tagParts[1] { case "decodedFields": decodedFieldsVal = append(decodedFieldsVal, field) - continue + continue field_loop case "key": field.SetString(o.Key()) - continue + continue field_loop case "object": // Increase the ref count o.Ref() // Sete the object field.Set(reflect.ValueOf(o)) - continue + continue field_loop case "unusedKeys": unusedKeysVal = append(unusedKeysVal, field) - continue + continue field_loop } } @@ -393,9 +397,10 @@ func decodeIntoStruct(name string, o *Object, result reflect.Value) error { // Do a slower search by iterating over each key and // doing case-insensitive search. iter := o.Iterate(true) + element_loop: for elem = iter.Next(); elem != nil; elem = iter.Next() { if strings.EqualFold(elem.Key(), fieldName) { - break + break element_loop } elem.Close() @@ -404,7 +409,7 @@ func decodeIntoStruct(name string, o *Object, result reflect.Value) error { if elem == nil { // No key matching this field. - continue + continue field_loop } } @@ -422,16 +427,17 @@ func decodeIntoStruct(name string, o *Object, result reflect.Value) error { err = decode(fieldName, elem, field) } else { iter := elem.Iterate(false) + iteration_loop: for { obj := iter.Next() if obj == nil { - break + break iteration_loop } err = decode(fieldName, obj, field) obj.Close() if err != nil { - break + break iteration_loop } } iter.Close() diff --git a/go-libucl.h b/go-libucl.h index bc7e52b..ff82f84 100644 --- a/go-libucl.h +++ b/go-libucl.h @@ -19,11 +19,11 @@ static inline unsigned char *_go_char_to_uchar(const char *c) { // This is declared in parser.go and invokes the Go function callback for // a specific macro (specified by the ID). -extern bool go_macro_call(int, char *data, int); +extern bool go_macro_call(int idx, ucl_object_t *arguments, char *data, int length); // Indirection that actually calls the Go macro handler. -static inline bool _go_macro_handler(const unsigned char *data, size_t len, void* ud) { - return go_macro_call((int)ud, (char*)data, (int)len); +static inline bool _go_macro_handler(const unsigned char *data, size_t len, const ucl_object_t *arguments, void* ud) { + return go_macro_call((int)ud, (ucl_object_t *)arguments, (char*)data, (int)len); } // Returns the ucl_macro_handler that we have, since we can't get this diff --git a/libucl.go b/libucl.go index d127697..e4e3d66 100644 --- a/libucl.go +++ b/libucl.go @@ -4,5 +4,4 @@ package libucl // #cgo CFLAGS: -Wno-int-to-void-pointer-cast // #cgo pkg-config: libucl -// import "C" diff --git a/parser.go b/parser.go index 7ea84c3..0dcc297 100644 --- a/parser.go +++ b/parser.go @@ -11,7 +11,14 @@ import ( import "C" // MacroFunc is the callback type for macros. -type MacroFunc func(string) +// return true os string is valid +// a macro call looks like: +// +// .macro(UCL-OBJECT) "body-text" +// .macro(key=value) "body-text" +// .macro(params={key1=value1;key2=value2}) "body" +// +type MacroFunc func(args Object, body string) bool // ParserFlag are flags that can be used to initialize a parser. type ParserFlag int @@ -137,18 +144,22 @@ func (p *Parser) RegisterMacro(name string, f MacroFunc) { } //export go_macro_call -func go_macro_call(id C.int, data *C.char, n C.int) C.bool { +func go_macro_call(id C.int, arguments *C.ucl_object_t, data *C.char, n C.int) C.bool { macrosLock.Lock() f := macros[int(id)] macrosLock.Unlock() + args := Object{ + object: arguments, + } + // Macro not found, return error if f == nil { return false } // Macro found, call it! - f(C.GoStringN(data, n)) + f(args, C.GoStringN(data, n)) return true } diff --git a/parser_test.go b/parser_test.go index 4cf3c5b..ad18edc 100644 --- a/parser_test.go +++ b/parser_test.go @@ -98,11 +98,18 @@ func TestParserAddFile(t *testing.T) { func TestParserRegisterMacro(t *testing.T) { value := "" - macro := func(data string) { - value = data + parameter := "" + macro := func(args Object, body string) bool { + thing := args.Get("thing") + if nil == thing { + return false + } + parameter = thing.ToString() + value = body + return true } - config := `.foo "bar";` + config := `.foo(thing=something) "bar";` p := NewParser(0) defer p.Close() @@ -114,7 +121,10 @@ func TestParserRegisterMacro(t *testing.T) { } if value != "bar" { - t.Fatalf("bad: %#v", value) + t.Fatalf("body bad: %#v", value) + } + if parameter != "something" { + t.Fatalf("parameter bad: %#v", parameter) } } From 9b16c124aff9f08279737cdd6528f3352ff1becd Mon Sep 17 00:00:00 2001 From: Christopher Hall Date: Fri, 11 Aug 2017 11:21:08 +0800 Subject: [PATCH 23/24] change URLs, add note about macro fixes Signed-off-by: Christopher Hall --- README.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 994b4a1..6cf4301 100644 --- a/README.md +++ b/README.md @@ -1,13 +1,12 @@ # Libucl Library for Go -[![Build Status](https://travis-ci.org/draringi/go-libucl.svg?branch=master)](https://travis-ci.org/draringi/go-libucl) -[![GoDoc](https://godoc.org/github.com/draringi/go-libucl?status.svg)](https://godoc.org/github.com/draringi/go-libucl) +[![Build Status](https://travis-ci.org/bitmark-inc/go-libucl.svg?branch=master)](https://travis-ci.org/bitmark-inc/go-libucl) +[![GoDoc](https://godoc.org/github.com/bitmark-inc/go-libucl?status.svg)](https://godoc.org/github.com/bitmark-inc/go-libucl) -This version of go-libucl is forked from the [mitchellh version](https://github.com/mitchellh/go-libucl), +This version of go-libucl is forked from the[mitchellh version](https://github.com/mitchellh/go-libucl), +and [draring version](http://godoc.org/github.com/draringi/go-libucl) with the goal of having a version with a focus on using the OS's copy of libucl, in a portable manner, as well as improve the Documentation quality. As such, it uses pkg-config to determine the location of libucl. -In addition, patches from [hwx](https://github.com/bitmark-inc/go-libucl) have been -pulled in with 64-bit integer support. go-libucl is a [libucl](https://github.com/vstakhov/libucl) library for [Go](http://golang.org). Rather than re-implement libucl in Go, this library @@ -18,6 +17,10 @@ the central source of knowledge. This project has been tested on Linux and FreeB is not guaranteed. Additionally, it is not feature complete yet, though it is certainly usable for real purposes (we do!). +## Notes +* macro calling convention changed +* macro callback now gets paramters object in addion to body text + ## Prerequisites * libucl (This is a wrapper for this library) * pkg-config (cgo uses this for locate where libucl is) @@ -25,7 +28,7 @@ it is certainly usable for real purposes (we do!). ## Installation ``` -$ go get github.com/draringi/go-libucl +$ go get github.com/bitmark-inc/go-libucl ``` -Documentation is available on GoDoc: http://godoc.org/github.com/draringi/go-libucl +Documentation is available on GoDoc: http://godoc.org/github.com/bitmark-inc/go-libucl From 700172f16e59fb0ae72626b935ed3f812f2b9330 Mon Sep 17 00:00:00 2001 From: Christopher Hall Date: Fri, 8 Mar 2019 13:33:51 +0800 Subject: [PATCH 24/24] make into go module Signed-off-by: Christopher Hall --- go.mod | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 go.mod diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..3c7f987 --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/bitmark-inc/go-libucl + +go 1.12