diff --git a/.gitmodules b/.gitmodules index cf5011a66..3469ad541 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "test/data"] path = test/data url = https://github.com/mapnik/test-data.git +[submodule "mason"] + path = mason + url = https://github.com/mapbox/mason.git diff --git a/.travis.yml b/.travis.yml index 6d3648a6c..84995b03b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,22 +1,24 @@ language: generic +git: + submodules: false + matrix: include: - os: linux - sudo: false compiler: clang - # note: only using ccache for CC is intentional here to - # workaround an odd bug in distutils that manifests when only `ccache` is used to link - # because distutils also has a bug whereby CC is used to compile instead of CXX, this works :) - env: JOBS=8 CXX="clang++-3.9 -Qunused-arguments" CC="ccache clang-3.9 -Qunused-arguments" + env: >- + CC="clang-3.9" + CXX="clang++-3.9" addons: apt: - sources: [ 'ubuntu-toolchain-r-test'] - packages: [ 'libstdc++-5-dev', 'gdb', 'apport'] + sources: [ ubuntu-toolchain-r-test ] + packages: [ clang-3.9, libstdc++-5-dev ] - os: osx - osx_image: xcode8.2 compiler: clang - env: JOBS=4 + env: >- + CC="clang" + CXX="clang++" cache: directories: @@ -39,43 +41,80 @@ before_install: if [[ "${TRAVIS_OS_NAME}" == "osx" ]]; then rvm get head || true fi - - source scripts/setup_mason.sh - - export PYTHONUSERBASE=$(pwd)/mason_packages/.link - - export PYTHONPATH=$(pwd)/mason_packages/.link/lib/python2.7/site-packages - - export PATH=$(pwd)/mason_packages/.link/bin:${PYTHONUSERBASE}/bin:${PATH} + - declare -p HOME PATH "${!CCACHE@}" "${!TRAVIS@}" # print variables by prefix + + # An ancient bug in distutils (https://bugs.python.org/issue8027) + # prevents having multi-word CXX="ccache c++". Luckily distutils + # uses CC to compile C++ sources, and only uses CXX for linking, + # for which ccache is useless. + - export CC="ccache $CC" + - export MASON_BUILD=true + - export MASON_ROOT=$PWD/mason_packages + - export PYTHONUSERBASE=$MASON_ROOT/.link + - export PATH=$MASON_ROOT/.link/bin:$PATH - export COMMIT_MESSAGE=$(git show -s --format=%B $TRAVIS_COMMIT | tr -d '\n') - - | - if [[ $(uname -s) == 'Linux' ]]; then - export LDSHARED=$(python -c "import os;from distutils import sysconfig;print sysconfig.get_config_var('LDSHARED').replace('cc ','clang++-3.9 ')"); - mason install clang++ 3.9.1 - export PATH=$(mason prefix clang++ 3.9.1)/bin:${PATH} - which clang++ - else - sudo easy_install pip; - export LDSHARED=$(python -c "import os;from distutils import sysconfig;print sysconfig.get_config_var('LDSHARED').replace('cc ','clang++ ')"); - fi + +install: + - source ./bootstrap.sh + - install postgres 9.6.5 + - install postgis 2.4.1 + - install ccache 3.7.2 + - ccache --version + - test "$TRAVIS_OS_NAME" != osx || sudo easy_install pip + - pip install --upgrade --user pip - pip install --upgrade --user nose - pip install --upgrade --user wheel - pip install --upgrade --user twine - pip install --upgrade --user setuptools - pip install --upgrade --user PyPDF2 - - python --version - -install: - - mkdir -p ${PYTHONPATH} - - python setup.py install --prefix ${PYTHONUSERBASE} before_script: - # start postgres/postgis - - source mason-config.env - - ./mason_packages/.link/bin/postgres -k ${PGHOST} > postgres.log & + - touch SCRIPT_START + - python --version + - python -m site + - mkdir -p "$(python -m site --user-site)" + + # distutils uses LDSHARED for linking, which it somehow constructs from CC + # with additional flags, but the resulting command is ultimately wrong. + # + # LDSHARED[osx]='clang -bundle -undefined dynamic_lookup' + # LDSHARED[linux]='x86_64-linux-gnu-gcc -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions -Wl,-Bsymbolic-functions -Wl,-z,relro -fno-strict-aliasing -DNDEBUG -g -fwrapv -O2 -Wall -Wstrict-prototypes -Wdate-time -D_FORTIFY_SOURCE=2 -g -fstack-protector-strong -Wformat -Werror=format-security' + # + # On Linux there are some misplaced options that don't apply to linker. + # OSX command looks fine, but for some reason leads to the execution of + # 'clang++ clang -bundle -undefined dynamic_lookup ...' and fails. + # + # LDCXXSHARED[osx]='clang++ -bundle -undefined dynamic_lookup' + # LDCXXSHARED[linux]='c++ -pthread -shared -Wl,-O1 -Wl,-Bsymbolic-functions' + # + # So let's just replace the value of LDSHARED with LDCXXSHARED, which + # apparently works on both OSX and Linux. Nevermind compiler 'c++', + # distutils substitutes that with CXX. + # + - LDSHARED=$(python -c 'from distutils import sysconfig as dusc; print(dusc.get_config_var("LDSHARED"))') + - LDCXXSHARED=$(python -c 'from distutils import sysconfig as dusc; print(dusc.get_config_var("LDCXXSHARED"))') + - printf 'default LDSHARED=%q\n' "$LDSHARED" + - printf 'default LDCXXSHARED=%q\n' "$LDCXXSHARED" + - export LDSHARED=${LDCXXSHARED} script: + - python setup.py install --user --prefix= + # why empty prefix is needed is explained here: + # https://stackoverflow.com/questions/4495120/combine-user-with-prefix-error-with-setup-py-install + - (( !TRAVIS_TEST_RESULT )) && source ./scripts/setup_postgres.sh + - (( !TRAVIS_TEST_RESULT )) && git submodule update --init --depth=20 test/ - python test/run_tests.py - python test/visual.py -q - # stop postgres - - ./mason_packages/.link/bin/pg_ctl -w stop + +before_cache: + - ccache --version + - ccache --show-config + - ccache --show-stats + - ccache --zero-stats + - find $HOME/.ccache -mindepth 3 -newer SCRIPT_START | head | file --files-from - + +after_success: - | if [[ ${COMMIT_MESSAGE} =~ "[publish]" ]]; then python setup.py bdist_wheel @@ -84,7 +123,7 @@ script: rename 's/linux_x86_64/any/;' $PRE_DISTS fi export DISTS='dist/*' - $(pwd)/mason_packages/.link/bin/twine upload -u $PYPI_USER -p $PYPI_PASSWORD $DISTS + "$MASON_ROOT/.link/bin/twine" upload -u $PYPI_USER -p $PYPI_PASSWORD $DISTS fi diff --git a/bootstrap.sh b/bootstrap.sh index fcd84d047..4029da386 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -1,20 +1,19 @@ #!/usr/bin/env bash -set -eu -set -o pipefail - -function install() { - MASON_PLATFORM_ID=$(mason env MASON_PLATFORM_ID) - if [[ ! -d ./mason_packages/${MASON_PLATFORM_ID}/${1}/ ]]; then - mason install $1 $2 - mason link $1 $2 - fi -} - +BOOST_VERSION="1.66.0" ICU_VERSION="57.1" +MAPNIK_VERSION="a0ea7db1a" + +function install_mason_deps() ( + # subshell + set -eu + install_mapnik_deps + install mapnik ${MAPNIK_VERSION} +) -function install_mason_deps() { - install mapnik df0bbe4 +function install_mapnik_deps() ( + # subshell + set -eu install jpeg_turbo 1.5.1 install libpng 1.6.28 install libtiff 4.0.7 @@ -27,61 +26,44 @@ function install_mason_deps() { install cairo 1.14.8 install webp 0.6.0 install libgdal 2.1.3 - install boost 1.63.0 - install boost_libsystem 1.63.0 - install boost_libfilesystem 1.63.0 - install boost_libprogram_options 1.63.0 - install boost_libregex_icu57 1.63.0 + install boost ${BOOST_VERSION} + install boost_libsystem ${BOOST_VERSION} + install boost_libfilesystem ${BOOST_VERSION} + install boost_libprogram_options ${BOOST_VERSION} + install boost_libregex_icu${ICU_VERSION%%.*} ${BOOST_VERSION} install freetype 2.7.1 install harfbuzz 1.4.2-ft # deps needed by python-mapnik (not mapnik core) - install boost_libthread 1.63.0 - install boost_libpython 1.63.0 - install postgis 2.3.2-1 -} + install boost_libthread ${BOOST_VERSION} + install boost_libpython ${BOOST_VERSION} +) function setup_runtime_settings() { - local MASON_LINKED_ABS=$(pwd)/mason_packages/.link - echo "export PROJ_LIB=${MASON_LINKED_ABS}/share/proj" > mason-config.env - echo "export ICU_DATA=${MASON_LINKED_ABS}/share/icu/${ICU_VERSION}" >> mason-config.env - echo "export GDAL_DATA=${MASON_LINKED_ABS}/share/gdal" >> mason-config.env - echo "export PATH=$(pwd)/mason_packages/.link/bin:${PATH}" >> mason-config.env - echo "export PGTEMP_DIR=$(pwd)/local-tmp" >> mason-config.env - echo "export PGDATA=$(pwd)/local-postgres" >> mason-config.env - echo "export PGHOST=$(pwd)/local-unix-socket" >> mason-config.env - echo "export PGPORT=1111" >> mason-config.env - - source mason-config.env - rm -rf ${PGHOST} - mkdir -p ${PGHOST} - rm -rf ${PGDATA} - mkdir -p ${PGDATA} - rm -rf ${PGTEMP_DIR} - mkdir -p ${PGTEMP_DIR} - ./mason_packages/.link/bin/initdb - sleep 2 - ./mason_packages/.link/bin/postgres -k ${PGHOST} > postgres.log & - sleep 2 - ./mason_packages/.link/bin/createdb template_postgis -T postgres - ./mason_packages/.link/bin/psql template_postgis -c "CREATE TABLESPACE temp_disk LOCATION '${PGTEMP_DIR}';" - ./mason_packages/.link/bin/psql template_postgis -c "SET temp_tablespaces TO 'temp_disk';" - ./mason_packages/.link/bin/psql template_postgis -c "CREATE PROCEDURAL LANGUAGE 'plpythonu' HANDLER plpython_call_handler;" - ./mason_packages/.link/bin/psql template_postgis -c "CREATE EXTENSION postgis;" - ./mason_packages/.link/bin/psql template_postgis -c "SELECT PostGIS_Full_Version();" - ./mason_packages/.link/bin/pg_ctl -w stop + # PWD and ICU_VERSION are expanded here, but MASON_ROOT and PATH + # expansion must be deferred to the generated script, so that it + # can be used later + printf 'export %s=${MASON_ROOT:-%q}\n' \ + MASON_ROOT "$PWD/mason_packages" \ + > ./mason-config.env + printf 'export %s=$%s\n' \ + MASON_BIN "{MASON_ROOT}/.link/bin" \ + PROJ_LIB "{MASON_ROOT}/.link/share/proj" \ + ICU_DATA "{MASON_ROOT}/.link/share/icu/${ICU_VERSION}" \ + GDAL_DATA "{MASON_ROOT}/.link/share/gdal" \ + PATH '{MASON_BIN}:${PATH}' \ + PATH '{PATH//":${MASON_BIN}:"/:} # remove duplicates' \ + >> ./mason-config.env + source ./mason-config.env } function main() { source scripts/setup_mason.sh - setup_mason - install_mason_deps - setup_runtime_settings + [ $? = 0 ] && install_mason_deps + [ $? = 0 ] && setup_runtime_settings + [ $? = 0 ] || return echo "Ready, now run:" echo "" echo " make test" } main - -set +eu -set +o pipefail \ No newline at end of file diff --git a/mason b/mason new file mode 160000 index 000000000..f00098c63 --- /dev/null +++ b/mason @@ -0,0 +1 @@ +Subproject commit f00098c635fc6633b3da96befd8f614d217eb43a diff --git a/scripts/setup_mason.sh b/scripts/setup_mason.sh index 97fb4e1e0..5faff5735 100755 --- a/scripts/setup_mason.sh +++ b/scripts/setup_mason.sh @@ -1,22 +1,51 @@ #!/bin/bash -set -eu -set -o pipefail - # we pin the mason version to avoid changes in mason breaking builds -MASON_VERSION="3870d9c" +MASON_ARCHIVE_VERSION="f00098c63" +MASON_ARCHIVE_BASE="https://github.com/mapbox/mason/archive" + +function install() { + local package_prefix= + package_prefix=$(mason prefix $1 $2) || return + if [ ! -d "$package_prefix/" ]; then + mason install $1 $2 || return + fi + mason link $1 $2 +} + +function mason_packages() { + printf '%s\n' "${MASON_ROOT:-"$PWD/mason_packages"}" +} function setup_mason() { - mkdir -p ./mason - curl -sSfL https://github.com/mapbox/mason/archive/${MASON_VERSION}.tar.gz | tar --gunzip --extract --strip-components=1 --exclude="*md" --exclude="test*" --directory=./mason - export MASON_HOME=$(pwd)/mason_packages/.link - export PATH=$(pwd)/mason:${PATH} + local cdup= + if cdup=$(git rev-parse --show-cdup 2>/dev/null); then + # we are inside *some* git repository (not necessarily python-mapnik) + if git submodule status "${cdup}mason" >/dev/null 2>&1; then + # there is submodule "mason" (assume we are in python-mapnik) + # update the submodule, bail out on failure + git submodule update --init "${cdup}mason" || + return + else + # there is no submodule named "mason" + # proceed as if we were outside git repository + cdup= + fi + fi + if [ ! -d "${cdup}mason/" ]; then + # the directory doesn't exist, and we are either outside any git + # repository, or in a repository with no submodule named "mason" + mkdir -p "${cdup}mason/" + # download and unpack mason archive + curl -sSfL "${MASON_ARCHIVE_BASE}/${MASON_ARCHIVE_VERSION}.tar.gz" | + tar --extract --gunzip --directory="${cdup}mason/" \ + --strip-components=1 --exclude="test" || + return + fi + export PATH=$(cd "${cdup}mason" && pwd):${PATH} export CXX=${CXX:-clang++} export CC=${CC:-clang} } setup_mason - -set +eu -set +o pipefail \ No newline at end of file diff --git a/scripts/setup_postgres.sh b/scripts/setup_postgres.sh new file mode 100644 index 000000000..4552b3068 --- /dev/null +++ b/scripts/setup_postgres.sh @@ -0,0 +1,23 @@ +#! /bin/bash + +function setup_postgres_settings() { + export PGDATA=$PWD/local-postgres + export PGHOST=$PWD/local-unix-socket + export PGPORT=1111 +} + +function setup_postgres() { + rm -rf -- "$PGDATA" "$PGHOST" + mkdir -p -- "$PGDATA" "$PGHOST" + + initdb -N + postgres -k "$PGHOST" -c fsync=off > postgres.log & + sleep 2 + createdb template_postgis -T postgres + psql template_postgis -e -c "CREATE PROCEDURAL LANGUAGE 'plpythonu' HANDLER plpython_call_handler;" + psql template_postgis -e -c "CREATE EXTENSION postgis;" + psql template_postgis -e -c "SELECT PostGIS_Full_Version();" +} + +setup_postgres_settings +setup_postgres diff --git a/setup.py b/setup.py index 38a4e5c79..c7c0e7058 100755 --- a/setup.py +++ b/setup.py @@ -54,7 +54,9 @@ def get_boost_library_names(): found = [] missing = [] for _id in wanted: - name = os.environ.get("%s_LIB" % _id.upper(), find_boost_library(_id)) + name = os.environ.get("%s_LIB" % _id.upper()) + if name is None: + name = _id if mason_build else find_boost_library(_id) if name: found.append(name) else: @@ -79,7 +81,9 @@ def finalize_options(self): pass def run(self): - print("\n".join(get_boost_library_names())) + print("Found boost libraries:") + for lib in get_boost_library_names(): + print("\t" + lib) cflags = sysconfig.get_config_var('CFLAGS') @@ -101,14 +105,16 @@ def run(self): os.environ['ARCHFLAGS'] = '' if os.environ.get("MASON_BUILD", "false") == "true": - # run bootstrap.sh to get mason builds - subprocess.call(['./bootstrap.sh']) - mapnik_config = 'mason_packages/.link/bin/mapnik-config' + mason_packages = os.environ.get('MASON_ROOT', 'mason_packages') + mapnik_config = mason_packages + '/.link/bin/mapnik-config' mason_build = True else: + mason_packages = None mapnik_config = 'mapnik-config' mason_build = False +mapnik_config = os.environ.get('MAPNIK_CONFIG', mapnik_config) + linkflags = [] lib_path = os.path.join(check_output([mapnik_config, '--prefix']),'lib') @@ -193,7 +199,7 @@ def run(self): if not os.path.exists(share_path): os.makedirs(share_path) - icu_path = 'mason_packages/.link/share/icu/*/*.dat' + icu_path = mason_packages + '/.link/share/icu/*/*.dat' icu_files = glob.glob(icu_path) if len(icu_files) != 1: raise Exception("Failed to find icu dat file at "+ icu_path) @@ -201,29 +207,26 @@ def run(self): shutil.copyfile(f, os.path.join( 'mapnik', share_dir, 'icu', os.path.basename(f))) - gdal_path = 'mason_packages/.link/share/gdal/' - gdal_files = os.listdir(gdal_path) - gdal_files = [os.path.join(gdal_path, f) for f in gdal_files] - for f in gdal_files: + gdal_path = mason_packages + '/.link/share/gdal/' + for f in os.listdir(gdal_path): try: - shutil.copyfile(f, os.path.join( - 'mapnik', share_dir, 'gdal', os.path.basename(f))) + src = os.path.join(gdal_path, f) + dst = os.path.join('mapnik', share_dir, 'gdal', f) + shutil.copyfile(src, dst) except shutil.Error: pass - proj_path = 'mason_packages/.link/share/proj/' - proj_files = os.listdir(proj_path) - proj_files = [os.path.join(proj_path, f) for f in proj_files] - for f in proj_files: + proj_path = mason_packages + '/.link/share/proj/' + for f in os.listdir(proj_path): try: - shutil.copyfile(f, os.path.join( - 'mapnik', share_dir, 'proj', os.path.basename(f))) + src = os.path.join(proj_path, f) + dst = os.path.join('mapnik', share_dir, 'proj', f) + shutil.copyfile(src, dst) except shutil.Error: pass extra_comp_args = check_output([mapnik_config, '--cflags']).split(' ') - -extra_comp_args = list(filter(lambda arg: arg != "-fvisibility=hidden", extra_comp_args)) +extra_comp_args = [a for a in extra_comp_args if a != "-fvisibility=hidden"] if os.environ.get("PYCAIRO", "false") == "true": try: diff --git a/test/data b/test/data index 99da07d5e..c3982b766 160000 --- a/test/data +++ b/test/data @@ -1 +1 @@ -Subproject commit 99da07d5e76ccf5978ef0a380bf5f631f9088584 +Subproject commit c3982b76602bcbfea6403793ac2f3fab44552270 diff --git a/test/data-visual b/test/data-visual index e040c3d9c..a5f4ef0ea 160000 --- a/test/data-visual +++ b/test/data-visual @@ -1 +1 @@ -Subproject commit e040c3d9c8f6bdf3319e25f42b1cf907725285c9 +Subproject commit a5f4ef0ea3d341cdc0fd7196be170e9811db64fc diff --git a/test/run_tests.py b/test/run_tests.py index 4430d3755..6b436ada2 100755 --- a/test/run_tests.py +++ b/test/run_tests.py @@ -84,6 +84,7 @@ def main(): print('') print("- Running nosetests:") print('') + sys.stdout.flush() argv = [ __file__,