[TheOneric] Speedup license extraction

.github/workflows/emscripten.yml

@@ -30,7 +30,7 @@ jobs:
 
     - name: Build Binaries
       run: |
-        docker run --rm -v "${PWD}":/code libass/jso:latest
+        docker run --rm --env LC_ALL=C.UTF-8 -v "${PWD}":/code libass/jso:latest
 
     - name: Upload Nightly Build
       uses: actions/upload-artifact@v2

Makefile

@@ -195,36 +195,9 @@ dist/js/subtitles-octopus.js: dist/license/all src/subtitles-octopus.js
 	mkdir -p dist/js
 	awk '1 {print "// "$$0}' dist/license/all | cat - src/subtitles-octopus.js > $@
 
-LIB_LICENSES := brotli expat freetype fribidi fontconfig harfbuzz libass
-LIB_LICENSES_FINDOPT_brotli   := -path ./research -prune -false -o ! -path ./js/decode.min.js
-LIB_LICENSES_FINDOPT_expat    := -path ./expat/fuzz -prune -false -o
-LIB_LICENSES_FINDOPT_freetype := -path ./src/tools -prune -false -o
-LIB_LICENSES_FINDOPT_fribidi  := -path ./bin -prune -false -o
-LIB_LICENSES_FINDOPT_harfbuzz := -path ./test -prune -false -o
-
-$(addprefix dist/license/, $(LIB_LICENSES)): dist/license/%: .git/modules/lib/%/HEAD build/license_extract.sh build/license_defaults
-	@mkdir -p dist/license
-	(cd "lib/$*" && FINDOPTS="$(LIB_LICENSES_FINDOPT_$*)" \
-	 ../../build/license_extract.sh ../../build/license_defaults "$*"  .) > $@
-
-dist/license/subtitlesoctopus: .git/HEAD build/license_extract.sh
-	@mkdir -p dist/license
-	build/license_extract.sh build/license_defaults subtitlesoctopus src > dist/license/subtitlesoctopus
-
-dist/license/all: dist/license/subtitlesoctopus $(addprefix dist/license/, $(LIB_LICENSES)) build/license_fullnotice build/license_lint.awk
-	@echo "# The following lists all copyright notices and licenses for the" >  dist/license/all
-	@echo "# work contained in JavascriptSubtitlesOctopus per project."      >> dist/license/all
-	@echo "" >> dist/license/all
-
-	@echo "Concatenate extracted license info..."
-	@$(foreach LIB_PROJ, subtitlesoctopus $(LIB_LICENSES), \
-		echo "# Project: $(LIB_PROJ)"  >> dist/license/all && \
-		cat  dist/license/$(LIB_PROJ)  >> dist/license/all && \
-	) :
-
-	mv dist/license/all dist/license/all.tmp
-	build/license_lint.awk dist/license/all.tmp build/license_fullnotice
-	cat dist/license/all.tmp build/license_fullnotice > dist/license/all
+dist/license/all:
+	@#FIXME: allow -j in toplevel Makefile and reintegrate licence extraction into this file
+	make -j "$$(nproc)" -f Makefile_licence all
 
 dist/js/COPYRIGHT: dist/license/all
 	cp "$<" "$@"

Makefile_licence

@@ -0,0 +1,37 @@
+# FIXME: temporarily split Makefile to parallelise licence info extraction
+#        once -j can be passed to the toplevel Makefile this should be moved
+#        back into the main Makefile
+
+all: dist/license/all
+.PHONY: all
+
+LIB_LICENSES := brotli expat freetype fribidi fontconfig harfbuzz libass
+LIB_LICENSES_FINDOPT_brotli   := -path ./research -prune -false -o ! -path ./js/decode.min.js
+LIB_LICENSES_FINDOPT_expat    := -path ./expat/fuzz -prune -false -o
+LIB_LICENSES_FINDOPT_freetype := -path ./src/tools -prune -false -o
+LIB_LICENSES_FINDOPT_fribidi  := -path ./bin -prune -false -o
+LIB_LICENSES_FINDOPT_harfbuzz := -path ./test -prune -false -o
+
+$(addprefix dist/license/, $(LIB_LICENSES)): dist/license/%: .git/modules/lib/%/HEAD build/license_extract.sh build/license_defaults
+	@mkdir -p dist/license
+	(cd "lib/$*" && FINDOPTS="$(LIB_LICENSES_FINDOPT_$*)" \
+	 ../../build/license_extract.sh ../../build/license_defaults "$*"  .) > $@
+
+dist/license/subtitlesoctopus: .git/HEAD build/license_extract.sh
+	@mkdir -p dist/license
+	build/license_extract.sh build/license_defaults subtitlesoctopus src > dist/license/subtitlesoctopus
+
+dist/license/all: dist/license/subtitlesoctopus $(addprefix dist/license/, $(LIB_LICENSES)) build/license_fullnotice build/license_lint.awk
+	@echo "# The following lists all copyright notices and licenses for the" >  dist/license/all
+	@echo "# work contained in JavascriptSubtitlesOctopus per project."      >> dist/license/all
+	@echo "" >> dist/license/all
+
+	@echo "Concatenate extracted license info..."
+	@$(foreach LIB_PROJ, subtitlesoctopus $(LIB_LICENSES), \
+		echo "# Project: $(LIB_PROJ)"  >> dist/license/all && \
+		cat  dist/license/$(LIB_PROJ)  >> dist/license/all && \
+	) :
+
+	mv dist/license/all dist/license/all.tmp
+	build/license_lint.awk dist/license/all.tmp build/license_fullnotice
+	cat dist/license/all.tmp build/license_fullnotice > dist/license/all

build/license_extract.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # Copyright 2021 Oneric
 # SPDX-License-Identifier: ISC
@@ -40,9 +40,11 @@ else
     exit 2
 fi
 
+# Do not ignore licensecheck errors
+set -o pipefail
 
-find "$base_dir" $FINDOPTS -type f -regextype egrep -regex '.*\.(c|h|cpp|hpp|js)$' -exec \
-    licensecheck --machine --copyright --deb-fmt '{}' \; \
+find "$base_dir" $FINDOPTS -type f -regextype egrep -regex '.*\.(c|h|cpp|hpp|js)$' -print0 \
+    | xargs -0 -P1 licensecheck --machine --copyright --deb-fmt --encoding UTF-8 \
     | awk -F"$tabulator" -v base_dir="$base_dir" \
         -v def_license="$def_license" -v def_copy="$def_copy" '
             BEGIN {

build/license_lint.awk

@@ -48,7 +48,7 @@ fulltext && /^License: / {
 
 END {
     for (l in licenses) {
-        if(l == "public-domain")
+        if(l == "public-domain" || l == "Unlicense")
           continue # No notice required
         print "Missing full text for '"l"'!" | "cat>&2"
         ++errors

run-buildah-build.sh

@@ -15,4 +15,4 @@ if [ "$FAST" -eq 0 ] ; then
     buildah rm "$CONTAINER" >/dev/null 2>&1 || :
     buildah from --name "$CONTAINER" "$IMAGE":latest
 fi
-buildah run -t -v "${PWD}":/code "$CONTAINER" $cmd "$@"
+buildah run -t --env LC_ALL=C.UTF-8 -v "${PWD}":/code "$CONTAINER" $cmd "$@"

run-docker-build.sh

@@ -8,7 +8,7 @@ if [ "$FAST" -eq 0 ] ; then
     docker build -t "$IMAGE" .
 fi
 if [ "$#" -eq 0 ] ; then
-    docker run -it --rm -v "${PWD}":/code --name "$CONTAINER" "$IMAGE":latest
+    docker run -it --rm --env LC_ALL=C.UTF-8 -v "${PWD}":/code --name "$CONTAINER" "$IMAGE":latest
 else
-    docker run -it --rm -v "${PWD}":/code --name "$CONTAINER" "$IMAGE":latest "$@"
+    docker run -it --rm --env LC_ALL=C.UTF-8 -v "${PWD}":/code --name "$CONTAINER" "$IMAGE":latest "$@"
 fi