Merge remote-tracking branch 'remotes/philmd-gitlab/tags/edk2-next-20191007' into staging

Improve scripts relying on the EDK2 submodule,
drop Python2 dependency in EDK2 build scripts.

# gpg: Signature made Mon 07 Oct 2019 14:31:38 BST
# gpg:                using RSA key 89C1E78F601EE86C867495CBA2A3FD6EDEADC0DE
# gpg: Good signature from "Philippe Mathieu-Daudé (Phil) <philmd@redhat.com>" [marginal]
# gpg: WARNING: This key is not certified with sufficiently trusted signatures!
# gpg:          It is not certain that the signature belongs to the owner.
# Primary key fingerprint: 89C1 E78F 601E E86C 8674  95CB A2A3 FD6E DEAD C0DE

* remotes/philmd-gitlab/tags/edk2-next-20191007:
  edk2 build scripts: work around TianoCore#1607 without forcing Python 2
  edk2 build scripts: honor external BaseTools flags with uefi-test-tools
  roms: Add a 'make help' target alias
  roms/Makefile.edk2: don't pull in submodules when building from tarball
  make-release: pull in edk2 submodules so we can build it from tarballs

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

roms/Makefile

@@ -51,7 +51,7 @@ SEABIOS_EXTRAVERSION="-prebuilt.qemu.org"
 #
 EDK2_EFIROM = edk2/BaseTools/Source/C/bin/EfiRom
 
-default:
+default help:
 	@echo "nothing is build by default"
 	@echo "available build targets:"
 	@echo "  bios               -- update bios.bin (seabios)"
@@ -141,6 +141,7 @@ build-efi-roms: build-pxe-roms
 #
 edk2-basetools:
 	$(MAKE) -C edk2/BaseTools \
+		PYTHON_COMMAND=$${EDK2_PYTHON_COMMAND:-python3} \
 		EXTRA_OPTFLAGS='$(EDK2_BASETOOLS_OPTFLAGS)' \
 		EXTRA_LDFLAGS='$(EDK2_BASETOOLS_LDFLAGS)'
 

roms/Makefile.edk2

@@ -46,8 +46,13 @@ all: $(foreach flashdev,$(flashdevs),../pc-bios/edk2-$(flashdev).fd.bz2) \
 # files.
 .INTERMEDIATE: $(foreach flashdev,$(flashdevs),../pc-bios/edk2-$(flashdev).fd)
 
+# Fetch edk2 submodule's submodules. If it is not in a git tree, assume
+# we're building from a tarball and that they've already been fetched by
+# make-release/tarball scripts.
 submodules:
-	cd edk2 && git submodule update --init --force
+	if test -d edk2/.git; then \
+		cd edk2 && git submodule update --init --force; \
+	fi
 
 # See notes on the ".NOTPARALLEL" target and the "+" indicator in
 # "tests/uefi-test-tools/Makefile".

roms/edk2-build.sh

@@ -27,8 +27,7 @@ shift $num_args
 
 cd edk2
 
-# Work around <https://bugzilla.tianocore.org/show_bug.cgi?id=1607>.
+export PYTHON_COMMAND=${EDK2_PYTHON_COMMAND:-python3}
-export PYTHON_COMMAND=python2
 
 # Source "edksetup.sh" carefully.
 set +e +u +C
@@ -43,6 +42,7 @@ fi
 # any), for the edk2 "build" utility.
 source ../edk2-funcs.sh
 edk2_toolchain=$(qemu_edk2_get_toolchain "$emulation_target")
+MAKEFLAGS=$(qemu_edk2_quirk_tianocore_1607 "$MAKEFLAGS")
 edk2_thread_count=$(qemu_edk2_get_thread_count "$MAKEFLAGS")
 qemu_edk2_set_cross_env "$emulation_target"
 

roms/edk2-funcs.sh

@@ -251,3 +251,20 @@ qemu_edk2_get_thread_count()
     printf '1\n'
   fi
 }
+
+
+# Work around <https://bugzilla.tianocore.org/show_bug.cgi?id=1607> by
+# filtering jobserver-related flags out of MAKEFLAGS. Print the result to the
+# standard output.
+#
+# Parameters:
+#   $1: the value of the MAKEFLAGS variable
+qemu_edk2_quirk_tianocore_1607()
+{
+  local makeflags="$1"
+
+  printf %s "$makeflags" \
+  | LC_ALL=C sed --regexp-extended \
+      --expression='s/--jobserver-(auth|fds)=[0-9]+,[0-9]+//' \
+      --expression='s/-j([0-9]+)?//'
+}

scripts/make-release

@@ -20,6 +20,14 @@ git checkout "v${version}"
 git submodule update --init
 (cd roms/seabios && git describe --tags --long --dirty > .version)
 (cd roms/skiboot && ./make_version.sh > .version)
+# Fetch edk2 submodule's submodules, since it won't have access to them via
+# the tarball later.
+#
+# A more uniform way to handle this sort of situation would be nice, but we
+# don't necessarily have much control over how a submodule handles its
+# submodule dependencies, so we continue to handle these on a case-by-case
+# basis for now.
+(cd roms/edk2 && git submodule update --init)
 popd
 tar --exclude=.git -cjf ${destination}.tar.bz2 ${destination}
 rm -rf ${destination}

tests/uefi-test-tools/Makefile

@@ -99,7 +99,10 @@ Build/bios-tables-test.%.efi: build-edk2-tools
 	+./build.sh $(edk2_dir) BiosTablesTest $* $@
 
 build-edk2-tools:
-	$(MAKE) -C $(edk2_dir)/BaseTools
+	$(MAKE) -C $(edk2_dir)/BaseTools \
+		PYTHON_COMMAND=$${EDK2_PYTHON_COMMAND:-python3} \
+		EXTRA_OPTFLAGS='$(EDK2_BASETOOLS_OPTFLAGS)' \
+		EXTRA_LDFLAGS='$(EDK2_BASETOOLS_LDFLAGS)'
 
 clean:
 	rm -rf Build Conf log

tests/uefi-test-tools/build.sh

@@ -29,8 +29,7 @@ export PACKAGES_PATH=$(realpath -- "$edk2_dir")
 export WORKSPACE=$PWD
 mkdir -p Conf
 
-# Work around <https://bugzilla.tianocore.org/show_bug.cgi?id=1607>.
+export PYTHON_COMMAND=${EDK2_PYTHON_COMMAND:-python3}
-export PYTHON_COMMAND=python2
 
 # Source "edksetup.sh" carefully.
 set +e +u +C
@@ -46,12 +45,15 @@ fi
 source "$edk2_dir/../edk2-funcs.sh"
 edk2_arch=$(qemu_edk2_get_arch "$emulation_target")
 edk2_toolchain=$(qemu_edk2_get_toolchain "$emulation_target")
+MAKEFLAGS=$(qemu_edk2_quirk_tianocore_1607 "$MAKEFLAGS")
+edk2_thread_count=$(qemu_edk2_get_thread_count "$MAKEFLAGS")
 qemu_edk2_set_cross_env "$emulation_target"
 
 # Build the UEFI binary
 mkdir -p log
 build \
   --arch="$edk2_arch" \
+  -n "$edk2_thread_count" \
   --buildtarget=DEBUG \
   --platform=UefiTestToolsPkg/UefiTestToolsPkg.dsc \
   --tagname="$edk2_toolchain" \