isar-bootstrap: Eliminate multiple debootstraps for the same distro/host

After applying this patch only single debootstrap for
particular platform is created and pointed by corresponding
link under DEPLOY_DIR_IMAGE.

For proper locking of parallel builds shell tasks
apt_config_install, set_locale, setup_chroot, apt_update
and deploy were collapsed to common debootstrap helper.
Due to problems in bitbake with shell functions expansion
under quotes these functions were substituted  by their
bodies.

Signed-off-by: Maxim Yu. Osipov <mosipov@ilbers.de>

meta/classes/image.bbclass

@@ -66,7 +66,7 @@ INITRD_IMAGE ?= "${@get_image_name(d, 'initrd.img')[1]}"
 inherit ${IMAGE_TYPE}
 
 do_rootfs[stamp-extra-info] = "${MACHINE}-${DISTRO}"
-do_rootfs[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_deploy"
+do_rootfs[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_bootstrap"
 
 do_rootfs() {
     die "No root filesystem function defined, please implement in your recipe"

meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb

@@ -8,13 +8,15 @@
 Description = "Minimal host Debian root file system"
 
 WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}-${HOST_DISTRO}-${HOST_ARCH}"
+DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${HOST_DISTRO}-${HOST_ARCH}"
+ISAR_BOOTSTRAP_LOCK = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${HOST_DISTRO}-${HOST_ARCH}.lock"
 
 include isar-bootstrap.inc
 inherit isar-bootstrap-helper
 
-do_generate_keyring[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
+do_generate_keyring[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
 
-do_apt_config_prepare[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
+do_apt_config_prepare[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
 do_apt_config_prepare[dirs] = "${WORKDIR}"
 do_apt_config_prepare[vardeps] += "\
                                    APTPREFS \
@@ -22,54 +24,25 @@ do_apt_config_prepare[vardeps] += "\
                                    DEBDISTRONAME \
                                    APTSRCS \
                                    HOST_DISTRO_APT_SOURCES \
+                                   DEPLOY_ISAR_BOOTSTRAP \
                                   "
 python do_apt_config_prepare() {
-    apt_preferences_out = d.getVar("APTPREFS", True)
-    apt_preferences_list = (d.getVar("HOST_DISTRO_APT_PREFERENCES", True) or ""
-                           ).split()
-    aggregate_files(d, apt_preferences_list, apt_preferences_out)
+    if not os.path.islink(d.getVar("DEPLOY_ISAR_BOOTSTRAP", True)):
+        apt_preferences_out = d.getVar("APTPREFS", True)
+        apt_preferences_list = (d.getVar("HOST_DISTRO_APT_PREFERENCES", True) or ""
+                             ).split()
+        aggregate_files(d, apt_preferences_list, apt_preferences_out)
 
-    apt_sources_out = d.getVar("APTSRCS", True)
-    apt_sources_list = (d.getVar("HOST_DISTRO_APT_SOURCES", True) or "").split()
+        apt_sources_out = d.getVar("APTSRCS", True)
+        apt_sources_list = (d.getVar("HOST_DISTRO_APT_SOURCES", True) or "").split()
 
-    aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
+        aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
 }
-addtask apt_config_prepare before do_build after do_unpack
-
-do_apt_config_install[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
-
+addtask apt_config_prepare before do_bootstrap after do_unpack
 
 do_bootstrap[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
 do_bootstrap[vardeps] += "HOST_DISTRO_APT_SOURCES"
-do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
 do_bootstrap() {
-    if [ -e "${ROOTFSDIR}" ]; then
-       sudo umount -l "${ROOTFSDIR}/dev" || true
-       sudo umount -l "${ROOTFSDIR}/proc" || true
-       sudo rm -rf "${ROOTFSDIR}"
-    fi
-    E="${@bb.utils.export_proxies(d)}"
-    sudo -E "${DEBOOTSTRAP}" --verbose \
-                             --variant=minbase \
-                             --include=locales \
-                             ${@get_distro_components_argument(d, True)} \
-                             ${DEBOOTSTRAP_KEYRING} \
-                             "${@get_distro_suite(d, True)}" \
-                             "${ROOTFSDIR}" \
-                             "${@get_distro_source(d, True)}"
+    isar_bootstrap --host
 }
 addtask bootstrap before do_build after do_generate_keyring
-
-do_deploy[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
-do_deploy[dirs] = "${DEPLOY_DIR_IMAGE}"
-do_deploy() {
-    ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${HOST_DISTRO}-${HOST_ARCH}"
-}
-addtask deploy before do_build after do_apt_update
-
-do_apt_update[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
-
-CLEANFUNCS = "clean_deploy"
-clean_deploy() {
-     rm -f "${DEPLOY_DIR_IMAGE}/isar-bootstrap}-${HOST_DISTRO}-${HOST_ARCH}"
-}

meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb

@@ -8,6 +8,8 @@
 Description = "Minimal target Debian root file system"
 
 WORKDIR = "${TMPDIR}/work/${DISTRO}-${DISTRO_ARCH}/${PN}"
+DEPLOY_ISAR_BOOTSTRAP = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${DISTRO}-${DISTRO_ARCH}"
+ISAR_BOOTSTRAP_LOCK = "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${DISTRO}-${DISTRO_ARCH}.lock"
 
 include isar-bootstrap.inc
 
@@ -21,54 +23,26 @@ do_apt_config_prepare[vardeps] += "\
                                    DEBDISTRONAME \
                                    APTSRCS \
                                    DISTRO_APT_SOURCES \
+                                   DEPLOY_ISAR_BOOTSTRAP \
                                   "
 python do_apt_config_prepare() {
-    apt_preferences_out = d.getVar("APTPREFS", True)
-    apt_preferences_list = (d.getVar("DISTRO_APT_PREFERENCES", True) or ""
-                           ).split()
-    aggregate_files(d, apt_preferences_list, apt_preferences_out)
+    if not os.path.islink(d.getVar("DEPLOY_ISAR_BOOTSTRAP", True)):
+        apt_preferences_out = d.getVar("APTPREFS", True)
+        apt_preferences_list = (d.getVar("DISTRO_APT_PREFERENCES", True) or ""
+                             ).split()
+        aggregate_files(d, apt_preferences_list, apt_preferences_out)
 
-    apt_sources_out = d.getVar("APTSRCS", True)
-    apt_sources_list = (d.getVar("DISTRO_APT_SOURCES", True) or "").split()
+        apt_sources_out = d.getVar("APTSRCS", True)
+        apt_sources_list = (d.getVar("DISTRO_APT_SOURCES", True) or "").split()
 
-    aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
+        aggregate_aptsources_list(d, apt_sources_list, apt_sources_out)
 }
-addtask apt_config_prepare before do_build after do_unpack
-
-do_apt_config_install[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
+addtask apt_config_prepare before do_bootstrap after do_unpack
 
 do_bootstrap[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
 do_bootstrap[vardeps] += "DISTRO_APT_SOURCES"
-do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
 do_bootstrap() {
-    if [ -e "${ROOTFSDIR}" ]; then
-       sudo umount -l "${ROOTFSDIR}/dev" || true
-       sudo umount -l "${ROOTFSDIR}/proc" || true
-       sudo rm -rf "${ROOTFSDIR}"
-    fi
-    E="${@bb.utils.export_proxies(d)}"
-    sudo -E "${DEBOOTSTRAP}" --verbose \
-                             --variant=minbase \
-                             --arch="${DISTRO_ARCH}" \
-                             --include=locales \
-                             ${@get_distro_components_argument(d, False)} \
-                             ${DEBOOTSTRAP_KEYRING} \
-                             "${@get_distro_suite(d, False)}" \
-                             "${ROOTFSDIR}" \
-                             "${@get_distro_source(d, False)}"
+    isar_bootstrap
 }
 addtask bootstrap before do_build after do_generate_keyring
 
-do_deploy[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
-do_deploy[dirs] = "${DEPLOY_DIR_IMAGE}"
-do_deploy() {
-    ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_DIR_IMAGE}/isar-bootstrap-${DISTRO}-${DISTRO_ARCH}"
-}
-addtask deploy before do_build after do_apt_update
-
-do_apt_update[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
-
-CLEANFUNCS = "clean_deploy"
-clean_deploy() {
-     rm -f "${DEPLOY_DIR_IMAGE}/${PN}-${DISTRO}-${DISTRO_ARCH}"
-}

meta/recipes-core/isar-bootstrap/isar-bootstrap.inc

@@ -22,6 +22,7 @@ APTSRCS = "${WORKDIR}/apt-sources"
 APTKEYFILES = ""
 APTKEYRING = "${WORKDIR}/apt-keyring.gpg"
 DEBOOTSTRAP_KEYRING = ""
+DEPLOY_ISAR_BOOTSTRAP ?= ""
 
 python () {
     from urllib.parse import urlparse
@@ -149,52 +150,100 @@ do_generate_keyring() {
 }
 addtask generate_keyring before do_build after do_unpack
 
-do_set_locale() {
-    sudo install -v -m644 "${WORKDIR}/locale" "${ROOTFSDIR}/etc/locale"
 
-    sudo sed -i '/en_US.UTF-8 UTF-8/s/^#//g' "${ROOTFSDIR}/etc/locale.gen"
-    sudo -E chroot "${ROOTFSDIR}" /usr/sbin/locale-gen
-}
-addtask set_locale after do_bootstrap
-
-do_setup_chroot() {
-    sudo install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
-    sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
-}
-addtask setup_chroot before do_build after do_bootstrap
 
 def get_host_release():
     import platform
     rel = platform.release()
     return rel
 
-do_apt_config_install() {
-    sudo mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
-    sudo install -v -m644 "${APTPREFS}" \
-                          "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
-    sudo mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
-    sudo install -v -m644 "${APTSRCS}" \
-                          "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
-    sudo rm -f "${ROOTFSDIR}/etc/apt/sources.list"
-    sudo mkdir -p "${ROOTFSDIR}/etc/apt/apt.conf.d"
-    sudo install -v -m644 "${WORKDIR}/isar-apt.conf" \
-                          "${ROOTFSDIR}/etc/apt/apt.conf.d/50isar.conf"
-
-    if [ "${@get_distro_suite(d, True)}" = "stretch" ] && [ "${@get_host_release().split('.')[0]}" -lt "4" ]; then
-        sudo install -v -m644 "${WORKDIR}/isar-apt-fallback.conf" \
-                              "${ROOTFSDIR}/etc/apt/apt.conf.d/55isar-fallback.conf"
-    fi
-}
-addtask apt_config_install before do_build after do_bootstrap do_apt_config_prepare
 
-do_apt_update() {
-    sudo mount -t devtmpfs -o mode=0755,nosuid devtmpfs ${ROOTFSDIR}/dev
-    sudo mount -t proc none ${ROOTFSDIR}/proc
-
-    E="${@bb.utils.export_proxies(d)}"
-    export DEBIAN_FRONTEND=noninteractive
-    sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
-    sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
+do_bootstrap[vardeps] += "DISTRO_APT_PREMIRRORS"
+do_bootstrap[dirs] = "${DEPLOY_DIR_IMAGE}"
+
+isar_bootstrap() {
+    IS_HOST=""
+    while true; do
+        case "$1" in
+        --host) IS_HOST=1 ;;
+        -*) bbfatal "$0: invalid option specified: $1" ;;
+        *) break ;;
+        esac
+        shift
+    done
+    sudo flock "${ISAR_BOOTSTRAP_LOCK}" -c "\
+        if [ ! -e "${DEPLOY_ISAR_BOOTSTRAP}" ]; then
+            if [ -e "${ROOTFSDIR}" ]; then
+               sudo umount -l "${ROOTFSDIR}/dev" || true
+               sudo umount -l "${ROOTFSDIR}/proc" || true
+               sudo rm -rf "${ROOTFSDIR}"
+            fi
+            E="${@bb.utils.export_proxies(d)}"
+            if [ ${IS_HOST} ]; then
+                sudo -E "${DEBOOTSTRAP}" --verbose \
+                                         --variant=minbase \
+                                         --include=locales \
+                                         ${@get_distro_components_argument(d, True)} \
+                                         ${DEBOOTSTRAP_KEYRING} \
+                                         "${@get_distro_suite(d, True)}" \
+                                         "${ROOTFSDIR}" \
+                                         "${@get_distro_source(d, True)}"
+
+            else
+                 sudo -E "${DEBOOTSTRAP}" --verbose \
+                                          --variant=minbase \
+                                          --arch="${DISTRO_ARCH}" \
+                                          --include=locales \
+                                          ${@get_distro_components_argument(d, False)} \
+                                          ${DEBOOTSTRAP_KEYRING} \
+                                         "${@get_distro_suite(d, False)}" \
+                                         "${ROOTFSDIR}" \
+                                         "${@get_distro_source(d, False)}"
+            fi
+
+            # Install apt config
+            sudo mkdir -p "${ROOTFSDIR}/etc/apt/preferences.d"
+            sudo install -v -m644 "${APTPREFS}" \
+                                  "${ROOTFSDIR}/etc/apt/preferences.d/bootstrap"
+            sudo mkdir -p "${ROOTFSDIR}/etc/apt/sources.list.d"
+            sudo install -v -m644 "${APTSRCS}" \
+                                  "${ROOTFSDIR}/etc/apt/sources.list.d/bootstrap.list"
+            sudo rm -f "${ROOTFSDIR}/etc/apt/sources.list"
+            sudo mkdir -p "${ROOTFSDIR}/etc/apt/apt.conf.d"
+            sudo install -v -m644 "${WORKDIR}/isar-apt.conf" \
+                                  "${ROOTFSDIR}/etc/apt/apt.conf.d/50isar.conf"
+
+            if [ "${@get_distro_suite(d, True)}" = "stretch" ] && [ "${@get_host_release().split('.')[0]}" -lt "4" ]; then
+                sudo install -v -m644 "${WORKDIR}/isar-apt-fallback.conf" \
+                                      "${ROOTFSDIR}/etc/apt/apt.conf.d/55isar-fallback.conf"
+            fi
+
+            # Set locale
+            sudo install -v -m644 "${WORKDIR}/locale" "${ROOTFSDIR}/etc/locale"
+
+            sudo sed -i '/en_US.UTF-8 UTF-8/s/^#//g' "${ROOTFSDIR}/etc/locale.gen"
+            sudo -E chroot "${ROOTFSDIR}" /usr/sbin/locale-gen
+
+            # setup chroot
+            sudo install -v -m755 "${WORKDIR}/chroot-setup.sh" "${ROOTFSDIR}/chroot-setup.sh"
+            sudo "${ROOTFSDIR}/chroot-setup.sh" "setup" "${ROOTFSDIR}"
+
+            # update APT
+            sudo mount -t devtmpfs -o mode=0755,nosuid devtmpfs ${ROOTFSDIR}/dev
+            sudo mount -t proc none ${ROOTFSDIR}/proc
+
+            E="${@bb.utils.export_proxies(d)}"
+            export DEBIAN_FRONTEND=noninteractive
+            sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get update -y
+            sudo -E chroot "${ROOTFSDIR}" /usr/bin/apt-get dist-upgrade -y \
                                       -o Debug::pkgProblemResolver=yes
+
+            # Finalize debootstrap by setting the link in deploy
+            ln -Tfsr "${ROOTFSDIR}" "${DEPLOY_ISAR_BOOTSTRAP}"
+        fi"
+}
+
+CLEANFUNCS = "clean_deploy"
+clean_deploy() {
+    rm -f "${DEPLOY_ISAR_BOOTSTRAP}"
 }
-addtask apt_update before do_build after do_apt_config_install do_set_locale do_setup_chroot

meta/recipes-devtools/buildchroot/buildchroot.bb

@@ -66,7 +66,7 @@ do_build[root_cleandirs] = "${BUILDCHROOT_DIR} \
                             ${BUILDCHROOT_DIR}/isar-apt \
                             ${BUILDCHROOT_DIR}/downloads \
                             ${BUILDCHROOT_DIR}/home/builder"
-do_build[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_deploy"
+do_build[depends] = "isar-apt:do_cache_config isar-bootstrap-target:do_bootstrap"
 
 do_build() {
     setup_root_file_system "${BUILDCHROOT_DIR}" ${BUILDCHROOT_PREINSTALL}

meta/recipes-devtools/sdkchroot/sdkchroot.bb

@@ -31,7 +31,7 @@ do_build[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
 do_build[root_cleandirs] = "${S} \
                             ${S}/isar-apt"
 
-do_build[depends] = "isar-apt-host:do_cache_config isar-bootstrap-host:do_deploy"
+do_build[depends] = "isar-apt-host:do_cache_config isar-bootstrap-host:do_bootstrap"
 
 do_build() {