dpkg: Make mount buildroot reliable

We use mounting for several tasks. Mounting and unmounting is performed
in shell scripts. If a command fails before unmounting, the directories
remain mounted.

Implement exception handling around the scripts to ensure that
unmounting is performed also in case of script failure. The number of
unmounting attempts has been limited to avoid infinite loops.

Signed-off-by: Anton Mikanovich <amikan@ilbers.de>

meta/classes/dpkg-base.bbclass

@@ -61,12 +61,7 @@ addtask patch before do_adjust_git
 
 SRC_APT ?= ""
 
-do_apt_fetch() {
-    if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then
-        return 0
-    fi
-    dpkg_do_mounts
-    E="${@ isar_export_proxies(d)}"
+fetch_apt() {
     sudo -E chroot ${BUILDCHROOT_DIR} /usr/bin/apt-get update \
         -o Dir::Etc::SourceList="sources.list.d/isar-apt.list" \
         -o Dir::Etc::SourceParts="-" \
@@ -76,21 +71,25 @@ do_apt_fetch() {
         sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \
             sh -c 'mkdir -p /downloads/deb-src/"$1"/"$2" && cd /downloads/deb-src/"$1"/"$2" && apt-get -y --download-only --only-source source "$2"' my_script "${DISTRO}" "${uri}"
     done
+}
+
+python do_apt_fetch() {
+    src_apt = d.getVar("SRC_APT", True)
+    if not src_apt:
+        return 0
 
-    dpkg_undo_mounts
+    dpkg_do_mounts(d)
+    try:
+        isar_export_proxies(d)
+        bb.build.exec_func("fetch_apt", d)
+    finally:
+        dpkg_undo_mounts(d)
 }
 
 addtask apt_fetch after do_unpack before do_apt_unpack
 do_apt_fetch[lockfiles] += "${REPO_ISAR_DIR}/isar.lock"
 
-do_apt_unpack() {
-    if [ -z "${@d.getVar("SRC_APT", True).strip()}" ]; then
-        return 0
-    fi
-    rm -rf ${S}
-    dpkg_do_mounts
-    E="${@ isar_export_proxies(d)}"
-
+unpack_apt() {
     for uri in "${SRC_APT}"; do
         sudo -E chroot --userspec=$( id -u ):$( id -g ) ${BUILDCHROOT_DIR} \
             sh -c ' \
@@ -101,8 +100,25 @@ do_apt_unpack() {
                 dpkg-source -x "${dscfile}" "${PPS}"' \
                     my_script "${DISTRO}" "${uri}"
     done
+}
 
-    dpkg_undo_mounts
+python do_apt_unpack() {
+    import shutil
+
+    src_apt = d.getVar("SRC_APT", True)
+    if not src_apt:
+        return 0
+
+    srcsubdir = d.getVar('S', True)
+    if os.path.exists(srcsubdir):
+        shutil.rmtree(srcsubdir)
+
+    dpkg_do_mounts(d)
+    try:
+        isar_export_proxies(d)
+        bb.build.exec_func("unpack_apt", d)
+    finally:
+        dpkg_undo_mounts(d)
 }
 
 addtask apt_unpack after do_apt_fetch before do_patch
@@ -146,25 +162,37 @@ do_prepare_build[deptask] = "do_deploy_deb"
 
 BUILDROOT = "${BUILDCHROOT_DIR}/${PP}"
 
-dpkg_do_mounts() {
-    mkdir -p ${BUILDROOT}
-    sudo mount --bind ${WORKDIR} ${BUILDROOT}
-
-    buildchroot_do_mounts
-}
-
-dpkg_undo_mounts() {
-    i=1
-    while ! sudo umount ${BUILDROOT}; do
-        sleep 0.1
-        i=`expr $i + 1`
-        if [ $i -gt 100 ]; then
-            bbwarn "${BUILDROOT}: Couldn't unmount, retrying..."
-            i=1
-        fi
-    done
-    sudo rmdir ${BUILDROOT}
-}
+def ismount(path):
+    real = os.path.realpath(path)
+    with open('/proc/mounts') as f:
+        for line in f.readlines():
+            if len(line.split()) > 2 and real == line.split()[1]:
+                return True
+    return False
+
+def dpkg_do_mounts(d):
+    buildroot = d.getVar('BUILDROOT', True)
+    if ismount(buildroot):
+        bb.warn('Path %s already mounted!' % buildroot)
+        return
+    workdir = d.getVar('WORKDIR', True)
+    os.makedirs(buildroot, exist_ok=True)
+    os.system('sudo mount --bind %s %s' % (workdir, buildroot))
+    bb.build.exec_func("buildchroot_do_mounts", d)
+
+def dpkg_undo_mounts(d):
+    buildroot = d.getVar('BUILDROOT', True)
+    if not ismount(buildroot):
+        bb.warn('Path %s not mounted!' % buildroot)
+        return
+    for i in range(200):
+        if not os.system('sudo umount %s' % buildroot):
+            os.rmdir(buildroot)
+            return
+        if i % 100 == 0:
+            bb.warn("%s: Couldn't unmount, retrying..." % buildroot)
+        time.sleep(0.1)
+    bb.fatal("Couldn't unmount, exiting...")
 
 # Placeholder for actual dpkg_runbuild() implementation
 dpkg_runbuild() {
@@ -174,10 +202,12 @@ dpkg_runbuild() {
 python do_dpkg_build() {
     lock = bb.utils.lockfile(d.getVar("REPO_ISAR_DIR") + "/isar.lock",
                              shared=True)
-    bb.build.exec_func("dpkg_do_mounts", d)
-    bb.build.exec_func("dpkg_runbuild", d)
-    bb.build.exec_func("dpkg_undo_mounts", d)
-    bb.utils.unlockfile(lock)
+    dpkg_do_mounts(d)
+    try:
+        bb.build.exec_func("dpkg_runbuild", d)
+    finally:
+        dpkg_undo_mounts(d)
+        bb.utils.unlockfile(lock)
 }
 
 addtask dpkg_build before do_build
@@ -221,16 +251,16 @@ python do_devshell() {
     oe_lib_path = os.path.join(d.getVar('LAYERDIR_core'), 'lib')
     sys.path.insert(0, oe_lib_path)
 
-    bb.build.exec_func('dpkg_do_mounts', d)
-
-    isar_export_proxies(d)
-
-    buildchroot = d.getVar('BUILDCHROOT_DIR')
-    pp_pps = os.path.join(d.getVar('PP'), d.getVar('PPS'))
-    termcmd = "sudo -E chroot {0} sh -c 'cd {1}; $SHELL -i'"
-    oe_terminal(termcmd.format(buildchroot, pp_pps), "Isar devshell", d)
+    dpkg_do_mounts(d)
+    try:
+        isar_export_proxies(d)
 
-    bb.build.exec_func('dpkg_undo_mounts', d)
+        buildchroot = d.getVar('BUILDCHROOT_DIR')
+        pp_pps = os.path.join(d.getVar('PP'), d.getVar('PPS'))
+        termcmd = "sudo -E chroot {0} sh -c 'cd {1}; $SHELL -i'"
+        oe_terminal(termcmd.format(buildchroot, pp_pps), "Isar devshell", d)
+    finally:
+        dpkg_undo_mounts(d)
 }
 
 addtask devshell after do_prepare_build

meta/classes/dpkg-gbp.bbclass

@@ -12,12 +12,7 @@ PATCHTOOL ?= "git"
 GBP_DEPENDS ?= "git-buildpackage pristine-tar"
 GBP_EXTRA_OPTIONS ?= "--git-pristine-tar"
 
-do_install_builddeps_append() {
-    dpkg_do_mounts
-    distro="${DISTRO}"
-    if [ ${ISAR_CROSS_COMPILE} -eq 1 ]; then
-       distro="${HOST_DISTRO}"
-    fi
+builddeps_install_append() {
     deb_dl_dir_import "${BUILDCHROOT_DIR}" "${distro}"
     sudo -E chroot ${BUILDCHROOT_DIR} \
         apt-get install -y -o Debug::pkgProblemResolver=yes \
@@ -26,7 +21,6 @@ do_install_builddeps_append() {
     sudo -E chroot ${BUILDCHROOT_DIR} \
         apt-get install -y -o Debug::pkgProblemResolver=yes \
                         --no-install-recommends ${GBP_DEPENDS}
-    dpkg_undo_mounts
 }
 
 dpkg_runbuild_prepend() {

meta/classes/dpkg.bbclass

@@ -6,9 +6,7 @@ inherit dpkg-base
 PACKAGE_ARCH ?= "${DISTRO_ARCH}"
 
 # Install build dependencies for package
-do_install_builddeps() {
-    dpkg_do_mounts
-    E="${@ isar_export_proxies(d)}"
+builddeps_install() {
     distro="${DISTRO}"
     if [ ${ISAR_CROSS_COMPILE} -eq 1 ]; then
        distro="${HOST_DISTRO}"
@@ -19,7 +17,15 @@ do_install_builddeps() {
     deb_dl_dir_export "${BUILDCHROOT_DIR}" "${distro}"
     sudo -E chroot ${BUILDCHROOT_DIR} /isar/deps.sh \
         ${PP}/${PPS} ${PACKAGE_ARCH}
-    dpkg_undo_mounts
+}
+
+python do_install_builddeps() {
+    dpkg_do_mounts(d)
+    isar_export_proxies(d)
+    try:
+        bb.build.exec_func("builddeps_install", d)
+    finally:
+        dpkg_undo_mounts(d)
 }
 
 addtask install_builddeps after do_prepare_build before do_dpkg_build