isar-bootstrap: debootstrap https support

Building 'isar-bootstrap-target' fails when there are HTTPS URIs in
the distro APT sources and the first URI is not HTTPS.
The first URI in the APT sources is passed to 'debootstrap' and
the distro APT sources file is written to the isar bootstrap rootfs.
Then, following 'apt-get update' fails due to apt-transport-https,
ca-certificates being missing.

This patch allows a distro to specify 'DISTRO_BOOTSTRAP_BASE_PACKAGES'
and introduces 'https-support' concept using bitbake OVERRIDES.
An example usage is specifying the necessary packages for http support
via 'DISTRO_BOOTSTRAP_BASE_PACKAGES_append_https-support' in distro
configuration.

Signed-off-by: Hosgor, Tolga (CT RDA DS EU TR MTS) <tolga.hosgor@siemens.com>
Signed-off-by: Maxim Yu. Osipov <mosipov@ilbers.de>

meta-isar/conf/multiconfig/qemuamd64-buster.conf

@@ -17,3 +17,5 @@ QEMU_ARCH ?= "x86_64"
 QEMU_MACHINE ?= "q35"
 QEMU_CPU ?= ""
 QEMU_DISK_ARGS ?= "-hda ##ROOTFS_IMAGE## -bios /usr/local/share/ovmf/OVMF.fd"
+
+DISTRO_BOOTSTRAP_BASE_PACKAGES_append_https-support = " apt-transport-https ca-certificates"

meta-isar/conf/multiconfig/qemuamd64-jessie.conf

@@ -14,3 +14,5 @@ QEMU_ARCH ?= "x86_64"
 QEMU_MACHINE ?= "pc"
 QEMU_CPU ?= ""
 QEMU_DISK_ARGS ?= "-hda ##ROOTFS_IMAGE##"
+
+DISTRO_BOOTSTRAP_BASE_PACKAGES_append_https-support = " apt-transport-https ca-certificates"

meta/recipes-core/isar-bootstrap/isar-bootstrap-host.bb

@@ -42,6 +42,8 @@ python do_apt_config_prepare() {
 }
 addtask apt_config_prepare before do_bootstrap after do_unpack
 
+OVERRIDES_append = ":${@get_distro_needs_https_support(d, True)}"
+
 do_bootstrap[stamp-extra-info] = "${HOST_DISTRO}-${HOST_ARCH}"
 do_bootstrap[vardeps] += "HOST_DISTRO_APT_SOURCES"
 do_bootstrap() {

meta/recipes-core/isar-bootstrap/isar-bootstrap-target.bb

@@ -41,6 +41,8 @@ python do_apt_config_prepare() {
 }
 addtask apt_config_prepare before do_bootstrap after do_unpack
 
+OVERRIDES_append = ":${@get_distro_needs_https_support(d, False)}"
+
 do_bootstrap[stamp-extra-info] = "${DISTRO}-${DISTRO_ARCH}"
 do_bootstrap[vardeps] += "DISTRO_APT_SOURCES"
 do_bootstrap() {

meta/recipes-core/isar-bootstrap/isar-bootstrap.inc

@@ -26,6 +26,7 @@ APTKEYFILES = ""
 APTKEYRING = "${WORKDIR}/apt-keyring.gpg"
 DEBOOTSTRAP_KEYRING = ""
 DEPLOY_ISAR_BOOTSTRAP ?= ""
+DISTRO_BOOTSTRAP_BASE_PACKAGES = "locales"
 
 DISTRO_APT_PREMIRRORS ?= "${@ "http://ftp\.(\S+\.)?debian.org  file:///${REPO_BASE_DIR} \n" if bb.utils.to_boolean(d.getVar('ISAR_USE_CACHED_BASE_REPO')) else "" }"
 
@@ -116,11 +117,15 @@ def aggregate_aptsources_list(d, file_list, file_out):
                     out_fd.write("\n".encode())
             out_fd.write("\n".encode())
 
-def get_distro_primary_source_entry(d, is_host=False):
+def get_aptsources_list(d, is_host=False):
     if is_host:
         apt_sources_list = (d.getVar("HOST_DISTRO_APT_SOURCES", True) or "").split()
     else:
         apt_sources_list = (d.getVar("DISTRO_APT_SOURCES", True) or "").split()
+    return apt_sources_list
+
+def get_distro_primary_source_entry(d, is_host=False):
+    apt_sources_list = get_aptsources_list(d, is_host)
     for entry in apt_sources_list:
         entry_real = bb.parse.resolve_file(entry, d)
         with open(entry_real, "r") as in_fd:
@@ -132,6 +137,24 @@ def get_distro_primary_source_entry(d, is_host=False):
                         return parsed[2:]
     return ["", "", ""]
 
+def get_distro_have_https_source(d, is_host=False):
+    for entry in get_aptsources_list(d, is_host):
+        entry_real = bb.parse.resolve_file(entry, d)
+        with open(entry_real, "r") as in_fd:
+            for line in in_fd:
+                parsed = parse_aptsources_list_line(line)
+                if parsed:
+                    parsed = get_apt_source_mirror(d, parsed)
+                    if parsed[2].startswith("https://"):
+                        return True
+    return False
+
+def get_distro_needs_https_support(d, is_host=False):
+    if get_distro_have_https_source(d, is_host):
+        return "https-support"
+    else:
+        return ""
+
 def get_distro_source(d, is_host):
     return get_distro_primary_source_entry(d, is_host)[0]
 
@@ -178,7 +201,7 @@ isar_bootstrap() {
         esac
         shift
     done
-    debootstrap_args="--verbose --variant=minbase --include=locales "
+    debootstrap_args="--verbose --variant=minbase --include='${DISTRO_BOOTSTRAP_BASE_PACKAGES}'"
     if [ "${ISAR_USE_CACHED_BASE_REPO}" = "1" ]; then
         debootstrap_args="$debootstrap_args --no-check-gpg"
     fi