Release v1.15

This release adds support for using DNF on Clear Linux.

Signed-off-by: Patrick McCarty <patrick.mccarty@intel.com>

README

@@ -1,3 +1,23 @@
+Building this Project
+=====================
+For help using automake see
+http://www.gnu.org/software/automake/manual/html_node/index.html
+
+Configure:
+    aclocal
+    autoconf
+    automake --add-missing
+    # Defaults to /usr/local or specific --prefix=/usr
+    # to overwrite the installed version
+    ./configure
+
+Build:
+    make
+
+Install:
+    make -n install
+    sudo make install
+
 Configuration File
 ==================
 Please update the configuration file paths to point to where your

builder.conf

@@ -1,4 +1,17 @@
 [Builder]
-SERVER_STATE_DIR = /var/lib/update
-BUNDLE_DIR = /home/clr/mix/bundles
-YUM_CONF = /usr/share/defaults/bundle-chroot-builder/yum.conf
+SERVER_STATE_DIR=/home/clr/mix/update
+BUNDLE_DIR=/home/clr/mix/mix-bundles
+YUM_CONF=/home/clr/mix/.yum-mix.conf
+CERT=/home/clr/mix/Swupd_Root.pem
+VERSIONS_PATH=/home/clr/mix
+
+[swupd]
+BUNDLE=os-core-update
+CONTENTURL=<URL where the content will be hosted>
+VERSIONURL=<URL where the version of the mix will be hosted>
+FORMAT=1
+
+[Server]
+debuginfo_banned=true
+debuginfo_lib=/usr/lib/debug/
+debuginfo_src=/usr/src/debug/

bundle-chroot-builder.py

@@ -25,6 +25,7 @@ import argparse
 import configparser
 import os
 import os.path
+import io
 import platform
 import re
 import shutil
@@ -47,14 +48,36 @@ def handle_options():
     return args
 
 
-def read_config(args):
+def get_config(args):
     buildconf='/usr/share/defaults/bundle-chroot-builder/builder.conf'
     if os.path.isfile('/etc/bundle-chroot-builder/builder.conf'):
         buildconf = '/etc/bundle-chroot-builder/builder.conf'
     if args.config:
         buildconf = args.config
+
+    print("Reading from %s" % buildconf)
+    cfg_txt = ""
+    # Check that the environment variables in the config file are valid
+    pattern = re.compile("\$\{?(\w+)\}?")
+    for i, line in enumerate(open(buildconf, 'r')):
+        for match in re.finditer(pattern, line):
+            if not match.group(1) in os.environ:
+                print("ERROR:\nbuilder.conf contains an undefined environment variable: %s on line %s\n"
+                        % (i+1, match.group(1)))
+                exit(1)
+        cfg_txt += os.path.expandvars(line)
+
     config = configparser.ConfigParser()
-    config.read(buildconf)
+    config.readfp(io.StringIO(cfg_txt))
+    return config
+
+
+def read_config(args):
+    config = get_config(args)
+    for option in ['SERVER_STATE_DIR', 'BUNDLE_DIR', 'YUM_CONF']:
+            if config.has_option('Builder', option) == False:
+                print("ERROR:\nbuilder.conf is missing:\n[Builder]\n%s\n" % option)
+                exit(1)
 
     """Read the configuration file for our script values"""
     conf = config['Builder']
@@ -67,12 +90,19 @@ def read_config(args):
 
 def install_bundle(out_dir, postfix, bundle, bundles, yum_cmd):
     """Helper function to yum install a bundle"""
-    m4 = subprocess.Popen(["m4", bundles + "/" + bundle], cwd=bundles, stdout=subprocess.PIPE)
-    m4.wait()
-    pkgs = m4.stdout.readlines()
+    lines = []
+    try:
+        output = subprocess.check_output(["m4", bundles + "/" + bundle], cwd=bundles, bufsize=1, universal_newlines=True)
+    except subprocess.CalledProcessError as e:
+        print('ERROR {0}: m4 failed on {1}/{2}'.format(e.returncode, bundles, bundle))
+        raise
+    for line in output:
+        lines.append(line)
+
+    pkgs = "".join(lines)
     to_install = []
-    for pkg in pkgs:
-        pkg = pkg.decode("utf-8").strip()
+    for pkg in pkgs.splitlines():
+        pkg = pkg.strip()
         # Don't add blank lines or lines with leading '#'
         if len(pkg) == 0 or pkg[0] == "#":
             continue
@@ -138,6 +168,37 @@ def clean_bundle(out_dir, bundle, bundles, yum_cmd):
     os.chdir(prev_dir)
 
 
+def write_default_server_ini(state_dir, server_conf):
+    """
+    Write default server.ini file in the state dir in the following format
+
+    [Server]
+    emptydir={state_dir}/empty/
+    imagebase={state_dir}/image/
+    outputdir={state_dir}/www/
+
+    [Debuginfo]
+    banned=true
+    lib=/usr/lib/debug/
+    src=/usr/src/debug/
+    """
+    contents = ("[Server]\n"
+                "emptydir={0}/empty/\n"
+                "imagebase={0}/image/\n"
+                "outputdir={0}/www/\n".format(state_dir))
+    if server_conf:
+        contents += ("\n[Debuginfo]\n"
+                    "banned={}\n"
+                    "lib={}\n"
+                    "src={}\n"
+                    .format(server_conf.get('debuginfo_banned'),
+                            server_conf.get('debuginfo_lib'),
+                            server_conf.get('debuginfo_src')))
+
+    with open(state_dir + "/server.ini", "w+") as serverini:
+        serverini.write(contents)
+
+
 def create_chroots(args, state_dir, bundles, yum_conf):
     """The state_dir should always be created if it does not exist"""
     if os.path.isdir(state_dir) == False:
@@ -151,12 +212,15 @@ def create_chroots(args, state_dir, bundles, yum_conf):
     if os.path.isdir(state_dir + "/www/0") == False:
         os.makedirs(state_dir + "/www/0")
 
-    """Create server.ini and groups.ini for create_update later on"""
-    with open(state_dir + "/server.ini", "w+") as serverini:
-        serverini.write("[Server]\nemptydir=" + state_dir + "/empty/\nimagebase=" + state_dir + "/image/\noutputdir=" + state_dir + "/www/\n")
+    # Create server.ini and groups.ini for create_update later on
+    config = get_config(args)
+    config = config['Server'] if 'Server' in config else {}
+    write_default_server_ini(state_dir, config)
     with open(state_dir + "/groups.ini", "w+") as groupsini:
-        for bundle in os.listdir(bundles):
-            groupsini.write("[" + bundle + "]\ngroup=" + bundle + "\n\n")
+        bundle_list = os.listdir(bundles)
+        bundle_list = trim_bundles(bundle_list)
+        for bundle in bundle_list:
+            groupsini.write("[{0}]\ngroup={0}\n\n".format(bundle))
 
     """Setup chroots for bundles"""
     bversion = ""
@@ -167,8 +231,8 @@ def create_chroots(args, state_dir, bundles, yum_conf):
         out_version = args.version
     build_version = args.version
 
-    if os.path.exists(state_dir + "/image/latest.version") == False:
-        with open(state_dir + "/image/latest.version", "w") as latestver:
+    if os.path.exists(state_dir + "/image/LAST_VER") == False:
+        with open(state_dir + "/image/LAST_VER", "w") as latestver:
             latestver.write("0\n")
 
     config = configparser.ConfigParser()
@@ -185,12 +249,21 @@ def create_chroots(args, state_dir, bundles, yum_conf):
 
     out_dir = state_dir + "/image/" + out_version
 
-    if platform.dist()[0] == "fedora" and int(platform.dist()[1]) >= 22:
+    try:
+        with open("/usr/share/clear/version", "r") as verfile:
+            clrver = int(verfile.read().strip())
+    except Exception:
+        clrver = None
+
+    if clrver and clrver > 20650:
+        print("using dnf on clr")
+        packager = ["dnf"]
+    elif platform.dist()[0] == "fedora" and int(platform.dist()[1]) >= 22:
         print("using dnf instead of yum")
         packager = ["dnf"]
     else:
         packager = ["yum"]
-    yum_cmd = packager + ["--config={}".format(yum_conf), "-y", "--noplugins", "--releasever={}".format(build_version)]
+    yum_cmd = packager + ["--config={}".format(yum_conf), "-y", "--releasever={}".format(build_version)]
     if 'local' in config == False:
         try:
             urllib.request.urlopen(conf_baseurl)
@@ -236,7 +309,7 @@ def create_chroots(args, state_dir, bundles, yum_conf):
 
     print("Noting os-core package list")
     with open(out_dir + "/versions", "w") as file:
-        subprocess.Popen(yum_cmd + ["--installroot={}/os-core".format(out_dir), "list"], stdout=file).wait()
+        subprocess.Popen(yum_cmd + ["--quiet", "--installroot={}/os-core".format(out_dir), "list"], stdout=file).wait()
     with open(out_dir + "/packages-os-core", "w") as file:
         subprocess.Popen(['rpm', '--root={}/os-core'.format(out_dir),
                           '-qa', '--queryformat', '%{NAME}\t%{SOURCERPM}\n'], stdout=file).wait()
@@ -253,6 +326,7 @@ def create_chroots(args, state_dir, bundles, yum_conf):
 
     bundle_list = os.listdir(bundles)
     bundle_list.remove('os-core')
+    bundle_list = trim_bundles(bundle_list)
     pool = multiprocessing.Pool()
     results_list = []
 
@@ -271,6 +345,34 @@ def create_chroots(args, state_dir, bundles, yum_conf):
     for r in results_list:
         r.get()
 
+    """Read the URL values from builder.conf and insert them into os-core-update to swupd knows where to pull content from"""
+    config = get_config(args)
+
+    """Read the configuration file for our script values"""
+    for option in ['BUNDLE', 'CONTENTURL', 'VERSIONURL', 'FORMAT']:
+        if config.has_option('swupd', option) == False:
+            print("ERROR:\nbuilder.conf is missing:\n[swupd]\n%s\n" % option)
+            exit(1)
+    conf = config['swupd']
+    bundlename = conf['BUNDLE']
+    contenturl = conf['CONTENTURL']
+    versionurl = conf['VERSIONURL']
+    formatname = conf['FORMAT']
+
+    print("Adding SWUPD default values to '{}' bundle...".format(bundlename))
+    """Do not add a leading slash on anything except the first variable in os.path.join!"""
+    confpath = os.path.join(out_dir, bundlename, "usr/share/defaults/swupd/")
+    os.makedirs(confpath, exist_ok=True)
+    with open(os.path.join(confpath, "contenturl"), "w") as file:
+        file.writelines(contenturl)
+    print("  contenturl: {}".format(contenturl))
+    with open(os.path.join(confpath, "versionurl"), "w") as file:
+        file.writelines(versionurl)
+    print("  versionurl: {}".format(versionurl))
+    with open(os.path.join(confpath, "format"), "w") as file:
+        file.writelines(formatname)
+    print("  format: {}".format(formatname))
+
     print("Creating package to file mappings")
     package_mapping = {}
     map_files = [f for f in os.listdir(out_dir) if f.startswith("pkgmap-")]
@@ -280,14 +382,18 @@ def create_chroots(args, state_dir, bundles, yum_conf):
         # real package's filename I'm using it instead of recursing through
         # temporary per bundle folders
         package_name = map_file[map_file.find(" * ") + 3:-len(".src.rpm")]
-        with open(out_dir + "/" +  map_file, "r") as file:
+        with open(out_dir + "/" +  map_file, "rb") as file:
             path_list = file.readlines()
         if package_name not in package_mapping:
             package_mapping[package_name] = set()
         for path in path_list:
+            # RPM prints out a specific string for subpackages that contain no
+            # files. It should be excluded from the SRPM file list.
+            if re.match(br"\(contains no files\)\n", path):
+                continue
             package_mapping[package_name].add(path)
     for package_name, paths in package_mapping.items():
-        with open(out_dir + "/files-{}".format(package_name), "w") as file:
+        with open(out_dir + "/files-{}".format(package_name), "wb") as file:
             file.writelines(sorted(paths))
     for map_file in map_files:
         os.unlink(out_dir + "/" + map_file)
@@ -297,16 +403,17 @@ def create_chroots(args, state_dir, bundles, yum_conf):
 
     print("Cleaning package list")
     web_dir = state_dir + "/www/" + out_version + "/"
+    image_dir = state_dir + "/image/" + out_version + "/"
     if os.path.isdir(web_dir):
         print("  removing pre-existing {} before setting version file" .format(web_dir))
         os.system('rm -rf '+web_dir)
     # FIXME: Figure out problems with the below to replace the above.
     # Consider all uses of os.system("rm -fr") as FIXMEs of the same variety.
-    # if os.path.isdir(web_dir):
+    # if os.path.isdir(image_dir):
     #     print("  removing pre-existing {} before setting version file"
-    #           .format(web_dir))
-    #     shutil.rmtree(web_dir)
-    os.makedirs(web_dir + '/noship')
+    #           .format(imagedir())
+    #     shutil.rmtree(image_dir)
+    os.makedirs(image_dir + '/noship')
     versions_output = []
     with open(out_dir + "/versions", "r") as file:
         versions = set()
@@ -324,16 +431,24 @@ def create_chroots(args, state_dir, bundles, yum_conf):
         versions_output.append("{0: <50}{1}\n".format("Available", "Packages"))
         for name, pver in [line.split(":") for line in versions]:
             versions_output.append("{0: <50}{1}\n".format(name, pver))
-    with open(web_dir + "versions", "w") as file:
+    with open(image_dir + "versions", "w") as file:
         file.writelines(versions_output)
-    for bundle in os.listdir(bundles):
-        shutil.copyfile(out_dir + "/packages-{}".format(bundle), web_dir + "/noship/packages-{}".format(bundle))
+    bundle_list = os.listdir(bundles)
+    bundle_list = trim_bundles(bundle_list)
+    for bundle in bundle_list:
+        shutil.copyfile(out_dir + "/packages-{}".format(bundle), image_dir + "/noship/packages-{}".format(bundle))
         if os.path.isfile(out_dir + "/{}-includes".format(bundle)):
-            shutil.copyfile(out_dir + "/{}-includes".format(bundle), web_dir + "/noship/{}-includes".format(bundle))
+            shutil.copyfile(out_dir + "/{}-includes".format(bundle), image_dir + "/noship/{}-includes".format(bundle))
     for package_name in package_mapping.keys():
         shutil.copyfile(out_dir + "/files-{}".format(package_name),
-                        web_dir + "/noship/files-{}".format(package_name))
+                        image_dir + "/noship/files-{}".format(package_name))
 
+# Remove bundles with blacklisted characters, such as dot files
+def trim_bundles(bundles):
+    for bundle in bundles:
+        if bundle.startswith('.'):
+            bundles.remove(bundle)
+    return bundles
 
 def main():
     """Entry point for chroot creator"""

configure.ac

@@ -1,5 +1,5 @@
 AC_PREREQ([2.68])
-AC_INIT([bundle-chroot-builder],[1.00],[tudor.marcu@intel.com],[bundle-chroot-builder])
+AC_INIT([bundle-chroot-builder],[1.15],[tudor.marcu@intel.com],[bundle-chroot-builder])
 AM_INIT_AUTOMAKE([foreign silent-rules color-tests no-dist-gzip dist-xz])
 AC_CONFIG_FILES(Makefile)
 AC_PREFIX_DEFAULT(/usr/local)