update packages

Changes vendor from a target (it could still have a stand-alone target
but it wouldn't be used by the autospec target).

This change is needed because the Makefile is rewritten in the
autospec target but the ARCHIVES make variable is going to use the
value when make is executed. The common change causes the ARCHIVES
value to be eval'd during the autospec target execution in order for
the update from the vendor script to be seen by make.

The vendor script changes fix bugs in updating the options.conf and
Makefile. It also now handles returning the original ARCHIVES value as
well as prevent subshell scripts from outputting and adding badness
into the eval'd ARCHIVES variable.

Signed-off-by: William Douglas <william.douglas@intel.com>

Makefile.common

@@ -157,10 +157,6 @@ pullrebase:
 		fi \
 	fi
 
-# help vendor: Attempt to create an updated vendor tar archive if needed
-vendor: preautospec-checks pullrebase clean-old-content
-	@ $(TOPLVL)/projects/common/vendor.py $(firstword $(NEWURL) $(URL)) $(PKG_NAME) $(VND_BASE_URL)
-
 preautospec-checks:
 
 #help autospec: automatically generates a specfile. If there is
@@ -174,8 +170,8 @@ autospec: preautospec-checks pullrebase localreponotice clean-old-content
 		echo "Specfile already exists and was not created by autospec.py! Aborting."; \
 		exit 1; \
 	fi
+	$(eval ARCHIVES = $(shell $(TOPLVL)/projects/common/vendor.py '$(value ARCHIVES)' $(firstword $(NEWURL) $(URL)) $(PKG_NAME) $(VND_BASE_URL)))
 	@printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(PKG_NAME) '$(firstword $(value NEWURL) $(value URL))' '$(value ARCHIVES)' > Makefile
-	@$(MAKE) vendor
 	python3 $(TOPLVL)/projects/autospec/autospec/autospec.py \
 		--target . \
 		--integrity \
@@ -391,6 +387,17 @@ koji-waitrepo: kojidef
 		koji wait-repo --build=${SRPMVERS} ${WAIT_OPTS} ${KOJI_TAG}-build; \
 	fi
 
+.PHONY: koji-tag
+#help koji-tag: runs koji tag-pkg on what's in the current specfile's nvr
+koji-tag:
+	koji tag-pkg dist-clear $(SRPMVERS)
+
+.PHONY: koji-untag
+#help koji-untag: runs koji untag-pkg on what's in the current specfile's nvr
+koji-untag:
+	koji untag-pkg dist-clear $(SRPMVERS)
+
+
 update-versions:
 
 .PHONY: update

Makefile.toplevel

@@ -126,11 +126,17 @@ endif
 pull: ${PULL_PKGS}
 
 #help clean-pkgs-dir: For packages that are no longer present in the distro,
-#help removes the associated package repos from the ./packages tree
+#help removes the associated package repos from the ./packages tree and stores
+#help it in packages_old. If it already exists in packages_old, skip it.
 clean-pkgs-dir: $(PACKAGES_FILE)
 	@for p in ${PKGS_ORPHANED}; do \
-		echo "Removing $$p from packages, it is no longer in common/packages."; \
-		rm -rf packages/$$p; \
+		mkdir -p packages_old; \
+		if [ -d "packages_old/$$p" ]; then \
+			echo "packages_old/$$p already exists, not touching it"; \
+		else \
+			echo "Moving $$p from packages to packages_old, as it is no longer in common/packages."; \
+			mv packages/$$p packages_old/$$p; \
+		fi; \
 	done
 	@echo "packages directory cleaned";
 

blacklist

@@ -2,4 +2,3 @@
 /usr/lib/systemd/system/sysinit.target.wants/systemd-firstboot.service
 /usr/lib/systemd/system/sysinit.target.wants/systemd-hwdb-update.service
 /usr/lib/systemd/system/sysinit.target.wants/systemd-update-done.service
-/usr/lib/systemd/system/update-triggers.target.wants

patchfilter.py

@@ -98,9 +98,13 @@ def zap_entire_file_end(filename):
     global files
     global files_chunks
     global files_header
+    files_to_remove = list()
     for file in files:
         if file.endswith(filename):
-            files.remove(file)                
+            # Don't modify the original list while iterating over it
+            files_to_remove.append(file)
+    for file in files_to_remove:
+        files.remove(file)                
             
 def zap_line_in_file_substring(filename, match):
     global header
@@ -186,7 +190,7 @@ def zap_empty_chunks():
     files_to_remove = list()
     for file in files:
         if file not in files_chunks:
-            files.remove(file)
+            files_to_remove.append(file)
             continue
         to_remove = list()
         for chunk in files_chunks[file]:

vendor.py

@@ -30,6 +30,7 @@ def vendor_check():
 
 def get_args():
     parser = argparse.ArgumentParser()
+    parser.add_argument('archives')
     parser.add_argument('url')
     parser.add_argument('name')
     parser.add_argument('git')
@@ -44,7 +45,7 @@ def setup_content(url):
     with open(outfile, 'wb') as cfile:
         cfile.write(response.content)
 
-    subprocess.run(f"tar xf {outfile}", shell=True, cwd=tdir, check=True)
+    subprocess.run(f"tar xf {outfile}", shell=True, cwd=tdir, check=True, stdout=subprocess.DEVNULL)
     os.remove(outfile)
     return tdir
 
@@ -60,61 +61,88 @@ def setup_cargo_vendor(path):
 def update_cargo_vendor(path, name, git):
     git_uri = os.path.join(git, name)
     vendor_path = os.path.join(path, 'vendor')
-    subprocess.run(f"git clone {git_uri} {vendor_path}", shell=True, check=True)
+    subprocess.run(f"git clone {git_uri} {vendor_path}", shell=True, check=True,
+                   stdout=subprocess.DEVNULL)
     vendor_git = os.path.join(vendor_path, '.git')
     if not os.path.isdir(vendor_git):
         # initialize a git repo
-        subprocess.run('git init .', cwd=vendor_path, shell=True, check=True)
+        subprocess.run('git init .', cwd=vendor_path, shell=True, check=True,
+                       stdout=subprocess.DEVNULL)
         subprocess.run(f"git remote add origin {git_uri}", cwd=vendor_path,
-                       shell=True, check=True)
+                       shell=True, check=True, stdout=subprocess.DEVNULL)
     backup_vendor_git = os.path.join(path, 'clear-linux-vendor-git')
     subprocess.run(f"cp -a {vendor_git} {backup_vendor_git}", cwd=path,
-                   shell=True, check=True)
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
     shutil.rmtree(vendor_path)
-    subprocess.run('cargo vendor', cwd=path, shell=True, check=True)
+    subprocess.run('cargo vendor', cwd=path, shell=True, check=True,
+                   stdout=subprocess.DEVNULL)
     subprocess.run(f"cp -a {backup_vendor_git} {vendor_git}", cwd=path,
-                   shell=True, check=True)
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
     repo = Repo(vendor_path)
     if not (len(repo.untracked_files) > 0 or repo.is_dirty()):
         return False
-    subprocess.run('git add .', cwd=vendor_path, shell=True, check=True)
+    subprocess.run('git add .', cwd=vendor_path, shell=True, check=True,
+                   stdout=subprocess.DEVNULL)
     subprocess.run('git commit -m "vendor update"', cwd=vendor_path,
-                   shell=True, check=True)
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
     gmt = time.gmtime()
     tag = f"{gmt.tm_year}-{gmt.tm_mon:02d}-{gmt.tm_mday:02d}-{gmt.tm_hour:02d}-{gmt.tm_min:02d}-{gmt.tm_sec:02d}"
     subprocess.run(f"git tag {tag}", cwd=vendor_path, shell=True,
-                   check=True)
+                   check=True, stdout=subprocess.DEVNULL)
     subprocess.run(f"git push origin main:main {tag}", cwd=vendor_path,
-                   shell=True, check=True)
+                   shell=True, check=True, stdout=subprocess.DEVNULL)
     time.sleep(30)
     return tag
 
 
 def update_cargo_sources(name, tag):
     makefile = []
-    archive_match = os.path.join('$(CGIT_BASE_URL)', 'vendor', name,
+    options = []
+    archive_match = os.path.join(r'\$\(CGIT_BASE_URL\)', 'vendor', name,
                                  'snapshot', name)
+    archive_replace = os.path.join('$(CGIT_BASE_URL)', 'vendor', name,
+                                   'snapshot', name)
     with open('Makefile', encoding='utf8') as mfile:
         for line in mfile.readlines():
             if line.startswith('ARCHIVES'):
-                if re.match(archive_match + '[a-zA-Z0-9_\-.]+\.tar\.xz', line):
-                    new_archives = re.sub(archive_match + '[a-zA-Z0-9_\-.]+\.tar\.xz',
-                                          f"{archive_match}-{tag}.tar.xz\n", line)
+                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
+                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz',
+                                          f"{archive_replace}-{tag}.tar.xz", line)
                 else:
-                    new_archives = f"ARCHIVES = {archive_match}-{tag}.tar.xz ./vendor\n"
+                    new_archives = f"{line[:-1]} {archive_replace}-{tag}.tar.xz ./vendor\n"
+                print(new_archives.replace('ARCHIVES = ', '', 1))
                 makefile.append(new_archives)
             else:
                 makefile.append(line)
     with open('Makefile', 'w', encoding='utf8') as mfile:
         mfile.writelines(makefile)
 
+    archive_match = os.path.join('http://localhost', 'cgit', 'vendor', name,
+                                 'snapshot', name)
+    with open('options.conf', encoding='utf8') as ofile:
+        for line in ofile.readlines():
+            if line.startswith('archives'):
+                if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
+                    new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz',
+                                          f"{archive_match}-{tag}.tar.xz", line)
+                else:
+                    new_archives = f"{line[:-1]} {archive_match}-{tag}.tar.xz ./vendor\n"
+                options.append(new_archives)
+            else:
+                options.append(line)
+    with open('options.conf', 'w', encoding='utf8') as ofile:
+        ofile.writelines(options)
+
 
 def main():
+    updated = False
+    args = get_args()
+
     vtype = vendor_check()
     if not vtype:
+        print(args.archives)
         return
 
-    args = get_args()
     tdir = setup_content(args.url)
     if vtype == 'cargo':
         vdir = setup_cargo_vendor(tdir)
@@ -122,7 +150,11 @@ def main():
             tag = update_cargo_vendor(vdir, args.name, args.git)
             if tag:
                 update_cargo_sources(args.name, tag)
+                updated = True
+    if not updated:
+        print(args.archives)
     shutil.rmtree(tdir)
 
+
 if __name__ == '__main__':
     main()