mirror of
https://github.com/clearlinux/common.git
synced 2026-07-14 00:35:53 +00:00
Compare commits
41 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 59ea0b102e | |||
| e4c604a161 | |||
| 59c4520f37 | |||
| 16f813a390 | |||
| 79eebc6aa6 | |||
| d1accc8077 | |||
| b23d156f2c | |||
| 39e8d4e68a | |||
| 9d440aff3a | |||
| c1e426713a | |||
| d9e4f478eb | |||
| 96b366e71f | |||
| 84097fd526 | |||
| c2cd51b5f2 | |||
| ed1ef58d9a | |||
| 497f2a66f4 | |||
| 3b8d3c66ea | |||
| 373cbad44c | |||
| 0fcc984675 | |||
| 78023bd74a | |||
| d2b5b72346 | |||
| 4953d9e087 | |||
| ee37765ed2 | |||
| 492abf6d32 | |||
| 2562a8c7fb | |||
| 5fd02ece36 | |||
| 039e63fb1e | |||
| dcf8bc4978 | |||
| 3951710117 | |||
| 504d77bc58 | |||
| d3514ae496 | |||
| 4403e9280a | |||
| 0aed61be35 | |||
| cdda6bcece | |||
| 61203724ab | |||
| 3dfbdf4c58 | |||
| e0e5b4b1a7 | |||
| b95dd8c67f | |||
| 8ed4d2d0de | |||
| 023a2da624 | |||
| cdd4aeebe2 |
+19
-25
@@ -157,10 +157,6 @@ pullrebase:
|
|||||||
fi \
|
fi \
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# help vendor: Attempt to create an updated vendor tar archive if needed
|
|
||||||
vendor: preautospec-checks pullrebase clean-old-content
|
|
||||||
@ $(TOPLVL)/projects/common/vendor.py $(firstword $(NEWURL) $(URL)) $(PKG_NAME) $(VND_BASE_URL)
|
|
||||||
|
|
||||||
preautospec-checks:
|
preautospec-checks:
|
||||||
|
|
||||||
#help autospec: automatically generates a specfile. If there is
|
#help autospec: automatically generates a specfile. If there is
|
||||||
@@ -174,8 +170,8 @@ autospec: preautospec-checks pullrebase localreponotice clean-old-content
|
|||||||
echo "Specfile already exists and was not created by autospec.py! Aborting."; \
|
echo "Specfile already exists and was not created by autospec.py! Aborting."; \
|
||||||
exit 1; \
|
exit 1; \
|
||||||
fi
|
fi
|
||||||
|
$(eval ARCHIVES = $(shell $(TOPLVL)/projects/common/vendor.py '$(value ARCHIVES)' $(firstword $(NEWURL) $(URL)) $(PKG_NAME) $(VND_BASE_URL)))
|
||||||
@printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(PKG_NAME) '$(firstword $(value NEWURL) $(value URL))' '$(value ARCHIVES)' > Makefile
|
@printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(PKG_NAME) '$(firstword $(value NEWURL) $(value URL))' '$(value ARCHIVES)' > Makefile
|
||||||
@$(MAKE) vendor
|
|
||||||
python3 $(TOPLVL)/projects/autospec/autospec/autospec.py \
|
python3 $(TOPLVL)/projects/autospec/autospec/autospec.py \
|
||||||
--target . \
|
--target . \
|
||||||
--integrity \
|
--integrity \
|
||||||
@@ -391,6 +387,17 @@ koji-waitrepo: kojidef
|
|||||||
koji wait-repo --build=${SRPMVERS} ${WAIT_OPTS} ${KOJI_TAG}-build; \
|
koji wait-repo --build=${SRPMVERS} ${WAIT_OPTS} ${KOJI_TAG}-build; \
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
.PHONY: koji-tag
|
||||||
|
#help koji-tag: runs koji tag-pkg on what's in the current specfile's nvr
|
||||||
|
koji-tag:
|
||||||
|
koji tag-pkg dist-clear $(SRPMVERS)
|
||||||
|
|
||||||
|
.PHONY: koji-untag
|
||||||
|
#help koji-untag: runs koji untag-pkg on what's in the current specfile's nvr
|
||||||
|
koji-untag:
|
||||||
|
koji untag-pkg dist-clear $(SRPMVERS)
|
||||||
|
|
||||||
|
|
||||||
update-versions:
|
update-versions:
|
||||||
|
|
||||||
.PHONY: update
|
.PHONY: update
|
||||||
@@ -400,16 +407,18 @@ update-versions:
|
|||||||
#help If no update.sh exists, update the version, autospec and push the
|
#help If no update.sh exists, update the version, autospec and push the
|
||||||
#help update to koji (bumping the bump.list if found).
|
#help update to koji (bumping the bump.list if found).
|
||||||
update:
|
update:
|
||||||
|
@if [ -f update.first ]; then \
|
||||||
|
for pkg in $$(< update.first); do \
|
||||||
|
(cd "../$${pkg}" || exit 1; $(MAKE) update-versions || exit 0; $(MAKE) update) || exit; \
|
||||||
|
done; \
|
||||||
|
fi
|
||||||
@if [ -f update.sh ]; then \
|
@if [ -f update.sh ]; then \
|
||||||
./update.sh; \
|
./update.sh; \
|
||||||
else \
|
else \
|
||||||
$(MAKE) -s update-versions && \
|
$(MAKE) -s update-versions && \
|
||||||
$(MAKE) autospec CLEANUP=1 && \
|
$(MAKE) autospec CLEANUP=1 && \
|
||||||
if [ -f bump.list ]; then \
|
$(MAKE) -s koji && \
|
||||||
$(MAKE) -s koji; \
|
$(MAKE) koji-waitrepo; \
|
||||||
else \
|
|
||||||
$(MAKE) -s koji-nowait; \
|
|
||||||
fi \
|
|
||||||
fi \
|
fi \
|
||||||
|
|
||||||
logcheck:
|
logcheck:
|
||||||
@@ -433,21 +442,6 @@ repoadd: $(TOPLVL)/repo
|
|||||||
$(MAKE) -s localrepocreate
|
$(MAKE) -s localrepocreate
|
||||||
$(MAKE) -s localrepoenable
|
$(MAKE) -s localrepoenable
|
||||||
|
|
||||||
#help repostage: Stages locally-built RPMs for this package to the local RPM
|
|
||||||
#help directory. If this directory does not yet exist, it is created and
|
|
||||||
#help enabled.
|
|
||||||
repostage: $(TOPLVL)/repo
|
|
||||||
@if [ -z "${LATEST_RPMS}" ]; then \
|
|
||||||
echo "No rpms found in rpms/ directory."; \
|
|
||||||
exit 1; \
|
|
||||||
fi
|
|
||||||
$(MAKE) -s repodel NO_CREATEREPO=1
|
|
||||||
@for r in ${LATEST_RPMS}; do \
|
|
||||||
echo "+$${rpm#rpms/}"; \
|
|
||||||
echo "$${rpm#rpms/}" >> .repo-index; \
|
|
||||||
done; \
|
|
||||||
flock $</repo.lock ln -f -t $< ${LATEST_RPMS}
|
|
||||||
|
|
||||||
#help repodel: Removes RPMs from the local RPM repository that were
|
#help repodel: Removes RPMs from the local RPM repository that were
|
||||||
#help previously added by 'make repoadd' for this package.
|
#help previously added by 'make repoadd' for this package.
|
||||||
repodel: $(TOPLVL)/repo
|
repodel: $(TOPLVL)/repo
|
||||||
|
|||||||
+1
-46
@@ -237,7 +237,7 @@ autospecnew: preautospecnew-checks localreponotice
|
|||||||
--mock-config $(MOCK_CONFIG_VAL) \
|
--mock-config $(MOCK_CONFIG_VAL) \
|
||||||
--mock-opts="$(MOCK_OPTS)" \
|
--mock-opts="$(MOCK_OPTS)" \
|
||||||
$${SETVERSION:+ --version $${SETVERSION}} \
|
$${SETVERSION:+ --version $${SETVERSION}} \
|
||||||
${NON_INTERACTIVE} ${SKIP_GIT} ${CLEANUP} \
|
${NON_INTERACTIVE} ${SKIP_GIT} $(if $(CLEANUP),-C) \
|
||||||
$(URL); \
|
$(URL); \
|
||||||
if [ $$? -eq 0 ]; then \
|
if [ $$? -eq 0 ]; then \
|
||||||
$(MAKE) link-new-rpms PKG_REPO_DIR="${TOPLVL}/packages/${NAME}"; \
|
$(MAKE) link-new-rpms PKG_REPO_DIR="${TOPLVL}/packages/${NAME}"; \
|
||||||
@@ -256,51 +256,6 @@ autospecnew: preautospecnew-checks localreponotice
|
|||||||
exit 1; \
|
exit 1; \
|
||||||
fi
|
fi
|
||||||
|
|
||||||
#help autospecnewgo: Creates a new autospec package with (dependencies)
|
|
||||||
#help for a given URL=$(URL) with NAME=$(NAME). Several files used by
|
|
||||||
#help autospec will be created in the help process.
|
|
||||||
#help Use MOCK_OPTS environment varible to pass down arbitrary mock options
|
|
||||||
#help to autospec.
|
|
||||||
#help For more information about autospec, see the project page on Github https://github.com/clearlinux/autospec
|
|
||||||
autospecnewgo: preautospecnew-checks localreponotice
|
|
||||||
@if [ -z $(NAME) ] || [ -z $(URL) ]; then \
|
|
||||||
echo "Please specify NAME and URL. The ARCHIVES variable is optional."; \
|
|
||||||
exit 1; \
|
|
||||||
fi
|
|
||||||
-$(MAKE) clone_$(NAME)
|
|
||||||
@if [ ! -d $(TOPLVL)/packages/$(NAME)/.git ]; then \
|
|
||||||
echo "no remote repository found, creating new package repository and running autospec"; \
|
|
||||||
mkdir -p $(TOPLVL)/packages/$(NAME); \
|
|
||||||
( \
|
|
||||||
cd $(TOPLVL)/packages/$(NAME); \
|
|
||||||
git init -b main; \
|
|
||||||
git remote add origin $(PKG_BASE_URL)/$(NAME); \
|
|
||||||
$(call gitoliteurl,packages/$(NAME)); \
|
|
||||||
$(call subjectprefix,$(NAME)); \
|
|
||||||
); \
|
|
||||||
printf 'PKG_NAME := %s\nURL = %s\nARCHIVES = %s\n\ninclude ../common/Makefile.common\n' $(NAME) '$(value URL)' '$(value ARCHIVES)' > $(TOPLVL)/packages/$(NAME)/Makefile; \
|
|
||||||
python3 $(TOPLVL)/projects/common/go-builder.py \
|
|
||||||
--target packages/$(NAME) \
|
|
||||||
--integrity \
|
|
||||||
--config "$(AUTOSPEC_CONF)" \
|
|
||||||
--name $(NAME) \
|
|
||||||
--archives $(ARCHIVES) \
|
|
||||||
--mock-config $(MOCK_CONFIG_VAL) \
|
|
||||||
--mock-opts "$(MOCK_OPTS)" \
|
|
||||||
$(URL); \
|
|
||||||
if [ $$? -eq 0 ]; then \
|
|
||||||
$(MAKE) link-new-rpms PKG_REPO_DIR="${TOPLVL}/packages/${NAME}"; \
|
|
||||||
$(TOPLVL)/projects/common/checkblacklist.sh $(TOPLVL)/projects/common/blacklist $(TOPLVL)/packages/${NAME}/results/*.rpm; \
|
|
||||||
python3 $(TOPLVL)/projects/common/patchfilter.py <(git -C $(TOPLVL)/packages/$(NAME) show) > $(TOPLVL)/packages/$(NAME)/for-review.txt; \
|
|
||||||
python3 $(TOPLVL)/projects/common/patchfilter.py <(git -C $(TOPLVL)/packages/$(NAME) show); \
|
|
||||||
else \
|
|
||||||
exit 1; \
|
|
||||||
fi; \
|
|
||||||
else \
|
|
||||||
echo "$(NAME) already exists at $(TOPLVL)/packages/$(NAME)"; \
|
|
||||||
exit 1; \
|
|
||||||
fi
|
|
||||||
|
|
||||||
#help repoenable: Enables the local RPM repository for use with Yum/DNF and
|
#help repoenable: Enables the local RPM repository for use with Yum/DNF and
|
||||||
#help Mock. If this repository does not yet exist, it is created.
|
#help Mock. If this repository does not yet exist, it is created.
|
||||||
repoenable: localrepoenable ;
|
repoenable: localrepoenable ;
|
||||||
|
|||||||
@@ -2,4 +2,3 @@
|
|||||||
/usr/lib/systemd/system/sysinit.target.wants/systemd-firstboot.service
|
/usr/lib/systemd/system/sysinit.target.wants/systemd-firstboot.service
|
||||||
/usr/lib/systemd/system/sysinit.target.wants/systemd-hwdb-update.service
|
/usr/lib/systemd/system/sysinit.target.wants/systemd-hwdb-update.service
|
||||||
/usr/lib/systemd/system/sysinit.target.wants/systemd-update-done.service
|
/usr/lib/systemd/system/sysinit.target.wants/systemd-update-done.service
|
||||||
/usr/lib/systemd/system/update-triggers.target.wants
|
|
||||||
|
|||||||
-285
@@ -1,285 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
#
|
|
||||||
# gowrap.py - part of autospec
|
|
||||||
# Copyright (C) 2019 Intel Corporation
|
|
||||||
#
|
|
||||||
# This program is free software: you can redistribute it and/or modify
|
|
||||||
# it under the terms of the GNU General Public License as published by
|
|
||||||
# the Free Software Foundation, either version 3 of the License, or
|
|
||||||
# (at your option) any later version.
|
|
||||||
#
|
|
||||||
# This program is distributed in the hope that it will be useful,
|
|
||||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
||||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
||||||
# GNU General Public License for more details.
|
|
||||||
#
|
|
||||||
# You should have received a copy of the GNU General Public License
|
|
||||||
# along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import os
|
|
||||||
import shutil
|
|
||||||
import subprocess
|
|
||||||
import sys
|
|
||||||
import tempfile
|
|
||||||
|
|
||||||
|
|
||||||
def parse_args():
|
|
||||||
"""Parse commandline arguments."""
|
|
||||||
parser = argparse.ArgumentParser()
|
|
||||||
parser.add_argument("url", help="Location of package archive")
|
|
||||||
parser.add_argument("-t", "--target", dest="target", action="store",
|
|
||||||
default=None,
|
|
||||||
help="Target location to create or reuse")
|
|
||||||
parser.add_argument("-c", "--config", dest="config", action="store",
|
|
||||||
default="/usr/share/defaults/autospec/autospec.conf",
|
|
||||||
help="Set configuration file to use")
|
|
||||||
parser.add_argument("-n", "--name", action="store", dest="name", default="",
|
|
||||||
help="Override the package name")
|
|
||||||
parser.add_argument("-i", "--integrity", action="store_true",
|
|
||||||
default=False,
|
|
||||||
help="Search for package signature from source URL and "
|
|
||||||
"attempt to verify package")
|
|
||||||
parser.add_argument("-m", "--mock-config", action="store", default="clear",
|
|
||||||
help="Value to pass with Mock's -r option. Defaults to "
|
|
||||||
"\"clear\", meaning that Mock will use "
|
|
||||||
"/etc/mock/clear.cfg.")
|
|
||||||
parser.add_argument("-o", "--mock-opts", action="store", default="",
|
|
||||||
help="Arbitrary options to pass down to mock when "
|
|
||||||
"building a package.")
|
|
||||||
parser.add_argument('-a', "--archives", action="store",
|
|
||||||
dest="archives", default=[], nargs='*',
|
|
||||||
help="tarball URLs for additional source archives and"
|
|
||||||
" a location for the sources to be extacted to (e.g."
|
|
||||||
" http://example.com/downloads/dependency.tar.gz"
|
|
||||||
" /directory/relative/to/extract/root )")
|
|
||||||
|
|
||||||
return parser.parse_args()
|
|
||||||
|
|
||||||
|
|
||||||
def missing_dependencies(name):
|
|
||||||
"""Check if the build failed due to missing dependencies."""
|
|
||||||
root_log = os.path.join("packages", name, "results", "root.log")
|
|
||||||
if not os.path.exists(root_log):
|
|
||||||
return False
|
|
||||||
with open(root_log, "r") as lfile:
|
|
||||||
for line in lfile.readlines():
|
|
||||||
if "No matching package to install" in line:
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def already_built(name, version):
|
|
||||||
"""Check if the package and version are already built."""
|
|
||||||
if not os.path.exists(os.path.join("packages", name, "rpms")):
|
|
||||||
return False
|
|
||||||
vpath = os.path.join("packages", name, "versions")
|
|
||||||
if not os.path.exists(vpath):
|
|
||||||
return False
|
|
||||||
with open(vpath, "r") as vfile:
|
|
||||||
if version not in [x.strip() for x in vfile.readlines()]:
|
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
|
|
||||||
def build(path, args, name, url, version):
|
|
||||||
"""Try and build a package."""
|
|
||||||
if version:
|
|
||||||
ver_arg = ["-v", version]
|
|
||||||
else:
|
|
||||||
ver_arg = []
|
|
||||||
if already_built(name, version):
|
|
||||||
return True
|
|
||||||
print(f"Trying to autospec {name} - {version}")
|
|
||||||
|
|
||||||
proc = subprocess.run(["python3",
|
|
||||||
f"{path}/../autospec/autospec/autospec.py", url,
|
|
||||||
"-c", args.config, "-t", f"packages/{name}",
|
|
||||||
"-n", name, "-m", args.mock_config,
|
|
||||||
"-o", args.mock_opts, "-i"] + ver_arg, capture_output=True)
|
|
||||||
if proc.returncode == 0:
|
|
||||||
subprocess.run(["make", "link-new-rpms", f"PKG_REPO_DIR=packages/{name}"],
|
|
||||||
capture_output=True)
|
|
||||||
return proc.returncode == 0
|
|
||||||
|
|
||||||
|
|
||||||
def parse_go_mod(path):
|
|
||||||
"""Parse go.mod file for build requirements.
|
|
||||||
|
|
||||||
File content looks as follows:
|
|
||||||
|
|
||||||
module example.com/foo/bar
|
|
||||||
|
|
||||||
require (
|
|
||||||
github.com/BurntSushi/toml v0.3.1
|
|
||||||
git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999
|
|
||||||
github.com/inconshreveable/mousetrap v1.0.0 // indirect
|
|
||||||
"github.com/spf13/cobra" v0.0.3
|
|
||||||
github.com/spf13/pflag v1.0.3 // indirect
|
|
||||||
)
|
|
||||||
|
|
||||||
Need to handle all require lines including //indirect.
|
|
||||||
Skip requires that use .git for now. May need to be handled
|
|
||||||
differently.
|
|
||||||
"""
|
|
||||||
reqs = []
|
|
||||||
with open(path, "r") as gfile:
|
|
||||||
dep_start = False
|
|
||||||
for line in gfile.readlines():
|
|
||||||
# Ideally the mod file is generated and the format is
|
|
||||||
# always correct but add a few defenses just in case
|
|
||||||
line = line.strip()
|
|
||||||
if line.startswith("//"):
|
|
||||||
# Skip comments
|
|
||||||
continue
|
|
||||||
if dep_start:
|
|
||||||
# End of the require section
|
|
||||||
if line.startswith(")"):
|
|
||||||
break
|
|
||||||
req = line.split()[:2]
|
|
||||||
req[0] = req[0].replace('"', '')
|
|
||||||
if req[0].endswith(".git"):
|
|
||||||
continue
|
|
||||||
reqs.append(req)
|
|
||||||
continue
|
|
||||||
if line.startswith("require ("):
|
|
||||||
dep_start = True
|
|
||||||
return reqs
|
|
||||||
|
|
||||||
|
|
||||||
def get_dependencies(name):
|
|
||||||
"""Return path to the go.mod file if it exists."""
|
|
||||||
command = None
|
|
||||||
path = None
|
|
||||||
pdir = os.path.join("packages", name)
|
|
||||||
reqs = []
|
|
||||||
for fname in os.listdir(pdir):
|
|
||||||
# Find the archive
|
|
||||||
if fname.endswith("zip"):
|
|
||||||
command = ["unzip", fname]
|
|
||||||
elif ".tar." in fname:
|
|
||||||
command = ["tar", "xf", fname]
|
|
||||||
else:
|
|
||||||
command = None
|
|
||||||
|
|
||||||
if command:
|
|
||||||
# Decompress archive
|
|
||||||
tdir = tempfile.mkdtemp()
|
|
||||||
tfile = os.path.join(tdir, fname)
|
|
||||||
shutil.copyfile(os.path.join(pdir, fname), tfile)
|
|
||||||
proc = subprocess.run(command, cwd=tdir, capture_output=True)
|
|
||||||
if not proc.returncode:
|
|
||||||
# inspect contents for a "go.sum" file
|
|
||||||
for root, _, files in os.walk(tdir):
|
|
||||||
if "go.mod" in files:
|
|
||||||
path = f"{name}.gomod"
|
|
||||||
shutil.copyfile(os.path.join(root, "go.mod"), path)
|
|
||||||
reqs += parse_go_mod(path)
|
|
||||||
os.unlink(path)
|
|
||||||
shutil.rmtree(tdir)
|
|
||||||
|
|
||||||
return reqs
|
|
||||||
|
|
||||||
|
|
||||||
def encode_requirements(requirement):
|
|
||||||
"""Encode the project's module and version."""
|
|
||||||
# Default dependency base proxy url
|
|
||||||
base_url = "https://proxy.golang.org/"
|
|
||||||
# Encoding based on
|
|
||||||
# https://tip.golang.org/cmd/go/#hdr-Module_proxy_protocol
|
|
||||||
encoded_module = ""
|
|
||||||
for char in requirement[0]:
|
|
||||||
if char.isupper():
|
|
||||||
encoded_module += "!" + char.lower()
|
|
||||||
else:
|
|
||||||
encoded_module += char
|
|
||||||
encoded_version = ""
|
|
||||||
for char in requirement[1]:
|
|
||||||
if char.isupper():
|
|
||||||
encoded_version += "!" + char.lower()
|
|
||||||
else:
|
|
||||||
encoded_version += char
|
|
||||||
return (encoded_module, encoded_version)
|
|
||||||
|
|
||||||
|
|
||||||
def initialize_package(name, url, version, path, args):
|
|
||||||
"""Setup package and try to build it."""
|
|
||||||
if already_built(name, version):
|
|
||||||
return True
|
|
||||||
print(f"Trying to autospecnew {name} - {version}")
|
|
||||||
|
|
||||||
proc = subprocess.run(["make", "autospecnew", f"URL={url}", f"NAME={name}", f"SETVERSION={version}"], capture_output=True)
|
|
||||||
if f"{name} already exists at" in proc.stdout.decode("utf-8"):
|
|
||||||
return build(path, args, name, url, version)
|
|
||||||
return proc.returncode == 0
|
|
||||||
|
|
||||||
|
|
||||||
def build_recursive(path, args, name, url, version=None, success=None):
|
|
||||||
"""Try and recursively build packages."""
|
|
||||||
if not success:
|
|
||||||
# First package being built, common utilities already run
|
|
||||||
# so just use autospec directly
|
|
||||||
success = {}
|
|
||||||
ret = build(path, args, name, url, version)
|
|
||||||
else:
|
|
||||||
ret = initialize_package(name, url, version, path, args)
|
|
||||||
|
|
||||||
if ret:
|
|
||||||
# Completed so we are done!
|
|
||||||
return True
|
|
||||||
|
|
||||||
if not missing_dependencies(name):
|
|
||||||
# Failed but not due to missing dependencies, bail
|
|
||||||
print(f"Unknown error building {name} - {version}")
|
|
||||||
return False
|
|
||||||
|
|
||||||
reqs = get_dependencies(name)
|
|
||||||
if not reqs:
|
|
||||||
# Didn't try and add anything so don't need to build again
|
|
||||||
print(f"Build failed with no missing requirements: {name} - {version}")
|
|
||||||
return False
|
|
||||||
|
|
||||||
print(f"First attempt to build failed due to missing dependencies {name} - {version}\n")
|
|
||||||
for req in reqs:
|
|
||||||
ereq = encode_requirements(req)
|
|
||||||
rname = "go-" + req[0].replace("/", "-")
|
|
||||||
rurl = f"https://proxy.golang.org/{ereq[0]}/@v/list"
|
|
||||||
if (rname, req[1]) in success:
|
|
||||||
# Rebuild detected, figure out if it is okay or not
|
|
||||||
if success[(rname, req[1])]:
|
|
||||||
# Already succeeded building req short circuit success
|
|
||||||
continue
|
|
||||||
else:
|
|
||||||
# Encountered a build loop, bail out as this likely
|
|
||||||
# requires manual fixing
|
|
||||||
print(f"Detected build loop when building {rname} - {req[1]}")
|
|
||||||
return True
|
|
||||||
success[(rname, req[1])] = False
|
|
||||||
if not build_recursive(path, args, rname, rurl, req[1], success):
|
|
||||||
print(f"Failed to build dependency {rname} - {req[1]}")
|
|
||||||
return False
|
|
||||||
|
|
||||||
success[(rname, req[1])] = True
|
|
||||||
subprocess.run(["make", "repodel"], cwd=f"packages/{rname}", capture_output=True)
|
|
||||||
subprocess.run(["make", "repostage"], cwd=f"packages/{rname}", capture_output=True)
|
|
||||||
|
|
||||||
# Create the repo with the new packages added
|
|
||||||
subprocess.run(["make", "localrepocreate"], capture_output=True)
|
|
||||||
# Retry previously failed build assuming dependencies got added
|
|
||||||
print(f"Rebuilding after resolving dependencies {name} - {version}")
|
|
||||||
ret = build(path, args, name, url, version)
|
|
||||||
print("")
|
|
||||||
|
|
||||||
return ret
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
"""Start program execution."""
|
|
||||||
args = parse_args()
|
|
||||||
path = os.path.dirname(os.path.realpath(__file__))
|
|
||||||
return build_recursive(path, args, args.name, args.url)
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
if not main():
|
|
||||||
sys.exit(-1)
|
|
||||||
print("\n\nBuild completed\n\n")
|
|
||||||
+6
-2
@@ -98,9 +98,13 @@ def zap_entire_file_end(filename):
|
|||||||
global files
|
global files
|
||||||
global files_chunks
|
global files_chunks
|
||||||
global files_header
|
global files_header
|
||||||
|
files_to_remove = list()
|
||||||
for file in files:
|
for file in files:
|
||||||
if file.endswith(filename):
|
if file.endswith(filename):
|
||||||
files.remove(file)
|
# Don't modify the original list while iterating over it
|
||||||
|
files_to_remove.append(file)
|
||||||
|
for file in files_to_remove:
|
||||||
|
files.remove(file)
|
||||||
|
|
||||||
def zap_line_in_file_substring(filename, match):
|
def zap_line_in_file_substring(filename, match):
|
||||||
global header
|
global header
|
||||||
@@ -186,7 +190,7 @@ def zap_empty_chunks():
|
|||||||
files_to_remove = list()
|
files_to_remove = list()
|
||||||
for file in files:
|
for file in files:
|
||||||
if file not in files_chunks:
|
if file not in files_chunks:
|
||||||
files.remove(file)
|
files_to_remove.append(file)
|
||||||
continue
|
continue
|
||||||
to_remove = list()
|
to_remove = list()
|
||||||
for chunk in files_chunks[file]:
|
for chunk in files_chunks[file]:
|
||||||
|
|||||||
@@ -9,9 +9,8 @@ import subprocess
|
|||||||
import tempfile
|
import tempfile
|
||||||
import time
|
import time
|
||||||
|
|
||||||
from git import Repo
|
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
from git import Repo
|
||||||
|
|
||||||
|
|
||||||
def vendor_check():
|
def vendor_check():
|
||||||
@@ -30,6 +29,7 @@ def vendor_check():
|
|||||||
|
|
||||||
def get_args():
|
def get_args():
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument('archives')
|
||||||
parser.add_argument('url')
|
parser.add_argument('url')
|
||||||
parser.add_argument('name')
|
parser.add_argument('name')
|
||||||
parser.add_argument('git')
|
parser.add_argument('git')
|
||||||
@@ -44,7 +44,7 @@ def setup_content(url):
|
|||||||
with open(outfile, 'wb') as cfile:
|
with open(outfile, 'wb') as cfile:
|
||||||
cfile.write(response.content)
|
cfile.write(response.content)
|
||||||
|
|
||||||
subprocess.run(f"tar xf {outfile}", shell=True, cwd=tdir, check=True)
|
subprocess.run(f"tar xf {outfile}", shell=True, cwd=tdir, check=True, stdout=subprocess.DEVNULL)
|
||||||
os.remove(outfile)
|
os.remove(outfile)
|
||||||
return tdir
|
return tdir
|
||||||
|
|
||||||
@@ -60,69 +60,103 @@ def setup_cargo_vendor(path):
|
|||||||
def update_cargo_vendor(path, name, git):
|
def update_cargo_vendor(path, name, git):
|
||||||
git_uri = os.path.join(git, name)
|
git_uri = os.path.join(git, name)
|
||||||
vendor_path = os.path.join(path, 'vendor')
|
vendor_path = os.path.join(path, 'vendor')
|
||||||
subprocess.run(f"git clone {git_uri} {vendor_path}", shell=True, check=True)
|
subprocess.run(f"git clone {git_uri} {vendor_path}", shell=True, check=True,
|
||||||
|
stdout=subprocess.DEVNULL)
|
||||||
vendor_git = os.path.join(vendor_path, '.git')
|
vendor_git = os.path.join(vendor_path, '.git')
|
||||||
if not os.path.isdir(vendor_git):
|
if not os.path.isdir(vendor_git):
|
||||||
# initialize a git repo
|
# initialize a git repo
|
||||||
subprocess.run('git init .', cwd=vendor_path, shell=True, check=True)
|
subprocess.run('git init .', cwd=vendor_path, shell=True, check=True,
|
||||||
|
stdout=subprocess.DEVNULL)
|
||||||
subprocess.run(f"git remote add origin {git_uri}", cwd=vendor_path,
|
subprocess.run(f"git remote add origin {git_uri}", cwd=vendor_path,
|
||||||
shell=True, check=True)
|
shell=True, check=True, stdout=subprocess.DEVNULL)
|
||||||
backup_vendor_git = os.path.join(path, 'clear-linux-vendor-git')
|
backup_vendor_git = os.path.join(path, 'clear-linux-vendor-git')
|
||||||
subprocess.run(f"cp -a {vendor_git} {backup_vendor_git}", cwd=path,
|
subprocess.run(f"cp -a {vendor_git} {backup_vendor_git}", cwd=path,
|
||||||
shell=True, check=True)
|
shell=True, check=True, stdout=subprocess.DEVNULL)
|
||||||
shutil.rmtree(vendor_path)
|
shutil.rmtree(vendor_path)
|
||||||
subprocess.run('cargo vendor', cwd=path, shell=True, check=True)
|
cargo_vendors = subprocess.run('cargo vendor', cwd=path, shell=True,
|
||||||
|
check=True, stdout=subprocess.PIPE,
|
||||||
|
universal_newlines=True).stdout
|
||||||
subprocess.run(f"cp -a {backup_vendor_git} {vendor_git}", cwd=path,
|
subprocess.run(f"cp -a {backup_vendor_git} {vendor_git}", cwd=path,
|
||||||
shell=True, check=True)
|
shell=True, check=True, stdout=subprocess.DEVNULL)
|
||||||
repo = Repo(vendor_path)
|
repo = Repo(vendor_path)
|
||||||
if not (len(repo.untracked_files) > 0 or repo.is_dirty()):
|
if not (len(repo.untracked_files) > 0 or repo.is_dirty()):
|
||||||
return False
|
return False, ""
|
||||||
subprocess.run('git add .', cwd=vendor_path, shell=True, check=True)
|
subprocess.run('git add .', cwd=vendor_path, shell=True, check=True,
|
||||||
|
stdout=subprocess.DEVNULL)
|
||||||
subprocess.run('git commit -m "vendor update"', cwd=vendor_path,
|
subprocess.run('git commit -m "vendor update"', cwd=vendor_path,
|
||||||
shell=True, check=True)
|
shell=True, check=True, stdout=subprocess.DEVNULL)
|
||||||
gmt = time.gmtime()
|
gmt = time.gmtime()
|
||||||
tag = f"{gmt.tm_year}-{gmt.tm_mon:02d}-{gmt.tm_mday:02d}-{gmt.tm_hour:02d}-{gmt.tm_min:02d}-{gmt.tm_sec:02d}"
|
tag = f"{gmt.tm_year}-{gmt.tm_mon:02d}-{gmt.tm_mday:02d}-{gmt.tm_hour:02d}-{gmt.tm_min:02d}-{gmt.tm_sec:02d}"
|
||||||
subprocess.run(f"git tag {tag}", cwd=vendor_path, shell=True,
|
subprocess.run(f"git tag {tag}", cwd=vendor_path, shell=True,
|
||||||
check=True)
|
check=True, stdout=subprocess.DEVNULL)
|
||||||
subprocess.run(f"git push origin main:main {tag}", cwd=vendor_path,
|
subprocess.run(f"git push origin main:main {tag}", cwd=vendor_path,
|
||||||
shell=True, check=True)
|
shell=True, check=True, stdout=subprocess.DEVNULL)
|
||||||
time.sleep(30)
|
time.sleep(30)
|
||||||
return tag
|
return tag, cargo_vendors
|
||||||
|
|
||||||
|
|
||||||
def update_cargo_sources(name, tag):
|
def update_cargo_sources(name, tag, cargo_vendors):
|
||||||
makefile = []
|
makefile = []
|
||||||
archive_match = os.path.join('$(CGIT_BASE_URL)', 'vendor', name,
|
options = []
|
||||||
|
archive_match = os.path.join(r'\$\(CGIT_BASE_URL\)', 'vendor', name,
|
||||||
'snapshot', name)
|
'snapshot', name)
|
||||||
|
archive_replace = os.path.join('$(CGIT_BASE_URL)', 'vendor', name,
|
||||||
|
'snapshot', name)
|
||||||
with open('Makefile', encoding='utf8') as mfile:
|
with open('Makefile', encoding='utf8') as mfile:
|
||||||
for line in mfile.readlines():
|
for line in mfile.readlines():
|
||||||
if line.startswith('ARCHIVES'):
|
if line.startswith('ARCHIVES'):
|
||||||
if re.match(archive_match + '[a-zA-Z0-9_\-.]+\.tar\.xz', line):
|
if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
|
||||||
new_archives = re.sub(archive_match + '[a-zA-Z0-9_\-.]+\.tar\.xz',
|
new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz',
|
||||||
f"{archive_match}-{tag}.tar.xz\n", line)
|
f"{archive_replace}-{tag}.tar.xz", line)
|
||||||
else:
|
else:
|
||||||
new_archives = f"ARCHIVES = {archive_match}-{tag}.tar.xz ./vendor\n"
|
new_archives = f"{line[:-1]} {archive_replace}-{tag}.tar.xz ./vendor\n"
|
||||||
|
print(new_archives.replace('ARCHIVES = ', '', 1))
|
||||||
makefile.append(new_archives)
|
makefile.append(new_archives)
|
||||||
else:
|
else:
|
||||||
makefile.append(line)
|
makefile.append(line)
|
||||||
with open('Makefile', 'w', encoding='utf8') as mfile:
|
with open('Makefile', 'w', encoding='utf8') as mfile:
|
||||||
mfile.writelines(makefile)
|
mfile.writelines(makefile)
|
||||||
|
|
||||||
|
archive_match = os.path.join('http://localhost', 'cgit', 'vendor', name,
|
||||||
|
'snapshot', name)
|
||||||
|
with open('options.conf', encoding='utf8') as ofile:
|
||||||
|
for line in ofile.readlines():
|
||||||
|
if line.startswith('archives'):
|
||||||
|
if re.search(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz', line):
|
||||||
|
new_archives = re.sub(archive_match + r'[a-zA-Z0-9_\-.]+\.tar\.xz',
|
||||||
|
f"{archive_match}-{tag}.tar.xz", line)
|
||||||
|
else:
|
||||||
|
new_archives = f"{line[:-1]} {archive_match}-{tag}.tar.xz ./vendor\n"
|
||||||
|
options.append(new_archives)
|
||||||
|
else:
|
||||||
|
options.append(line)
|
||||||
|
with open('options.conf', 'w', encoding='utf8') as ofile:
|
||||||
|
ofile.writelines(options)
|
||||||
|
with open('cargo_vendors', 'w', encoding='utf8') as cfile:
|
||||||
|
cfile.write(cargo_vendors)
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
|
updated = False
|
||||||
|
args = get_args()
|
||||||
|
|
||||||
vtype = vendor_check()
|
vtype = vendor_check()
|
||||||
if not vtype:
|
if not vtype:
|
||||||
|
print(args.archives)
|
||||||
return
|
return
|
||||||
|
|
||||||
args = get_args()
|
|
||||||
tdir = setup_content(args.url)
|
tdir = setup_content(args.url)
|
||||||
if vtype == 'cargo':
|
if vtype == 'cargo':
|
||||||
vdir = setup_cargo_vendor(tdir)
|
vdir = setup_cargo_vendor(tdir)
|
||||||
if vdir:
|
if vdir:
|
||||||
tag = update_cargo_vendor(vdir, args.name, args.git)
|
tag, cargo_vendors = update_cargo_vendor(vdir, args.name, args.git)
|
||||||
if tag:
|
if tag:
|
||||||
update_cargo_sources(args.name, tag)
|
update_cargo_sources(args.name, tag, cargo_vendors)
|
||||||
|
updated = True
|
||||||
|
if not updated:
|
||||||
|
print(args.archives)
|
||||||
shutil.rmtree(tdir)
|
shutil.rmtree(tdir)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
main()
|
main()
|
||||||
|
|||||||
Reference in New Issue
Block a user