From a1ac13f79288c8cc57130f4ef8cd84483bdb8cc2 Mon Sep 17 00:00:00 2001 From: William Douglas Date: Mon, 22 Oct 2018 13:46:38 -0700 Subject: [PATCH] Initial koji setup and configuration with ansible on Clear --- inv/hosts | 6 + koji-setup/bootstrap-build.sh | 15 ++ koji-setup/deploy-koji.sh | 350 ++++++++++++++++++++++++++++++++++ koji-setup/gencert.sh | 17 ++ koji-setup/parameters.sh | 11 ++ koji.yaml | 38 ++++ 6 files changed, 437 insertions(+) create mode 100644 inv/hosts create mode 100755 koji-setup/bootstrap-build.sh create mode 100755 koji-setup/deploy-koji.sh create mode 100755 koji-setup/gencert.sh create mode 100644 koji-setup/parameters.sh create mode 100644 koji.yaml diff --git a/inv/hosts b/inv/hosts new file mode 100644 index 0000000..f2800bd --- /dev/null +++ b/inv/hosts @@ -0,0 +1,6 @@ +all: + hosts: + koji-all-in-one: + ansible_port: 2222 + ansible_host: localhost + ansible_user: test diff --git a/koji-setup/bootstrap-build.sh b/koji-setup/bootstrap-build.sh new file mode 100755 index 0000000..b871a99 --- /dev/null +++ b/koji-setup/bootstrap-build.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -xe +SCRIPT_DIR="$(dirname "$(realpath "$0")")" +source "$SCRIPT_DIR"/parameters.sh + +sudo -u kojiadmin koji add-tag dist-"$TAG_NAME" +sudo -u kojiadmin koji add-tag --parent dist-"$TAG_NAME" --arches "x86_64" dist-"$TAG_NAME"-build +sudo -u kojiadmin koji add-group dist-"$TAG_NAME"-build build +sudo -u kojiadmin koji add-group dist-"$TAG_NAME"-build srpm-build +sudo -u kojiadmin koji add-target dist-"$TAG_NAME" dist-"$TAG_NAME"-build +#sudo -u kojiadmin koji add-external-repo -t dist-"$TAG_NAME"-build dist-"$TAG_NAME"-external-repo "$EXTERNAL_REPO" +sudo -u kojiadmin koji taginfo dist-"$TAG_NAME"-build +sudo -u kojiadmin koji add-group-pkg dist-"$TAG_NAME"-build build autoconf automake automake-dev binutils bzip2 clr-rpm-config coreutils diffutils g++ gawk gcc gcc-dev gettext gettext-bin git glibc-dev glibc-locale glibc-utils grep gzip hostname lib6-locale libc6-dev libcap libgcc-s-dev libstdc++-dev libtool libtool-dev linux-libc-headers linux-libc-headers-dev m4 make netbase nss-altfiles patch pigz pkg-config pkg-config-dev rpm-build sed sed-doc shadow systemd-libs tar unzip which x +sudo -u kojiadmin koji add-group-pkg dist-"$TAG_NAME"-build srpm-build coreutils cpio curl-bin git glibc-utils grep gzip make pigz rpm-build sed shadow tar unzip wget xz +sudo -u kojiadmin koji regen-repo dist-"$TAG_NAME"-build diff --git a/koji-setup/deploy-koji.sh b/koji-setup/deploy-koji.sh new file mode 100755 index 0000000..5d4a750 --- /dev/null +++ b/koji-setup/deploy-koji.sh @@ -0,0 +1,350 @@ +#!/usr/bin/env bash +# Copyright (c) 2018 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -xe +SCRIPT_DIR=$(dirname $(realpath "$0")) +source "$SCRIPT_DIR"/parameters.sh + + +## SETTING UP SSL CERTIFICATES FOR AUTHENTICATION +KOJI_PKI_DIR=/etc/pki/koji +mkdir -p "$KOJI_PKI_DIR"/{certs,private} + +# Certificate generation +cat > "$KOJI_PKI_DIR"/ssl.cnf <<- EOF +HOME = $KOJI_PKI_DIR +RANDFILE = $KOJI_PKI_DIR/.rand + +[ca] +default_ca = ca_default + +[ca_default] +dir = $KOJI_PKI_DIR +certs = \$dir/certs +crl_dir = \$dir/crl +database = \$dir/index.txt +new_certs_dir = \$dir/newcerts +certificate = \$dir/%s_ca_cert.pem +private_key = \$dir/private/%s_ca_key.pem +serial = \$dir/serial +crl = \$dir/crl.pem +x509_extensions = usr_cert +name_opt = ca_default +cert_opt = ca_default +default_days = 3650 +default_crl_days = 30 +default_md = sha256 +preserve = no +policy = policy_match + +[policy_match] +countryName = match +stateOrProvinceName = match +organizationName = match +organizationalUnitName = optional +commonName = supplied +emailAddress = optional + +[req] +default_bits = 1024 +default_keyfile = privkey.pem +default_md = sha256 +distinguished_name = req_distinguished_name +attributes = req_attributes +x509_extensions = v3_ca # The extensions to add to the self signed cert +string_mask = MASK:0x2002 + +[req_distinguished_name] +countryName = Country Name (2 letter code) +countryName_min = 2 +countryName_max = 2 +stateOrProvinceName = State or Province Name (full name) +localityName = Locality Name (eg, city) +0.organizationName = Organization Name (eg, company) +organizationalUnitName = Organizational Unit Name (eg, section) +commonName = Common Name (eg, your name or your server\'s hostname) +commonName_max = 64 +emailAddress = Email Address +emailAddress_max = 64 + +[req_attributes] +challengePassword = A challenge password +challengePassword_min = 4 +challengePassword_max = 20 +unstructuredName = An optional company name + +[usr_cert] +basicConstraints = CA:FALSE +nsComment = "OpenSSL Generated Certificate" +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer:always + +[v3_ca] +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always,issuer:always +basicConstraints = CA:true +EOF + +# Generate CA +touch "$KOJI_PKI_DIR"/index.txt +echo 01 > "$KOJI_PKI_DIR"/serial +openssl genrsa -out "$KOJI_PKI_DIR"/private/koji_ca_cert.key 2048 +openssl req -subj "/C=$COUNTRY_CODE/ST=$STATE/L=$LOCATION/O=$ORGANIZATION/OU=koji_ca/CN=$KOJI_MASTER_FQDN" -config "$KOJI_PKI_DIR"/ssl.cnf -new -x509 -days 3650 -key "$KOJI_PKI_DIR"/private/koji_ca_cert.key -out "$KOJI_PKI_DIR"/koji_ca_cert.crt -extensions v3_ca +mkdir -p /etc/ca-certs/trusted +cp -a "$KOJI_PKI_DIR"/koji_ca_cert.crt /etc/ca-certs/trusted +clrtrust generate + +# Generate the koji component certificates and the admin certificate and generate a PKCS12 user certificate (for web browser) +cp "$SCRIPT_DIR"/gencert.sh "$KOJI_PKI_DIR" +pushd "$KOJI_PKI_DIR" +./gencert.sh kojiweb "/C=$COUNTRY_CODE/ST=$STATE/L=$LOCATION/O=$ORGANIZATION/OU=kojiweb/CN=$KOJI_MASTER_FQDN" +./gencert.sh kojihub "/C=$COUNTRY_CODE/ST=$STATE/L=$LOCATION/O=$ORGANIZATION/OU=kojihub/CN=$KOJI_MASTER_FQDN" +./gencert.sh kojiadmin "/C=$COUNTRY_CODE/ST=$STATE/L=$LOCATION/O=$ORGANIZATION/OU=$ORG_UNIT/CN=kojiadmin" +./gencert.sh kojira "/C=$COUNTRY_CODE/ST=$STATE/L=$LOCATION/O=$ORGANIZATION/OU=$ORG_UNIT/CN=kojira" +popd + +# Copy certificates into ~/.koji for kojiadmin +useradd kojiadmin +ADMIN_CERT_DIR=$(getent passwd kojiadmin | cut -d ':' -f 6)/.koji +mkdir -p "$ADMIN_CERT_DIR" +cp -f "$KOJI_PKI_DIR"/kojiadmin.pem "$ADMIN_CERT_DIR"/client.crt +cp -f "$KOJI_PKI_DIR"/koji_ca_cert.crt "$ADMIN_CERT_DIR"/clientca.crt +cp -f "$KOJI_PKI_DIR"/koji_ca_cert.crt "$ADMIN_CERT_DIR"/serverca.crt +chown -R kojiadmin:kojiadmin "$ADMIN_CERT_DIR" + + +## POSTGRESQL SERVER +# Initialize PostgreSQL DB +mkdir -p /var/lib/pgsql +chown -R postgres:postgres /var/lib/pgsql +sudo -u postgres initdb --pgdata /var/lib/pgsql/data +systemctl enable --now postgresql + +# Setup User Accounts +useradd -r koji + +# Setup PostgreSQL and populate schema +sudo -u postgres createuser --no-superuser --no-createrole --no-createdb koji +sudo -u postgres createdb -O koji koji +sudo -u koji psql koji koji < /usr/share/doc/koji*/docs/schema.sql + +# Authorize Koji-web and Koji-hub resources +# TODO: Add authentication with SSL certificates +cat > /var/lib/pgsql/data/pg_hba.conf <<- EOF +#TYPE DATABASE USER CIDR-ADDRESS METHOD +host koji all 127.0.0.1/32 trust +host koji all ::1/128 trust +local koji all trust +EOF + +# Bootstrapping the initial koji admin user into the PostgreSQL database +# SSL Certificate authentication +sudo -u koji psql -c "insert into users (name, status, usertype) values ('kojiadmin', 0, 0);" + +# Give yourself admin permissions +sudo -u koji psql -c "insert into user_perms (user_id, perm_id, creator_id) values (1, 1, 1);" + + +## KOJI CONFIGURATION FILES +# Koji Hub +mkdir -p /etc/koji-hub +cat > /etc/koji-hub/hub.conf <<- EOF +[hub] +DBName = koji +DBUser = koji +KojiDir = $KOJI_DIR +DNUsernameComponent = CN +ProxyDNs = C=$COUNTRY_CODE,ST=$STATE,L=$LOCATION,O=$ORGANIZATION,OU=kojiweb,CN=$KOJI_MASTER_FQDN +LoginCreatesUser = On +KojiWebURL = https://$KOJI_MASTER_FQDN/koji +DisableNotifications = True +EOF + +# Koji Web +mkdir -p /etc/kojiweb +cat > /etc/kojiweb/web.conf <<- EOF +[web] +SiteName = koji +KojiHubURL = https://$KOJI_MASTER_FQDN/kojihub +KojiFilesURL = https://$KOJI_MASTER_FQDN/kojifiles +WebCert = $KOJI_PKI_DIR/kojiweb.pem +ClientCA = $KOJI_PKI_DIR/koji_ca_cert.crt +KojiHubCA = $KOJI_PKI_DIR/koji_ca_cert.crt +LoginTimeout = 72 +Secret = NITRA_IS_NOT_CLEAR +LibPath = /usr/share/koji-web/lib +LiteralFooter = True +EOF + +# Koji CLI +cat > /etc/koji.conf <<- EOF +[koji] +server = https://$KOJI_MASTER_FQDN/kojihub +weburl = https://$KOJI_MASTER_FQDN/koji +topurl = https://$KOJI_MASTER_FQDN/kojifiles +topdir = $KOJI_DIR +cert = ~/.koji/client.crt +ca = ~/.koji/clientca.crt +serverca = ~/.koji/serverca.crt +anon_retry = true +EOF + + +## KOJI APPLICATION HOSTING +# Koji Filesystem Skeleton +mkdir -p "$KOJI_DIR"/{packages,repos,work,scratch,repos-dist} +chown -R httpd:httpd "$KOJI_DIR" + +## Apache Configuration Files +mkdir -p /etc/httpd/conf.d +cat > /etc/httpd/conf.d/koji.conf <<- EOF +LoadModule ssl_module lib/httpd/modules/mod_ssl.so + + + AllowEncodedSlashes NoDecode + RedirectMatch permanent /(.*) https://$KOJI_MASTER_FQDN/$1 + + +Listen 443 + + ServerName $KOJI_MASTER_FQDN + + SSLEngine on + SSLProtocol -all +TLSv1.2 + SSLCipherSuite EECDH+ECDSA+AESGCM:EECDH+aRSA+AESGCM:EECDH+ECDSA+SHA384:EECDH+ECDSA+SHA256:EECDH+aRSA+SHA384:EECDH+aRSA+SHA256:EECDH:EDH+aRSA:HIGH:!aNULL:!eNULL:!LOW:!3DES:!MD5:!EXP:!PSK:!SRP:!DSS:!RC4:!DH:!SHA1 + SSLHonorCipherOrder on + SSLOptions +StrictRequire +StdEnvVars + + SSLCertificateFile $KOJI_PKI_DIR/kojihub.pem + SSLCertificateKeyFile $KOJI_PKI_DIR/private/kojihub.key + SSLCertificateChainFile $KOJI_PKI_DIR/koji_ca_cert.crt + SSLCACertificateFile $KOJI_PKI_DIR/koji_ca_cert.crt + SSLVerifyClient optional + SSLVerifyDepth 10 + + # Koji Hub + Alias /kojihub /usr/share/koji-hub/kojixmlrpc.py + + Options ExecCGI + SetHandler wsgi-script + Require all granted + + Alias /kojifiles "/srv/koji/" + + Options Indexes SymLinksIfOwnerMatch + AllowOverride None + Require all granted + + + SSLVerifyClient require + + + # Koji Web + Alias /koji "/usr/share/koji-web/scripts/wsgi_publisher.py" + + Options ExecCGI + SetHandler wsgi-script + Require all granted + + Alias /koji-static/ "/usr/share/koji-web/static/" + + Options None + AllowOverride None + Require all granted + + +EOF + +## Apache Configuration Files +mkdir -p /etc/httpd/conf.modules.d +cat > /etc/httpd/conf.modules.d/wsgi.conf <<- EOF +LoadModule wsgi_module lib/python2.7/site-packages/mod_wsgi/server/mod_wsgi-py27.so +WSGISocketPrefix /run/httpd/wsgi +EOF + +systemctl enable --now httpd + + +## TEST KOJI CONNECTIVITY +sudo -u kojiadmin koji moshimoshi + + +## KOJI DAEMON - BUILDER +# Add the host entry for the koji builder to the database +sudo -u kojiadmin koji add-host "$KOJI_SLAVE_FQDN" i386 x86_64 + +# Add the host to the createrepo channel +sudo -u kojiadmin koji add-host-to-channel "$KOJI_SLAVE_FQDN" createrepo + +# A note on capacity +sudo -u kojiadmin koji edit-host --capacity=16 "$KOJI_SLAVE_FQDN" + +# Generate certificates +pushd "$KOJI_PKI_DIR" +./gencert.sh "$KOJI_SLAVE_FQDN" "/C=$COUNTRY_CODE/ST=$STATE/L=$LOCATION/O=$ORGANIZATION/CN=$KOJI_SLAVE_FQDN" +popd + +# Create mock folders and permissions +mkdir -p /etc/mock/koji +mkdir -p /var/lib/mock +chown -R root:mock /var/lib/mock + +# Setup User Accounts +useradd -r kojibuilder +usermod -G mock kojibuilder + +# Kojid Configuration Files +mkdir -p /etc/kojid +cat > /etc/kojid/kojid.conf <<- EOF +[kojid] +sleeptime=5 +maxjobs=16 +topdir=$KOJI_DIR +workdir=/tmp/koji +mockdir=/var/lib/mock +mockuser=kojibuilder +mockhost=generic-linux-gnu +user=$KOJI_SLAVE_FQDN +server=https://$KOJI_MASTER_FQDN/kojihub +topurl=https://$KOJI_MASTER_FQDN/kojifiles +use_createrepo_c=True +allowed_scms=$CGIT_FQDN:/packages/* +cert = $KOJI_PKI_DIR/$KOJI_SLAVE_FQDN.pem +ca = $KOJI_PKI_DIR/koji_ca_cert.crt +serverca = $KOJI_PKI_DIR/koji_ca_cert.crt +EOF + +systemctl enable --now kojid + + +## KOJIRA - DNF|YUM REPOSITORY CREATION AND MAINTENANCE +# Add the user entry for the kojira user +sudo -u kojiadmin koji add-user kojira +sudo -u kojiadmin koji grant-permission repo kojira + +# Kojira Configuration Files +mkdir -p /etc/kojira +cat > /etc/kojira/kojira.conf <<- EOF +[kojira] +server=https://$KOJI_MASTER_FQDN/kojihub +topdir=$KOJI_DIR +logfile=/var/log/kojira.log +with_src=no +cert = $KOJI_PKI_DIR/kojira.pem +ca = $KOJI_PKI_DIR/koji_ca_cert.crt +serverca = $KOJI_PKI_DIR/koji_ca_cert.crt +EOF + +systemctl enable --now kojira diff --git a/koji-setup/gencert.sh b/koji-setup/gencert.sh new file mode 100755 index 0000000..646fd8c --- /dev/null +++ b/koji-setup/gencert.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# user is equal to parameter one or the first argument when you actually +# run the script +user=$1 +# subject is equal to parameter two or the second argument when you actually +# run the script +subject="$2" + +openssl genrsa -out private/${user}.key 2048 +if [ -z "$subject" ]; then + openssl req -config ssl.cnf -new -nodes -out certs/${user}.csr -key private/${user}.key +else + openssl req -subj "$subject" -config ssl.cnf -new -nodes -out certs/${user}.csr -key private/${user}.key +fi +openssl ca -batch -config ssl.cnf -keyfile private/koji_ca_cert.key -cert koji_ca_cert.crt -out certs/${user}.crt -outdir certs -infiles certs/${user}.csr +cat certs/${user}.crt private/${user}.key > ${user}.pem +openssl pkcs12 -export -inkey private/${user}.key -in certs/${user}.crt -CAfile koji_ca_cert.crt -out certs/${user}_browser_cert.p12 -passout pass: diff --git a/koji-setup/parameters.sh b/koji-setup/parameters.sh new file mode 100644 index 0000000..3c99e27 --- /dev/null +++ b/koji-setup/parameters.sh @@ -0,0 +1,11 @@ +export COUNTRY_CODE=US +export STATE=Oregon +export LOCATION=Hillsboro +export ORGANIZATION='Example org' +export ORG_UNIT='Example unit' +export KOJI_MASTER_FQDN=$(hostname -f) +export KOJI_SLAVE_FQDN="$KOJI_MASTER_FQDN" +export KOJI_DIR=/srv/koji +export CGIT_FQDN="$KOJI_MASTER_FQDN" +export EXTERNAL_REPO=https://cdn.download.clearlinux.org/current/\$arch/os/ +export TAG_NAME=clear diff --git a/koji.yaml b/koji.yaml new file mode 100644 index 0000000..e843fd7 --- /dev/null +++ b/koji.yaml @@ -0,0 +1,38 @@ +--- +- hosts: koji-all-in-one + tasks: + - name: Add required bundles for koji + swupd: + name: koji postgresql web-server-basic + state: present + become: true + - name: copy koji-setup scripts + copy: + src: koji-setup + dest: /tmp/ + owner: "{{ ansible_ssh_user }}" + group: "{{ ansible_ssh_user }}" + - name: update bootstrap-build file permissions + file: + path: /tmp/koji-setup/bootstrap-build.sh + mode: 0755 + - name: update deploy-koji file permissions + file: + path: /tmp/koji-setup/deploy-koji.sh + mode: 0755 + - name: update gencert file permissions + file: + path: /tmp/koji-setup/gencert.sh + mode: 0755 + - name: run koji deployment + command: + ./deploy-koji.sh + become: true + args: + chdir: /tmp/koji-setup + - name: bootstrap build tags and targets + command: + ./bootstrap-build.sh + become: true + args: + chdir: /tmp/koji-setup