mirror of
https://github.com/clearlinux/docker.git
synced 2026-09-05 13:11:30 +00:00
Merge pull request #1823 from dotcloud/1766-nightly-release
testing, issue #1766: Add nightly release to docker-ci
This commit is contained in:
Daniel Mizyrycki
@@ -1,5 +1,38 @@
|
||||
docker-ci github pull request
|
||||
=============================
|
||||
docker-ci
|
||||
=========
|
||||
|
||||
docker-ci is our buildbot continuous integration server,
|
||||
building and testing docker, hosted on EC2 and reachable at
|
||||
http://docker-ci.dotcloud.com
|
||||
|
||||
|
||||
Deployment
|
||||
==========
|
||||
|
||||
# Load AWS credentials
|
||||
export AWS_ACCESS_KEY_ID=''
|
||||
export AWS_SECRET_ACCESS_KEY=''
|
||||
export AWS_KEYPAIR_NAME=''
|
||||
export AWS_SSH_PRIVKEY=''
|
||||
|
||||
# Load buildbot credentials and config
|
||||
export BUILDBOT_PWD=''
|
||||
export IRC_PWD=''
|
||||
export IRC_CHANNEL='docker-dev'
|
||||
export SMTP_USER=''
|
||||
export SMTP_PWD=''
|
||||
export EMAIL_RCP=''
|
||||
|
||||
# Load registry test credentials
|
||||
export REGISTRY_USER=''
|
||||
export REGISTRY_PWD=''
|
||||
|
||||
cd docker/testing
|
||||
vagrant up --provider=aws
|
||||
|
||||
|
||||
github pull request
|
||||
===================
|
||||
|
||||
The entire docker pull request test workflow is event driven by github. Its
|
||||
usage is fully automatic and the results are logged in docker-ci.dotcloud.com
|
||||
@@ -13,3 +46,11 @@ buildbot (0.8.7p1) was patched using ./testing/buildbot/github.py, so it
|
||||
can understand the PR data github sends to it. Originally PR #1603 (ee64e099e0)
|
||||
implemented this capability. Also we added a new scheduler to exclusively filter
|
||||
PRs. and the 'pullrequest' builder to rebase the PR on top of master and test it.
|
||||
|
||||
|
||||
nighthly release
|
||||
================
|
||||
|
||||
The nightly release process is done by buildbot, running a DinD container that downloads
|
||||
the docker repository and builds the release container. The resulting docker
|
||||
binary is then tested, and if everything is fine, the release is done.
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
# VERSION: 0.22
|
||||
# DOCKER-VERSION 0.6.3
|
||||
# AUTHOR: Daniel Mizyrycki <daniel@dotcloud.com>
|
||||
# DESCRIPTION: Deploy docker-ci on Amazon EC2
|
||||
# COMMENTS:
|
||||
# CONFIG_JSON is an environment variable json string loaded as:
|
||||
#
|
||||
# export CONFIG_JSON='
|
||||
# { "AWS_TAG": "EC2_instance_name",
|
||||
# "AWS_ACCESS_KEY": "EC2_access_key",
|
||||
# "AWS_SECRET_KEY": "EC2_secret_key",
|
||||
# "DOCKER_CI_PUB": "$(cat docker-ci_ssh_public_key.pub)",
|
||||
# "DOCKER_CI_KEY": "$(cat docker-ci_ssh_private_key.key)",
|
||||
# "BUILDBOT_PWD": "Buildbot_server_password",
|
||||
# "IRC_PWD": "Buildbot_IRC_password",
|
||||
# "SMTP_USER": "SMTP_server_user",
|
||||
# "SMTP_PWD": "SMTP_server_password",
|
||||
# "PKG_ACCESS_KEY": "Docker_release_S3_bucket_access_key",
|
||||
# "PKG_SECRET_KEY": "Docker_release_S3_bucket_secret_key",
|
||||
# "PKG_GPG_PASSPHRASE": "Docker_release_gpg_passphrase",
|
||||
# "INDEX_AUTH": "Index_encripted_user_password",
|
||||
# "REGISTRY_USER": "Registry_test_user",
|
||||
# "REGISTRY_PWD": "Registry_test_password",
|
||||
# "REGISTRY_BUCKET": "Registry_S3_bucket_name",
|
||||
# "REGISTRY_ACCESS_KEY": "Registry_S3_bucket_access_key",
|
||||
# "REGISTRY_SECRET_KEY": "Registry_S3_bucket_secret_key",
|
||||
# "IRC_CHANNEL": "Buildbot_IRC_channel",
|
||||
# "EMAIL_RCP": "Buildbot_mailing_receipient" }'
|
||||
#
|
||||
#
|
||||
# TO_BUILD: docker build -t docker-ci .
|
||||
# TO_DEPLOY: docker run -e CONFIG_JSON="${CONFIG_JSON}" docker-ci
|
||||
|
||||
from ubuntu:12.04
|
||||
|
||||
run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list
|
||||
run apt-get update; apt-get install -y python2.7 python-dev python-pip ssh rsync less vim
|
||||
run pip install boto fabric
|
||||
|
||||
# Add deployment code and set default container command
|
||||
add . /docker-ci
|
||||
cmd "/docker-ci/deployment.py"
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
Daniel Mizyrycki <daniel@dotcloud.com> (@mzdaniel)
|
||||
@@ -0,0 +1,26 @@
|
||||
=======
|
||||
testing
|
||||
=======
|
||||
|
||||
This directory contains docker-ci testing related files.
|
||||
|
||||
|
||||
Buildbot
|
||||
========
|
||||
|
||||
Buildbot is a continuous integration system designed to automate the
|
||||
build/test cycle. By automatically rebuilding and testing the tree each time
|
||||
something has changed, build problems are pinpointed quickly, before other
|
||||
developers are inconvenienced by the failure.
|
||||
|
||||
We are running buildbot in Amazon's EC2 to verify docker passes all
|
||||
tests when commits get pushed to the master branch and building
|
||||
nightly releases using Docker in Docker awesome implementation made
|
||||
by Jerome Petazzoni.
|
||||
|
||||
https://github.com/jpetazzo/dind
|
||||
|
||||
Docker's buildbot instance is at http://docker-ci.dotcloud.com/waterfall
|
||||
|
||||
For deployment instructions, please take a look at
|
||||
hack/infrastructure/docker-ci/Dockerfile
|
||||
@@ -0,0 +1 @@
|
||||
Buildbot configuration and setup files
|
||||
@@ -0,0 +1,18 @@
|
||||
[program:buildmaster]
|
||||
command=twistd --nodaemon --no_save -y buildbot.tac
|
||||
directory=/data/buildbot/master
|
||||
chown= root:root
|
||||
redirect_stderr=true
|
||||
stdout_logfile=/var/log/supervisor/buildbot-master.log
|
||||
stderr_logfile=/var/log/supervisor/buildbot-master.log
|
||||
|
||||
[program:buildworker]
|
||||
command=twistd --nodaemon --no_save -y buildbot.tac
|
||||
directory=/data/buildbot/slave
|
||||
chown= root:root
|
||||
redirect_stderr=true
|
||||
stdout_logfile=/var/log/supervisor/buildbot-slave.log
|
||||
stderr_logfile=/var/log/supervisor/buildbot-slave.log
|
||||
|
||||
[group:buildbot]
|
||||
programs=buildmaster,buildworker
|
||||
@@ -0,0 +1,173 @@
|
||||
# This file is part of Buildbot. Buildbot is free software: you can
|
||||
# redistribute it and/or modify it under the terms of the GNU General Public
|
||||
# License as published by the Free Software Foundation, version 2.
|
||||
#
|
||||
# This program is distributed in the hope that it will be useful, but WITHOUT
|
||||
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
|
||||
# FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
|
||||
# details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License along with
|
||||
# this program; if not, write to the Free Software Foundation, Inc., 51
|
||||
# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
|
||||
#
|
||||
# Copyright Buildbot Team Members
|
||||
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
github_buildbot.py is based on git_buildbot.py
|
||||
|
||||
github_buildbot.py will determine the repository information from the JSON
|
||||
HTTP POST it receives from github.com and build the appropriate repository.
|
||||
If your github repository is private, you must add a ssh key to the github
|
||||
repository for the user who initiated the build on the buildslave.
|
||||
|
||||
"""
|
||||
|
||||
import re
|
||||
import datetime
|
||||
from twisted.python import log
|
||||
import calendar
|
||||
|
||||
try:
|
||||
import json
|
||||
assert json
|
||||
except ImportError:
|
||||
import simplejson as json
|
||||
|
||||
# python is silly about how it handles timezones
|
||||
class fixedOffset(datetime.tzinfo):
|
||||
"""
|
||||
fixed offset timezone
|
||||
"""
|
||||
def __init__(self, minutes, hours, offsetSign = 1):
|
||||
self.minutes = int(minutes) * offsetSign
|
||||
self.hours = int(hours) * offsetSign
|
||||
self.offset = datetime.timedelta(minutes = self.minutes,
|
||||
hours = self.hours)
|
||||
|
||||
def utcoffset(self, dt):
|
||||
return self.offset
|
||||
|
||||
def dst(self, dt):
|
||||
return datetime.timedelta(0)
|
||||
|
||||
def convertTime(myTestTimestamp):
|
||||
#"1970-01-01T00:00:00+00:00"
|
||||
# Normalize myTestTimestamp
|
||||
if myTestTimestamp[-1] == 'Z':
|
||||
myTestTimestamp = myTestTimestamp[:-1] + '-00:00'
|
||||
matcher = re.compile(r'(\d\d\d\d)-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)([-+])(\d\d):(\d\d)')
|
||||
result = matcher.match(myTestTimestamp)
|
||||
(year, month, day, hour, minute, second, offsetsign, houroffset, minoffset) = \
|
||||
result.groups()
|
||||
if offsetsign == '+':
|
||||
offsetsign = 1
|
||||
else:
|
||||
offsetsign = -1
|
||||
|
||||
offsetTimezone = fixedOffset( minoffset, houroffset, offsetsign )
|
||||
myDatetime = datetime.datetime( int(year),
|
||||
int(month),
|
||||
int(day),
|
||||
int(hour),
|
||||
int(minute),
|
||||
int(second),
|
||||
0,
|
||||
offsetTimezone)
|
||||
return calendar.timegm( myDatetime.utctimetuple() )
|
||||
|
||||
def getChanges(request, options = None):
|
||||
"""
|
||||
Reponds only to POST events and starts the build process
|
||||
|
||||
:arguments:
|
||||
request
|
||||
the http request object
|
||||
"""
|
||||
payload = json.loads(request.args['payload'][0])
|
||||
import urllib,datetime
|
||||
fname = str(datetime.datetime.now()).replace(' ','_').replace(':','-')[:19]
|
||||
open('github_{0}.json'.format(fname),'w').write(json.dumps(json.loads(urllib.unquote(request.args['payload'][0])), sort_keys = True, indent = 2))
|
||||
|
||||
if 'pull_request' in payload:
|
||||
user = payload['pull_request']['user']['login']
|
||||
repo = payload['pull_request']['head']['repo']['name']
|
||||
repo_url = payload['pull_request']['head']['repo']['html_url']
|
||||
else:
|
||||
user = payload['repository']['owner']['name']
|
||||
repo = payload['repository']['name']
|
||||
repo_url = payload['repository']['url']
|
||||
project = request.args.get('project', None)
|
||||
if project:
|
||||
project = project[0]
|
||||
elif project is None:
|
||||
project = ''
|
||||
# This field is unused:
|
||||
#private = payload['repository']['private']
|
||||
changes = process_change(payload, user, repo, repo_url, project)
|
||||
log.msg("Received %s changes from github" % len(changes))
|
||||
return (changes, 'git')
|
||||
|
||||
def process_change(payload, user, repo, repo_url, project):
|
||||
"""
|
||||
Consumes the JSON as a python object and actually starts the build.
|
||||
|
||||
:arguments:
|
||||
payload
|
||||
Python Object that represents the JSON sent by GitHub Service
|
||||
Hook.
|
||||
"""
|
||||
changes = []
|
||||
|
||||
newrev = payload['after'] if 'after' in payload else payload['pull_request']['head']['sha']
|
||||
refname = payload['ref'] if 'ref' in payload else payload['pull_request']['head']['ref']
|
||||
|
||||
# We only care about regular heads, i.e. branches
|
||||
match = re.match(r"^(refs\/heads\/|)([^/]+)$", refname)
|
||||
if not match:
|
||||
log.msg("Ignoring refname `%s': Not a branch" % refname)
|
||||
return []
|
||||
|
||||
branch = match.groups()[1]
|
||||
if re.match(r"^0*$", newrev):
|
||||
log.msg("Branch `%s' deleted, ignoring" % branch)
|
||||
return []
|
||||
else:
|
||||
if 'pull_request' in payload:
|
||||
changes = [{
|
||||
'category' : 'github_pullrequest',
|
||||
'who' : '{0} - PR#{1}'.format(user,payload['number']),
|
||||
'files' : [],
|
||||
'comments' : payload['pull_request']['title'],
|
||||
'revision' : newrev,
|
||||
'when' : convertTime(payload['pull_request']['updated_at']),
|
||||
'branch' : branch,
|
||||
'revlink' : '{0}/commit/{1}'.format(repo_url,newrev),
|
||||
'repository' : repo_url,
|
||||
'project' : project }]
|
||||
return changes
|
||||
for commit in payload['commits']:
|
||||
files = []
|
||||
if 'added' in commit:
|
||||
files.extend(commit['added'])
|
||||
if 'modified' in commit:
|
||||
files.extend(commit['modified'])
|
||||
if 'removed' in commit:
|
||||
files.extend(commit['removed'])
|
||||
when = convertTime( commit['timestamp'])
|
||||
log.msg("New revision: %s" % commit['id'][:8])
|
||||
chdict = dict(
|
||||
who = commit['author']['name']
|
||||
+ " <" + commit['author']['email'] + ">",
|
||||
files = files,
|
||||
comments = commit['message'],
|
||||
revision = commit['id'],
|
||||
when = when,
|
||||
branch = branch,
|
||||
revlink = commit['url'],
|
||||
repository = repo_url,
|
||||
project = project)
|
||||
changes.append(chdict)
|
||||
return changes
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
import os
|
||||
from buildbot.buildslave import BuildSlave
|
||||
from buildbot.schedulers.forcesched import ForceScheduler
|
||||
from buildbot.schedulers.basic import SingleBranchScheduler
|
||||
from buildbot.schedulers.timed import Nightly
|
||||
from buildbot.changes import filter
|
||||
from buildbot.config import BuilderConfig
|
||||
from buildbot.process.factory import BuildFactory
|
||||
from buildbot.process.properties import Interpolate
|
||||
from buildbot.steps.shell import ShellCommand
|
||||
from buildbot.status import html, words
|
||||
from buildbot.status.web import authz, auth
|
||||
from buildbot.status.mail import MailNotifier
|
||||
|
||||
PORT_WEB = 80 # Buildbot webserver port
|
||||
PORT_GITHUB = 8011 # Buildbot github hook port
|
||||
PORT_MASTER = 9989 # Port where buildbot master listen buildworkers
|
||||
TEST_USER = 'buildbot' # Credential to authenticate build triggers
|
||||
TEST_PWD = 'docker' # Credential to authenticate build triggers
|
||||
BUILDER_NAME = 'docker'
|
||||
GITHUB_DOCKER = 'github.com/dotcloud/docker'
|
||||
BUILDBOT_PATH = '/data/buildbot'
|
||||
DOCKER_PATH = '/go/src/github.com/dotcloud/docker'
|
||||
DOCKER_CI_PATH = '/docker-ci'
|
||||
BUILDER_PATH = '/data/buildbot/slave/{0}/build'.format(BUILDER_NAME)
|
||||
PULL_REQUEST_PATH = '/data/buildbot/slave/pullrequest/build'
|
||||
|
||||
# Credentials set by setup.sh and Vagrantfile
|
||||
BUILDBOT_PWD = ''
|
||||
IRC_PWD = ''
|
||||
IRC_CHANNEL = ''
|
||||
SMTP_USER = ''
|
||||
SMTP_PWD = ''
|
||||
EMAIL_RCP = ''
|
||||
|
||||
|
||||
c = BuildmasterConfig = {}
|
||||
|
||||
c['title'] = "Docker"
|
||||
c['titleURL'] = "waterfall"
|
||||
c['buildbotURL'] = "http://docker-ci.dotcloud.com/"
|
||||
c['db'] = {'db_url':"sqlite:///state.sqlite"}
|
||||
c['slaves'] = [BuildSlave('buildworker', BUILDBOT_PWD)]
|
||||
c['slavePortnum'] = PORT_MASTER
|
||||
|
||||
|
||||
# Schedulers
|
||||
c['schedulers'] = [ForceScheduler(name='trigger', builderNames=[BUILDER_NAME,
|
||||
'index','registry','coverage','nightlyrelease'])]
|
||||
c['schedulers'] += [SingleBranchScheduler(name="all",
|
||||
change_filter=filter.ChangeFilter(branch='master'), treeStableTimer=None,
|
||||
builderNames=[BUILDER_NAME])]
|
||||
c['schedulers'] += [SingleBranchScheduler(name='pullrequest',
|
||||
change_filter=filter.ChangeFilter(category='github_pullrequest'), treeStableTimer=None,
|
||||
builderNames=['pullrequest'])]
|
||||
c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['nightlyrelease'],
|
||||
hour=7, minute=00)]
|
||||
c['schedulers'] += [Nightly(name='every4hrs', branch=None, builderNames=['registry','index'],
|
||||
hour=range(0,24,4), minute=15)]
|
||||
|
||||
# Builders
|
||||
# Docker commit test
|
||||
factory = BuildFactory()
|
||||
factory.addStep(ShellCommand(description='Docker', logEnviron=False,
|
||||
usePTY=True, command=['sh', '-c', Interpolate(
|
||||
'{0}/docker-test/test_docker.sh %(src::revision)s'.format(DOCKER_CI_PATH))]))
|
||||
c['builders'] = [BuilderConfig(name='docker',slavenames=['buildworker'],
|
||||
factory=factory)]
|
||||
|
||||
# Docker pull request test
|
||||
factory = BuildFactory()
|
||||
factory.addStep(ShellCommand(description='pull_request', logEnviron=False,
|
||||
usePTY=True, command=['sh', '-c', Interpolate(
|
||||
'{0}/docker-test/test_docker.sh %(src::revision)s %(src::repository)s'
|
||||
' %(src::branch)s'.format(DOCKER_CI_PATH))]))
|
||||
c['builders'] += [BuilderConfig(name='pullrequest',slavenames=['buildworker'],
|
||||
factory=factory)]
|
||||
|
||||
# Docker coverage test
|
||||
factory = BuildFactory()
|
||||
factory.addStep(ShellCommand(description='Coverage', logEnviron=False,
|
||||
usePTY=True, command='{0}/docker-coverage/coverage-docker.sh'.format(
|
||||
DOCKER_CI_PATH)))
|
||||
c['builders'] += [BuilderConfig(name='coverage',slavenames=['buildworker'],
|
||||
factory=factory)]
|
||||
|
||||
# Registry functional test
|
||||
factory = BuildFactory()
|
||||
factory.addStep(ShellCommand(description='registry', logEnviron=False,
|
||||
command='. {0}/master/credentials.cfg; '
|
||||
'/docker-ci/functionaltests/test_registry.sh'.format(BUILDBOT_PATH),
|
||||
usePTY=True))
|
||||
c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'],
|
||||
factory=factory)]
|
||||
|
||||
# Index functional test
|
||||
factory = BuildFactory()
|
||||
factory.addStep(ShellCommand(description='index', logEnviron=False,
|
||||
command='. {0}/master/credentials.cfg; '
|
||||
'{1}/testing/functionaltests/test_index.py'.format(BUILDBOT_PATH,
|
||||
DOCKER_PATH), usePTY=True))
|
||||
c['builders'] += [BuilderConfig(name='index',slavenames=['buildworker'],
|
||||
factory=factory)]
|
||||
|
||||
# Docker nightly release
|
||||
nightlyrelease_cmd = ('docker run -i -t -privileged -lxc-conf=lxc.aa_profile=unconfined'
|
||||
' -e AWS_S3_BUCKET=test.docker.io dockerbuilder')
|
||||
factory = BuildFactory()
|
||||
factory.addStep(ShellCommand(description='NightlyRelease',logEnviron=False,usePTY=True,
|
||||
command=nightlyrelease_cmd))
|
||||
c['builders'] += [BuilderConfig(name='nightlyrelease',slavenames=['buildworker'],
|
||||
factory=factory)]
|
||||
|
||||
# Status
|
||||
authz_cfg = authz.Authz(auth=auth.BasicAuth([(TEST_USER, TEST_PWD)]),
|
||||
forceBuild='auth')
|
||||
c['status'] = [html.WebStatus(http_port=PORT_WEB, authz=authz_cfg)]
|
||||
c['status'].append(html.WebStatus(http_port=PORT_GITHUB, allowForce=True,
|
||||
change_hook_dialects={ 'github': True }))
|
||||
c['status'].append(MailNotifier(fromaddr='buildbot@docker.io',
|
||||
sendToInterestedUsers=False, extraRecipients=[EMAIL_RCP],
|
||||
mode='failing', relayhost='smtp.mailgun.org', smtpPort=587, useTls=True,
|
||||
smtpUser=SMTP_USER, smtpPassword=SMTP_PWD))
|
||||
c['status'].append(words.IRC("irc.freenode.net", "dockerqabot",
|
||||
channels=[IRC_CHANNEL], password=IRC_PWD, allowForce=True,
|
||||
notify_events={'exception':1, 'successToFailure':1, 'failureToSuccess':1}))
|
||||
@@ -0,0 +1,9 @@
|
||||
sqlalchemy<=0.7.9
|
||||
sqlalchemy-migrate>=0.7.2
|
||||
buildbot==0.8.7p1
|
||||
buildbot_slave==0.8.7p1
|
||||
nose==1.2.1
|
||||
requests==1.1.0
|
||||
flask==0.10.1
|
||||
simplejson==2.3.2
|
||||
selenium==2.35.0
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Setup of buildbot configuration. Package installation is being done by
|
||||
# Vagrantfile
|
||||
# Dependencies: buildbot, buildbot-slave, supervisor
|
||||
|
||||
USER=$1
|
||||
CFG_PATH=$2
|
||||
DOCKER_PATH=$3
|
||||
BUILDBOT_PWD=$4
|
||||
IRC_PWD=$5
|
||||
IRC_CHANNEL=$6
|
||||
SMTP_USER=$7
|
||||
SMTP_PWD=$8
|
||||
EMAIL_RCP=$9
|
||||
REGISTRY_USER=${10}
|
||||
REGISTRY_PWD=${11}
|
||||
REGISTRY_BUCKET=${12}
|
||||
REGISTRY_ACCESS_KEY=${13}
|
||||
REGISTRY_SECRET_KEY=${14}
|
||||
BUILDBOT_PATH="/data/buildbot"
|
||||
SLAVE_NAME="buildworker"
|
||||
SLAVE_SOCKET="localhost:9989"
|
||||
|
||||
export PATH="/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin"
|
||||
|
||||
function run { su $USER -c "$1"; }
|
||||
|
||||
# Exit if buildbot has already been installed
|
||||
[ -d "$BUILDBOT_PATH" ] && exit 0
|
||||
|
||||
# Setup buildbot
|
||||
run "mkdir -p $BUILDBOT_PATH"
|
||||
cd $BUILDBOT_PATH
|
||||
run "buildbot create-master master"
|
||||
run "cp $CFG_PATH/master.cfg master"
|
||||
run "sed -i -E 's#(BUILDBOT_PWD = ).+#\1\"$BUILDBOT_PWD\"#' master/master.cfg"
|
||||
run "sed -i -E 's#(IRC_PWD = ).+#\1\"$IRC_PWD\"#' master/master.cfg"
|
||||
run "sed -i -E 's#(IRC_CHANNEL = ).+#\1\"$IRC_CHANNEL\"#' master/master.cfg"
|
||||
run "sed -i -E 's#(SMTP_USER = ).+#\1\"$SMTP_USER\"#' master/master.cfg"
|
||||
run "sed -i -E 's#(SMTP_PWD = ).+#\1\"$SMTP_PWD\"#' master/master.cfg"
|
||||
run "sed -i -E 's#(EMAIL_RCP = ).+#\1\"$EMAIL_RCP\"#' master/master.cfg"
|
||||
run "buildslave create-slave slave $SLAVE_SOCKET $SLAVE_NAME $BUILDBOT_PWD"
|
||||
run "echo 'export DOCKER_CREDS=\"$REGISTRY_USER:$REGISTRY_PWD\"' > $BUILDBOT_PATH/master/credentials.cfg"
|
||||
run "echo 'export S3_BUCKET=\"$REGISTRY_BUCKET\"' >> $BUILDBOT_PATH/master/credentials.cfg"
|
||||
run "echo 'export S3_ACCESS_KEY=\"$REGISTRY_ACCESS_KEY\"' >> $BUILDBOT_PATH/master/credentials.cfg"
|
||||
run "echo 'export S3_SECRET_KEY=\"$REGISTRY_SECRET_KEY\"' >> $BUILDBOT_PATH/master/credentials.cfg"
|
||||
|
||||
# Patch github webstatus to capture pull requests
|
||||
cp $CFG_PATH/github.py /usr/local/lib/python2.7/dist-packages/buildbot/status/web/hooks
|
||||
|
||||
# Allow buildbot subprocesses (docker tests) to properly run in containers,
|
||||
# in particular with docker -u
|
||||
run "sed -i 's/^umask = None/umask = 000/' slave/buildbot.tac"
|
||||
|
||||
# Setup supervisor
|
||||
cp $CFG_PATH/buildbot.conf /etc/supervisor/conf.d/buildbot.conf
|
||||
sed -i -E "s/^chmod=0700.+/chmod=0770\nchown=root:$USER/" /etc/supervisor/supervisord.conf
|
||||
kill -HUP $(pgrep -f "/usr/bin/python /usr/bin/supervisord")
|
||||
Executable
+155
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import os, sys, re, json, base64
|
||||
from boto.ec2.connection import EC2Connection
|
||||
from subprocess import call
|
||||
from fabric import api
|
||||
from fabric.api import cd, run, put, sudo
|
||||
from os import environ as env
|
||||
from time import sleep
|
||||
|
||||
# Remove SSH private key as it needs more processing
|
||||
CONFIG = json.loads(re.sub(r'("DOCKER_CI_KEY".+?"(.+?)",)','',
|
||||
env['CONFIG_JSON'], flags=re.DOTALL))
|
||||
|
||||
# Populate environment variables
|
||||
for key in CONFIG:
|
||||
env[key] = CONFIG[key]
|
||||
|
||||
# Load SSH private key
|
||||
env['DOCKER_CI_KEY'] = re.sub('^.+"DOCKER_CI_KEY".+?"(.+?)".+','\\1',
|
||||
env['CONFIG_JSON'],flags=re.DOTALL)
|
||||
|
||||
|
||||
AWS_TAG = env.get('AWS_TAG','docker-ci')
|
||||
AWS_KEY_NAME = 'dotcloud-dev' # Same as CONFIG_JSON['DOCKER_CI_PUB']
|
||||
AWS_AMI = 'ami-d582d6bc' # Ubuntu 13.04
|
||||
AWS_REGION = 'us-east-1'
|
||||
AWS_TYPE = 'm1.small'
|
||||
AWS_SEC_GROUPS = 'gateway'
|
||||
AWS_IMAGE_USER = 'ubuntu'
|
||||
DOCKER_PATH = '/go/src/github.com/dotcloud/docker'
|
||||
DOCKER_CI_PATH = '/docker-ci'
|
||||
CFG_PATH = '{}/buildbot'.format(DOCKER_CI_PATH)
|
||||
|
||||
|
||||
class AWS_EC2:
|
||||
'''Amazon EC2'''
|
||||
def __init__(self, access_key, secret_key):
|
||||
'''Set default API parameters'''
|
||||
self.handler = EC2Connection(access_key, secret_key)
|
||||
def create_instance(self, tag, instance_type):
|
||||
reservation = self.handler.run_instances(**instance_type)
|
||||
instance = reservation.instances[0]
|
||||
sleep(10)
|
||||
while instance.state != 'running':
|
||||
sleep(5)
|
||||
instance.update()
|
||||
print "Instance state: %s" % (instance.state)
|
||||
instance.add_tag("Name",tag)
|
||||
print "instance %s done!" % (instance.id)
|
||||
return instance.ip_address
|
||||
def get_instances(self):
|
||||
return self.handler.get_all_instances()
|
||||
def get_tags(self):
|
||||
return dict([(i.instances[0].id, i.instances[0].tags['Name'])
|
||||
for i in self.handler.get_all_instances() if i.instances[0].tags])
|
||||
def del_instance(self, instance_id):
|
||||
self.handler.terminate_instances(instance_ids=[instance_id])
|
||||
|
||||
|
||||
def json_fmt(data):
|
||||
'''Format json output'''
|
||||
return json.dumps(data, sort_keys = True, indent = 2)
|
||||
|
||||
|
||||
# Create EC2 API handler
|
||||
ec2 = AWS_EC2(env['AWS_ACCESS_KEY'], env['AWS_SECRET_KEY'])
|
||||
|
||||
# Stop processing if AWS_TAG exists on EC2
|
||||
if AWS_TAG in ec2.get_tags().values():
|
||||
print ('Instance: {} already deployed. Not further processing.'
|
||||
.format(AWS_TAG))
|
||||
exit(1)
|
||||
|
||||
ip = ec2.create_instance(AWS_TAG, {'image_id':AWS_AMI, 'instance_type':AWS_TYPE,
|
||||
'security_groups':[AWS_SEC_GROUPS], 'key_name':AWS_KEY_NAME})
|
||||
|
||||
# Wait 30 seconds for the machine to boot
|
||||
sleep(30)
|
||||
|
||||
# Create docker-ci ssh private key so docker-ci docker container can communicate
|
||||
# with its EC2 instance
|
||||
os.makedirs('/root/.ssh')
|
||||
open('/root/.ssh/id_rsa','w').write(env['DOCKER_CI_KEY'])
|
||||
os.chmod('/root/.ssh/id_rsa',0600)
|
||||
open('/root/.ssh/config','w').write('StrictHostKeyChecking no\n')
|
||||
|
||||
api.env.host_string = ip
|
||||
api.env.user = AWS_IMAGE_USER
|
||||
api.env.key_filename = '/root/.ssh/id_rsa'
|
||||
|
||||
# Correct timezone
|
||||
sudo('echo "America/Los_Angeles" >/etc/timezone')
|
||||
sudo('dpkg-reconfigure --frontend noninteractive tzdata')
|
||||
|
||||
# Load public docker-ci key
|
||||
sudo("echo '{}' >> /root/.ssh/authorized_keys".format(env['DOCKER_CI_PUB']))
|
||||
|
||||
# Create docker nightly release credentials file
|
||||
credentials = {
|
||||
'AWS_ACCESS_KEY': env['PKG_ACCESS_KEY'],
|
||||
'AWS_SECRET_KEY': env['PKG_SECRET_KEY'],
|
||||
'GPG_PASSPHRASE': env['PKG_GPG_PASSPHRASE'],
|
||||
'INDEX_AUTH': env['INDEX_AUTH']}
|
||||
open(DOCKER_CI_PATH + '/nightlyrelease/release_credentials.json', 'w').write(
|
||||
base64.b64encode(json.dumps(credentials)))
|
||||
|
||||
# Transfer docker
|
||||
sudo('mkdir -p ' + DOCKER_CI_PATH)
|
||||
sudo('chown {}.{} {}'.format(AWS_IMAGE_USER, AWS_IMAGE_USER, DOCKER_CI_PATH))
|
||||
call('/usr/bin/rsync -aH {} {}@{}:{}'.format(DOCKER_CI_PATH, AWS_IMAGE_USER, ip,
|
||||
os.path.dirname(DOCKER_CI_PATH)), shell=True)
|
||||
|
||||
# Install Docker and Buildbot dependencies
|
||||
sudo('addgroup docker')
|
||||
sudo('usermod -a -G docker ubuntu')
|
||||
sudo('mkdir /mnt/docker; ln -s /mnt/docker /var/lib/docker')
|
||||
sudo('wget -q -O - https://get.docker.io/gpg | apt-key add -')
|
||||
sudo('echo deb https://get.docker.io/ubuntu docker main >'
|
||||
' /etc/apt/sources.list.d/docker.list')
|
||||
sudo('echo -e "deb http://archive.ubuntu.com/ubuntu raring main universe\n'
|
||||
'deb http://us.archive.ubuntu.com/ubuntu/ raring-security main universe\n"'
|
||||
' > /etc/apt/sources.list; apt-get update')
|
||||
sudo('DEBIAN_FRONTEND=noninteractive apt-get install -q -y wget python-dev'
|
||||
' python-pip supervisor git mercurial linux-image-extra-$(uname -r)'
|
||||
' aufs-tools make libfontconfig libevent-dev')
|
||||
sudo('wget -O - https://go.googlecode.com/files/go1.1.2.linux-amd64.tar.gz | '
|
||||
'tar -v -C /usr/local -xz; ln -s /usr/local/go/bin/go /usr/bin/go')
|
||||
sudo('GOPATH=/go go get -d github.com/dotcloud/docker')
|
||||
sudo('pip install -r {}/requirements.txt'.format(CFG_PATH))
|
||||
|
||||
# Install docker and testing dependencies
|
||||
sudo('apt-get install -y -q lxc-docker')
|
||||
sudo('curl -s https://phantomjs.googlecode.com/files/'
|
||||
'phantomjs-1.9.1-linux-x86_64.tar.bz2 | tar jx -C /usr/bin'
|
||||
' --strip-components=2 phantomjs-1.9.1-linux-x86_64/bin/phantomjs')
|
||||
|
||||
#### FIXME. Temporarily install docker with proper apparmor handling
|
||||
sudo('stop docker')
|
||||
sudo('wget -q -O /usr/bin/docker http://test.docker.io/test/docker')
|
||||
sudo('start docker')
|
||||
|
||||
# Build docker-ci containers
|
||||
sudo('cd {}; docker build -t docker .'.format(DOCKER_PATH))
|
||||
sudo('cd {}/nightlyrelease; docker build -t dockerbuilder .'.format(
|
||||
DOCKER_CI_PATH))
|
||||
|
||||
# Setup buildbot
|
||||
sudo('mkdir /data')
|
||||
sudo('{0}/setup.sh root {0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}'
|
||||
' {11} {12}'.format(CFG_PATH, DOCKER_PATH, env['BUILDBOT_PWD'],
|
||||
env['IRC_PWD'], env['IRC_CHANNEL'], env['SMTP_USER'],
|
||||
env['SMTP_PWD'], env['EMAIL_RCP'], env['REGISTRY_USER'],
|
||||
env['REGISTRY_PWD'], env['REGISTRY_BUCKET'], env['REGISTRY_ACCESS_KEY'],
|
||||
env['REGISTRY_SECRET_KEY']))
|
||||
@@ -0,0 +1,32 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -x
|
||||
# Generate a random string of $1 characters
|
||||
function random {
|
||||
cat /dev/urandom | tr -cd 'a-f0-9' | head -c $1
|
||||
}
|
||||
|
||||
# Compute test paths
|
||||
BASE_PATH=`pwd`/test_docker_$(random 12)
|
||||
DOCKER_PATH=$BASE_PATH/go/src/github.com/dotcloud/docker
|
||||
export GOPATH=$BASE_PATH/go:$DOCKER_PATH/vendor
|
||||
|
||||
# Fetch latest master
|
||||
mkdir -p $DOCKER_PATH
|
||||
cd $DOCKER_PATH
|
||||
git init .
|
||||
git fetch -q http://github.com/dotcloud/docker master
|
||||
git reset --hard FETCH_HEAD
|
||||
|
||||
# Fetch go coverage
|
||||
cd $BASE_PATH/go
|
||||
GOPATH=$BASE_PATH/go go get github.com/axw/gocov/gocov
|
||||
sudo -E GOPATH=$GOPATH ./bin/gocov test -deps -exclude-goroot -v\
|
||||
-exclude github.com/gorilla/context,github.com/gorilla/mux,github.com/kr/pty,\
|
||||
code.google.com/p/go.net/websocket,github.com/dotcloud/tar\
|
||||
github.com/dotcloud/docker | ./bin/gocov report; exit_status=$?
|
||||
|
||||
# Cleanup testing directory
|
||||
rm -rf $BASE_PATH
|
||||
|
||||
exit $exit_status
|
||||
@@ -0,0 +1,35 @@
|
||||
#!/bin/bash
|
||||
|
||||
set -x
|
||||
COMMIT=${1-HEAD}
|
||||
REPO=${2-http://github.com/dotcloud/docker}
|
||||
BRANCH=${3-master}
|
||||
|
||||
# Generate a random string of $1 characters
|
||||
function random {
|
||||
cat /dev/urandom | tr -cd 'a-f0-9' | head -c $1
|
||||
}
|
||||
|
||||
# Compute test paths
|
||||
BASE_PATH=`pwd`/test_docker_$(random 12)
|
||||
DOCKER_PATH=$BASE_PATH/go/src/github.com/dotcloud/docker
|
||||
export GOPATH=$BASE_PATH/go:$DOCKER_PATH/vendor
|
||||
|
||||
# Fetch latest master
|
||||
mkdir -p $DOCKER_PATH
|
||||
cd $DOCKER_PATH
|
||||
git init .
|
||||
git fetch -q http://github.com/dotcloud/docker master
|
||||
git reset --hard FETCH_HEAD
|
||||
|
||||
# Merge commit
|
||||
git fetch -q "$REPO" "$BRANCH"
|
||||
git merge --no-edit $COMMIT || exit 1
|
||||
|
||||
# Test commit
|
||||
go test -v; exit_status=$?
|
||||
|
||||
# Cleanup testing directory
|
||||
rm -rf $BASE_PATH
|
||||
|
||||
exit $exit_status
|
||||
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/python
|
||||
|
||||
import os
|
||||
username, password = os.environ['DOCKER_CREDS'].split(':')
|
||||
|
||||
from selenium import webdriver
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.common.keys import Keys
|
||||
from selenium.webdriver.support.ui import Select
|
||||
from selenium.common.exceptions import NoSuchElementException
|
||||
import unittest, time, re
|
||||
|
||||
class Docker(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.driver = webdriver.PhantomJS()
|
||||
self.driver.implicitly_wait(30)
|
||||
self.base_url = "http://www.docker.io/"
|
||||
self.verificationErrors = []
|
||||
self.accept_next_alert = True
|
||||
|
||||
def test_docker(self):
|
||||
driver = self.driver
|
||||
print "Login into {0} as login user {1} ...".format(self.base_url,username)
|
||||
driver.get(self.base_url + "/")
|
||||
driver.find_element_by_link_text("INDEX").click()
|
||||
driver.find_element_by_link_text("login").click()
|
||||
driver.find_element_by_id("id_username").send_keys(username)
|
||||
driver.find_element_by_id("id_password").send_keys(password)
|
||||
print "Checking login user ..."
|
||||
driver.find_element_by_css_selector("input[type=\"submit\"]").click()
|
||||
try: self.assertEqual("test", driver.find_element_by_css_selector("h3").text)
|
||||
except AssertionError as e: self.verificationErrors.append(str(e))
|
||||
print "Login user {0} found".format(username)
|
||||
|
||||
def is_element_present(self, how, what):
|
||||
try: self.driver.find_element(by=how, value=what)
|
||||
except NoSuchElementException, e: return False
|
||||
return True
|
||||
|
||||
def is_alert_present(self):
|
||||
try: self.driver.switch_to_alert()
|
||||
except NoAlertPresentException, e: return False
|
||||
return True
|
||||
|
||||
def close_alert_and_get_its_text(self):
|
||||
try:
|
||||
alert = self.driver.switch_to_alert()
|
||||
alert_text = alert.text
|
||||
if self.accept_next_alert:
|
||||
alert.accept()
|
||||
else:
|
||||
alert.dismiss()
|
||||
return alert_text
|
||||
finally: self.accept_next_alert = True
|
||||
|
||||
def tearDown(self):
|
||||
self.driver.quit()
|
||||
self.assertEqual([], self.verificationErrors)
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,26 @@
|
||||
#!/bin/sh
|
||||
|
||||
set -x
|
||||
|
||||
# Cleanup
|
||||
rm -rf docker-registry
|
||||
|
||||
# Setup the environment
|
||||
export SETTINGS_FLAVOR=test
|
||||
export DOCKER_REGISTRY_CONFIG=config_test.yml
|
||||
|
||||
# Get latest docker registry
|
||||
git clone -q https://github.com/dotcloud/docker-registry.git
|
||||
cd docker-registry
|
||||
|
||||
# Get dependencies
|
||||
pip install -q -r requirements.txt
|
||||
pip install -q -r test-requirements.txt
|
||||
pip install -q tox
|
||||
|
||||
# Run registry tests
|
||||
tox || exit 1
|
||||
export PYTHONPATH=$(pwd)/docker-registry
|
||||
python -m unittest discover -p s3.py -s test || exit 1
|
||||
python -m unittest discover -p workflow.py -s test
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
# VERSION: 1.2
|
||||
# DOCKER-VERSION 0.6.3
|
||||
# AUTHOR: Daniel Mizyrycki <daniel@dotcloud.com>
|
||||
# DESCRIPTION: Build docker nightly release using Docker in Docker.
|
||||
# REFERENCES: This code reuses the excellent implementation of docker in docker
|
||||
# made by Jerome Petazzoni. https://github.com/jpetazzo/dind
|
||||
# COMMENTS:
|
||||
# release_credentials.json is a base64 json encoded file containing:
|
||||
# { "AWS_ACCESS_KEY": "Test_docker_AWS_S3_bucket_id",
|
||||
# "AWS_SECRET_KEY='Test_docker_AWS_S3_bucket_key'
|
||||
# "GPG_PASSPHRASE='Test_docker_GPG_passphrase_signature'
|
||||
# "INDEX_AUTH='Encripted_index_authentication' }
|
||||
# TO_BUILD: docker build -t dockerbuilder .
|
||||
# TO_RELEASE: docker run -i -t -privileged -lxc-conf="lxc.aa_profile = unconfined" -e AWS_S3_BUCKET="test.docker.io" dockerbuilder
|
||||
|
||||
from docker
|
||||
maintainer Daniel Mizyrycki <daniel@dotcloud.com>
|
||||
|
||||
# Add docker dependencies and downloading packages
|
||||
run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list
|
||||
run apt-get update; apt-get install -y -q wget python2.7
|
||||
|
||||
# Add production docker binary
|
||||
run wget -q -O /usr/bin/docker http://get.docker.io/builds/Linux/x86_64/docker-latest; chmod +x /usr/bin/docker
|
||||
|
||||
#### FIXME. Temporarily install docker with proper apparmor handling
|
||||
run wget -q -O /usr/bin/docker http://test.docker.io/test/docker; chmod +x /usr/bin/docker
|
||||
|
||||
# Add proto docker builder
|
||||
add ./dockerbuild /usr/bin/dockerbuild
|
||||
run chmod +x /usr/bin/dockerbuild
|
||||
|
||||
# Add release credentials
|
||||
add ./release_credentials.json /root/release_credentials.json
|
||||
|
||||
# Launch build process in a container
|
||||
cmd dockerbuild
|
||||
@@ -0,0 +1,45 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Variables AWS_ACCESS_KEY, AWS_SECRET_KEY, PG_PASSPHRASE and INDEX_AUTH
|
||||
# are decoded from /root/release_credentials.json
|
||||
# Variable AWS_S3_BUCKET is passed to the environment from docker run -e
|
||||
|
||||
# Enable debugging
|
||||
set -x
|
||||
|
||||
# Fetch docker master branch
|
||||
rm -rf /go/src/github.com/dotcloud/docker
|
||||
cd /
|
||||
git clone -q http://github.com/dotcloud/docker /go/src/github.com/dotcloud/docker
|
||||
cd /go/src/github.com/dotcloud/docker
|
||||
|
||||
echo FIXME. Temporarily add Jerome changeset with proper apparmor handling
|
||||
git fetch http://github.com/jpetazzo/docker escape-apparmor-confinement:escape-apparmor-confinement
|
||||
git rebase --onto master master escape-apparmor-confinement
|
||||
|
||||
# Launch docker daemon using dind inside the container
|
||||
./hack/dind /usr/bin/docker -d &
|
||||
sleep 5
|
||||
|
||||
# Add an uncommitted change to generate a timestamped release
|
||||
date > timestamp
|
||||
|
||||
# Build the docker package using /Dockerfile
|
||||
docker build -t docker .
|
||||
|
||||
# Run Docker unittests binary and Ubuntu package
|
||||
docker run -privileged -lxc-conf=lxc.aa_profile=unconfined docker hack/make.sh || exit 1
|
||||
|
||||
# Turn debug off to load credentials from the environment
|
||||
set +x
|
||||
eval $(cat /root/release_credentials.json | python -c '
|
||||
import sys,json,base64;
|
||||
d=json.loads(base64.b64decode(sys.stdin.read()));
|
||||
exec("""for k in d: print "export {0}=\\"{1}\\"".format(k,d[k])""")')
|
||||
echo '{"https://index.docker.io/v1/":{"auth":"'$INDEX_AUTH'","email":"engineering@dotcloud.com"}}' > /.dockercfg
|
||||
set -x
|
||||
|
||||
# Push docker nightly
|
||||
echo docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=XXXXX -e AWS_SECRET_KEY=XXXXX -e GPG_PASSPHRASE=XXXXX release hack/release.sh
|
||||
set +x
|
||||
docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=$AWS_ACCESS_KEY -e AWS_SECRET_KEY=$AWS_SECRET_KEY -e GPG_PASSPHRASE=$GPG_PASSPHRASE release hack/release.sh
|
||||
@@ -0,0 +1 @@
|
||||
eyAiQVdTX0FDQ0VTU19LRVkiOiAiIiwKICAiQVdTX1NFQ1JFVF9LRVkiOiAiIiwKICAiR1BHX1BBU1NQSFJBU0UiOiAiIiwKICAiSU5ERVhfQVVUSCI6ICIiIH0=
|
||||
Reference in New Issue
Block a user