diff --git a/README.md b/README.md index c8160c44b..dc71da2a5 100644 --- a/README.md +++ b/README.md @@ -241,7 +241,3 @@ Optionally, if `` is present, Bashbrew will look for the `Docker ## Bashbrew Bashbrew is a set of bash scripts for cloning, building, tagging, and pushing the Docker official images. See [`README.md` in the `bashbrew/` subfolder](bashbrew/README.md) for more information. - -## Stackbrew (deprecated) - -Stackbrew is a web-application that performs continuous building of the Docker official images. See [`README.md` in the `stackbrew/` subfolder](stackbrew/README.md) for more information. diff --git a/stackbrew/LICENSE b/stackbrew/LICENSE deleted file mode 100644 index 27448585a..000000000 --- a/stackbrew/LICENSE +++ /dev/null @@ -1,191 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - Copyright 2014 Docker, Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/stackbrew/README.md b/stackbrew/README.md deleted file mode 100644 index 16bceae2c..000000000 --- a/stackbrew/README.md +++ /dev/null @@ -1,114 +0,0 @@ -# Stackbrew - -Stackbrew is a web-application that performs continuous building of the docker -standard library. - -## Install instructions - -1. Install python if it isn't already available on your OS of choice -1. Install the easy_install tool (`sudo apt-get install python-setuptools` -for Debian/Ubuntu) -1. Install the python package manager, `pip` (`easy_install pip`) -1. Run the following command: `sudo pip install -r requirements.txt` -1. You should now be able to use the `brew-cli` script as such. - -### Deploying stackbrew - -These additional instructions are necessary for the stackbrew application to -function. - -1. Install sqlite3 (`sudo apt-get install sqlite3` on Debian/Ubuntu) -1. Create the /opt/stackbrew/repos (`mkdir -p /opt/stackbrew/repos`) folder. -2. Run the `create_db.py` script (`python create_db.py`) -3. Edit `config.json` appropriately to your needs. -4. If you're using the `push` option, you will need to have a valid - `.dockercfg` file in your HOME directory. -5. You can start the application with the command `python app.py` - -## Builds - -Builds are performed regularly and pushed to the public index. - -## API - -A small JSON API allows users to check the status of past builds. - -### Latest build summary - -* `GET /summary` or `GET /status` - - - GET /summary - - { - "build_date": "2013-10-04 18:08:45.685881", - "id": 16, - "result": true - } - -### Summary details - -* `GET /summary/` - - - GET /summary/16 - - [ - { - "commit_id": "7362ff5b812f93eceafbdbf5e5959f676f731f80", - "exception": null, - "source_desc": "git://github.com/dotcloud/hipache@C:7362ff5b812f93eceafbdbf5e5959f676f731f80", - "image_id": "5d313f0ec5af", - "tag": "0.2.4", - "summary_id": 16, - "id": 1, - "repo_name": "hipache" - }, { - "commit_id": "7362ff5b812f93eceafbdbf5e5959f676f731f80", - "exception": null, - "source_desc": "git://github.com/dotcloud/hipache@C:7362ff5b812f93eceafbdbf5e5959f676f731f80", - "image_id": "5d313f0ec5af", - "tag": "latest", - "summary_id": 16, - "id": 2, - "repo_name": "hipache" - }, ... - ] - -### Latest successful build - -* `GET /success/?tag=` -* `tag` parameter is optional, defaults to `latest` - - - GET /success/ubuntu?tag=12.10 - - { - "commit_id": "abd58c43ceec4d4a21622a1e3d45f676fe912e745d31", - "exception": null, - "source_desc": "git://github.com/dotcloud/ubuntu-quantal@B:master", - "image_id": "d462fecc33e1", - "tag": "12.10", - "summary_id": 17, - "id": 19, - "repo_name": "ubuntu" - } - -## Stackbrew CLI - - ./brew-cli -h - -Display usage and help. - - ./brew-cli - -Default build from the default repo/branch. Images will be created under the -`library/` namespace. Does not perform a remote push. - - ./brew-cli -n mycorp.com -b stable --push git://github.com/mycorp/docker - -Will fetch the library definition files in the `stable` branch of the -`git://github.com/mycorp/docker` repository and create images under the -`mycorp.com` namespace (e.g. `mycorp.com/ubuntu`). Created images will then -be pushed to the official docker repository (pending: support for private -repositories) \ No newline at end of file diff --git a/stackbrew/app.py b/stackbrew/app.py deleted file mode 100755 index cc7db8d45..000000000 --- a/stackbrew/app.py +++ /dev/null @@ -1,79 +0,0 @@ -import sys -import json - -import flask - -import brew -import lib.db as db -import lib.periodic as periodic -import lib.utils as utils - -app = flask.Flask('stackbrew') -config = None -with open('./config.json') as config_file: - config = json.load(config_file) -data = db.DbManager(config['db_url'], debug=config['debug']) -history = utils.load_history() -brew.logger = app.logger -brew.set_loglevel('DEBUG' if config['debug'] else 'INFO') - - -@app.route('/') -def home(): - return utils.resp(app, 'stackbrew') - - -@app.route('/summary') -@app.route('/status') -def latest_summary(): - result = data.latest_status() - return utils.resp(app, result) - - -@app.route('/summary/') -def get_summary(id): - result = data.get_summary(id) - return utils.resp(app, result) - - -@app.route('/success/') -def latest_success(repo_name): - tag = flask.request.args.get('tag', None) - result = data.get_latest_successful(repo_name, tag) - return utils.resp(app, result) - - -if config['debug']: - @app.route('/build/force', methods=['POST']) - def force_build(): - build_task() - return utils.resp(app, 'OK') - - -def build_task(): - summary = data.new_summary(config['repos_folder']) - library = brew.StackbrewLibrary(config['library_repo']) - builder = brew.LocalBuilder( - library=library, namespaces=config['namespaces'], - repo_cache=config['repos_folder'], targetlist=config.get('targets') - ) - builder.build_repo_list() - builder.history = history - builder.build_all(callback=summary.handle_build_result) - utils.save_history(builder.history) - if config['push']: - builder.push_all() - - -try: - periodic.init_task(build_task, config['build_interval'], - logger=app.logger) - app.logger.info('Periodic build task initiated.') -except RuntimeError: - app.logger.warning('Periodic build task already locked.') - -app.run( - host=config.get('host', '127.0.0.1'), - port=config.get('port', 5000), - debug=config['debug'] -) diff --git a/stackbrew/brew-cli b/stackbrew/brew-cli deleted file mode 100755 index b25941816..000000000 --- a/stackbrew/brew-cli +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python - -import argparse -import logging -import sys - -try: - import brew -except ImportError as e: - print str(e) - print 'Please install the required dependencies first' - print 'sudo pip install -r requirements.txt' - sys.exit(1) - -logger = logging.getLogger(__name__) -logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', - level='INFO') - - -if __name__ == '__main__': - parser = argparse.ArgumentParser('Build the docker standard library') - parser.add_argument( - '--push', action='store_true', default=False, - help='Push generated repositories' - ) - parser.add_argument( - '--debug', default=False, action='store_true', - help='Enable debugging output' - ) -# parser.add_argument( -# '--noprefill', default=True, action='store_false', -# dest='prefill', help='Disable cache prefill' -# ) - parser.add_argument( - '-n', metavar='NAMESPACE', default='library', - help='Comma-separated list of namespaces used for generated' - ' repositories. Default is library' - ) - parser.add_argument( - '--no-namespace', default=False, action='store_true', - help='In addition to -n, also tag generated images with no' - ' namespace (ie, "library/busybox" as "busybox")' - ) - parser.add_argument( - '-b', metavar='BRANCH', default=brew.DEFAULT_BRANCH, - help='Branch in the repository where the library definition' - ' files will be fetched. Default is ' + brew.DEFAULT_BRANCH - ) - parser.add_argument( - 'repository', default=brew.DEFAULT_REPOSITORY, - nargs='?', help='git repository containing the library definition' - ' files. Default is ' + brew.DEFAULT_REPOSITORY - ) -# parser.add_argument( -# '--reg', default=None, help='Registry address to' -# ' push build results to. Also sets push to true.' -# ) - parser.add_argument( - '--targets', default=None, help='Comma-separated list' - ' of images to build.' - ) - args = parser.parse_args() - if args.debug: - brew.set_loglevel('DEBUG') - namespaces = args.n.split(',') - if args.no_namespace: - namespaces.append('') - targets = args.targets.split(',') - sb_library = brew.StackbrewLibrary(args.repository, args.b) - builder = brew.LocalBuilder(sb_library, namespaces, targets) - builder.build_repo_list() - builder.build_all() - if args.push: - builder.push_all() - -# summary = brew.build_library( -# args.repository, args.b, args.n, args.push or args.reg is not None, -# args.debug, args.prefill, args.reg, args.targets, None, logger -# ) diff --git a/stackbrew/brew/__init__.py b/stackbrew/brew/__init__.py deleted file mode 100644 index bf77a4ae6..000000000 --- a/stackbrew/brew/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .brew import * \ No newline at end of file diff --git a/stackbrew/brew/brew.py b/stackbrew/brew/brew.py deleted file mode 100644 index 582f53100..000000000 --- a/stackbrew/brew/brew.py +++ /dev/null @@ -1,353 +0,0 @@ -import json -import logging -import os -import random -import re -from shutil import rmtree -import string - -import docker - -import git - -DEFAULT_REPOSITORY = 'git://github.com/docker-library/official-images.git' -DEFAULT_BRANCH = 'master' - -logger = logging.getLogger(__name__) -logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', - level='INFO') - - -def set_loglevel(level): - logger.setLevel(level) - git.logger.setLevel(level) - - -class StackbrewError(Exception): - def __init__(self, message, cause=None): - super(StackbrewError, self).__init__(message) - self.cause = cause - - def log(self, logger): - logger.exception(self) - if self.cause: - logger.error('The cause of this error is the following:') - logger.exception(self.cause) - - -class StackbrewLibrary(object): - def __init__(self, repository, branch=None): - self.logger = logging.getLogger(__name__) - logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', - level='INFO') - - self.branch = branch or DEFAULT_BRANCH - self.repository = repository - self.library = None - if not self.repository.startswith(('https://', 'git://')): - self.logger.info('Repository provided assumed to be a local path') - self.library = self.repository - - def clone_library(self): - if self.library: - return self.library - - try: - rep, library = git.clone_branch(self.repository, self.branch) - self.library = library - except git.GitException as e: - raise StackbrewError( - 'Source repository could not be fetched. Ensure ' - 'the address is correct and the branch exists.', - e - ) - - def list_repositories(self): - if not self.library: - self.clone_library() - try: - return [e for e in os.listdir( - os.path.join(self.library, 'library')) if e != 'MAINTAINERS'] - except OSError as e: - raise StackbrewError( - 'The path provided ({0}) could not be found or ' - 'didn\'t contain a library/ folder'.format(self.library), - e - ) - - -class StackbrewRepo(object): - def __init__(self, name, definition_file): - self.buildlist = {} - self.buildorder = [] - self.git_folders = {} - self.name = name - for line in definition_file: - if not line or line.strip() == '': - continue - elif line.lstrip().startswith('#'): # # It's a comment! - continue - logger.debug(line) - tag, url, ref, dfile = self._parse_line(line) - repo = (url, ref, dfile) - if repo in self.buildlist: - self.buildlist[repo].append(tag) - else: - self.buildlist[repo] = [tag] - self.buildorder.append(repo) - - def _parse_line(self, line): - df_folder = '.' - args = line.split(':', 1) - if len(args) != 2: - logger.debug("Invalid line: {0}".format(line)) - raise StackbrewError( - 'Incorrect line format, please refer to the docs' - ) - - try: - repo = args[1].strip().split() - if len(repo) == 2: - df_folder = repo[1].strip() - url, ref = repo[0].strip().rsplit('@', 1) - return (args[0].strip(), url, ref, df_folder) - except ValueError: - logger.debug("Invalid line: {0}".format(line)) - raise StackbrewError( - 'Incorrect line format, please refer to the docs' - ) - - def list_versions(self): - return self.buildorder - - def get_associated_tags(self, repo): - return self.buildlist.get(repo, None) - - def add_git_repo(self, url, repo): - self.git_folders[url] = repo - - def get_git_repo(self, url): - return self.git_folders.get(url, (None, None)) - - -class StackbrewBuilder(object): - def __init__(self, library, namespaces=None, targetlist=None, - repo_cache=None): - self.lib = library - if not hasattr(self.lib, 'list_repositories'): - raise StackbrewError('Invalid library passed to StackbrewBuilder') - self.namespaces = namespaces or ['stackbrew'] - self.targetlist = targetlist - self.repo_cache = repo_cache - self.history = {} - - def build_repo_list(self): - self.repos = [] - for repo in self.lib.list_repositories(): - if self.targetlist and repo not in self.targetlist: - continue - try: - with open(os.path.join(self.lib.library, 'library', repo)) as f: - self.repos.append(StackbrewRepo(repo, f)) - except IOError as e: - raise StackbrewError( - 'Failed to read definition file for {0}'.format(repo), - e - ) - for repo in self.repos: - for version in repo.list_versions(): - logger.debug('{0}: {1}'.format( - repo.name, - ','.join(repo.get_associated_tags(version)) - )) - return self.repos - - def build_all(self, continue_on_error=True, callback=None): - self.pushlist = [] - for repo in self.repos: - self.build_repo(repo, continue_on_error, callback) - for namespace in self.namespaces: - if namespace != '': - self.pushlist.append('/'.join([namespace, repo.name])) - - def build_repo(self, repo, continue_on_error=True, callback=None): - for version in repo.list_versions(): - try: - self.build_version(repo, version, callback) - except StackbrewError as e: - if not continue_on_error: - raise e - e.log(logger) - - def build_version(self, repo, version, callback=None): - if version in self.history: - return self.history[version], None - url, ref, dfile = version - try: - rep, dst_folder = self.clone_version(repo, version) - except StackbrewError as exc: - if callback: - callback(exc, repo, version, None, None) - raise exc - dockerfile_location = os.path.join(dst_folder, dfile) - if not 'Dockerfile' in os.listdir(dockerfile_location): - exc = StackbrewError('Dockerfile not found in cloned repository') - if callback: - callback(exc, repo, version, None, None) - raise exc - img_id, build_result = self.do_build( - repo, version, dockerfile_location, callback - ) - self.history[version] = img_id - return img_id, build_result - - def do_build(self, repo, version, dockerfile_location, callback=None): - raise NotImplementedError - - def _clone_or_checkout(self, url, ref, dst_folder, rep): - if rep: - try: - # The ref already exists, we just need to checkout - dst_folder = git.checkout(rep, ref) - except git.GitException: - # ref is not present, try pulling it from the remote origin - rep, dst_folder = git.pull(url, rep, ref) - return rep, dst_folder - - if dst_folder: - rmtree(dst_folder) - return git.clone(url, ref, dst_folder) - - def clone_version(self, repo, version): - url, ref, dfile = version - rep, dst_folder = repo.get_git_repo(url) - if not dst_folder and self.repo_cache: - dst_folder = os.path.join( - self.repo_cache, repo.name + _random_suffix() - ) - os.mkdir(dst_folder) - try: - rep, dst_folder = self._clone_or_checkout( - url, ref, dst_folder, rep - ) - except Exception as e: - raise StackbrewError( - 'Failed to clone repository {0}@{1}'.format(url, ref), - e - ) - - repo.add_git_repo(url, (rep, dst_folder)) - return rep, dst_folder - - def get_pushlist(self): - return self.pushlist - - def push_all(self, continue_on_error=True, callback=None): - for repo in self.pushlist: - try: - self.do_push(repo, callback) - except StackbrewError as e: - if continue_on_error: - e.log(logger) - else: - raise e - - def do_push(self, repo_name, callback=None): - raise NotImplementedError - - -def _random_suffix(): - return ''.join([ - random.choice(string.ascii_letters + string.digits) for i in xrange(6) - ]) - - -class LocalBuilder(StackbrewBuilder): - def __init__(self, library, namespaces=None, targetlist=None, - repo_cache=None): - super(LocalBuilder, self).__init__( - library, namespaces, targetlist, repo_cache - ) - self.client = docker.Client(version='1.9', timeout=10000, - base_url=os.getenv('DOCKER_HOST')) - self.build_success_re = r'^Successfully built ([a-f0-9]+)\n$' - - def do_build(self, repo, version, dockerfile_location, callback=None): - logger.info( - 'Build start: {0} {1}'.format(repo.name, version) - ) - build_result = self.client.build(path=dockerfile_location, rm=True, - stream=True, quiet=True) - img_id, logs = self._parse_result(build_result) - if not img_id: - exc = StackbrewError( - 'Build failed for {0} ({1})'.format(repo.name, version) - ) - if callback: - callback(exc, repo, version, None, logs) - raise exc - for tag in repo.get_associated_tags(version): - logger.info( - 'Build success: {0} ({1}:{2})'.format(img_id, repo.name, tag) - ) - for namespace in self.namespaces: - if namespace != '': - name = '/'.join([namespace, repo.name]) - else: - name = repo.name - self.client.tag(img_id, name, tag) - - if callback: - callback(None, repo, version, img_id, logs) - return img_id, build_result - - def _parse_result(self, build_result): - if isinstance(build_result, tuple): - img_id, logs = build_result - return img_id, logs - else: - lines = [line for line in build_result] - try: - parsed_lines = [json.loads(e).get('stream', '') for e in lines] - except ValueError: - # sometimes all the data is sent on a single line ???? - # - # ValueError: Extra data: line 1 column 87 - line 1 column - # 33268 (char 86 - 33267) - line = lines[0] - # This ONLY works because every line is formatted as - # {"stream": STRING} - parsed_lines = [ - json.loads(obj).get('stream', '') for obj in - re.findall('{\s*"stream"\s*:\s*"[^"]*"\s*}', line) - ] - - for line in parsed_lines: - match = re.match(self.build_success_re, line) - if match: - return match.group(1), parsed_lines - return None, parsed_lines - - def do_push(self, repo_name, callback=None): - exc = None - for i in xrange(4): - try: - pushlog = self.client.push(repo_name) - if '"error":"' in pushlog: - raise RuntimeError( - 'Error while pushing: {0}'.format(pushlog) - ) - logger.info('Succesfully pushed {0}'.format(repo_name)) - except Exception as e: - exc = e - continue - if callback: - callback(None, repo_name, pushlog) - return - if not callback: - raise StackbrewError( - 'Error while pushing {0}'.format(repo_name), - exc - ) - else: - return callback(exc, repo_name, None) diff --git a/stackbrew/brew/git.py b/stackbrew/brew/git.py deleted file mode 100644 index b802c6d75..000000000 --- a/stackbrew/brew/git.py +++ /dev/null @@ -1,77 +0,0 @@ -import logging -import os -import subprocess -import tempfile - - -logger = logging.getLogger(__name__) - - -class GitException(Exception): - pass - - -class Repo(object): - def __init__(self, path, repo_url): - self.path = path - self.repo_url = repo_url - - def clone(self): - logger.debug('Cloning {0} into {1}'.format(self.repo_url, self.path)) - result = _execute('clone', [self.repo_url, '.'], self.path) - if result != 0: - raise GitException('git clone failed') - - -def _execute(command, args, cwd): - cmd = ['git', command] + args - logger.debug('Executing "{0}" in {1}'.format(' '.join(cmd), cwd)) - return subprocess.Popen(cmd, cwd=cwd).wait() - - -def clone_branch(repo_url, branch="master", folder=None): - return clone(repo_url, branch, folder) - - -def clone_tag(repo_url, tag, folder=None): - return clone(repo_url, 'refs/tags/' + tag, folder) - - -def checkout(rep, ref=None): - if ref is None: - ref = 'refs/heads/master' - logger.debug("Checkout ref:{0} in {1}".format(ref, rep.path)) - result = _execute('checkout', [ref], rep.path) - - if result != 0: - raise GitException('git checkout failed') - - return rep.path - - -def pull(origin, rep, ref=None): - if ref is None: - ref = 'refs/heads/master' - logger.debug("Pull ref:{0} in {1}".format(ref, rep.path)) - result = _execute('pull', ['origin', ref], rep.path) - if result != 0: - raise GitException('git pull failed') - checkout(rep, ref) - return rep, rep.path - - -def clone(repo_url, ref=None, folder=None, rep=None): - if ref is None: - ref = 'refs/heads/master' - logger.debug("Cloning repo_url={0}, ref={1}".format(repo_url, ref)) - if folder is None: - folder = tempfile.mkdtemp() - else: - os.mkdir(folder) - logger.debug("folder = {0}".format(folder)) - rep = Repo(folder, repo_url) - rep.clone() - if ref: - checkout(rep, ref) - - return rep, folder diff --git a/stackbrew/brew/summary.py b/stackbrew/brew/summary.py deleted file mode 100644 index cf2f3d662..000000000 --- a/stackbrew/brew/summary.py +++ /dev/null @@ -1,72 +0,0 @@ - -class SummaryItem(object): - def __init__(self, data): - self.line = data.get('line', None) - self.repository = data.get('repository', None) - self.commit_id = data.get('commit', None) - self.exc = data.get('exc', None) - self.image_id = data.get('id', None) - self.source = data.get('source', None) - self.tag = data.get('tag', None) - - -class Summary(object): - def __init__(self): - self._summary = {} - self._has_exc = False - - def _add_data(self, image, linestr, data): - linestr = linestr.strip('\n') - parts = linestr.split(':', 1) - data.tag = parts[0] - data.source = parts[1] - if image not in self._summary: - self._summary[image] = {linestr: data} - else: - self._summary[image][linestr] = data - - def add_exception(self, image, line, exc, commit=None): - lineno, linestr = line - self._add_data(image, linestr, SummaryItem({ - 'line': lineno, - 'exc': str(exc), - 'repository': image, - 'commit': commit - })) - self._has_exc = True - - def add_success(self, image, line, img_id, commit=None): - lineno, linestr = line - self._add_data(image, linestr, SummaryItem({ - 'line': lineno, - 'id': img_id, - 'repository': image, - 'commit': commit - })) - - def print_summary(self, logger=None): - linesep = ''.center(61, '-') + '\n' - s = 'BREW BUILD SUMMARY\n' + linesep - success = 'OVERALL SUCCESS: {}\n'.format(not self._has_exc) - details = linesep - for image, lines in self._summary.iteritems(): - details = details + '{}\n{}'.format(image, linesep) - for linestr, data in lines.iteritems(): - details = details + '{0:2} | {1} | {2:50}\n'.format( - data.line, - 'KO' if data.exc else 'OK', - data.exc or data.image_id - ) - details = details + linesep - if logger is not None: - logger.info(s + success + details) - else: - print s, success, details - - def exit_code(self): - return 1 if self._has_exc else 0 - - def items(self): - for lines in self._summary.itervalues(): - for item in lines.itervalues(): - yield item diff --git a/stackbrew/config.json b/stackbrew/config.json deleted file mode 100644 index bbb94acff..000000000 --- a/stackbrew/config.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "debug": true, - "push": false, - "build_interval": 6000, - "repos_folder": "/opt/stackbrew/repos", - "db_url": "/opt/stackbrew/data.db", - "library_repo": "https://github.com/dotcloud/stackbrew.git", - "port": 2048, - "namespaces": ["sbv2"] -} diff --git a/stackbrew/create_db.py b/stackbrew/create_db.py deleted file mode 100644 index 00e3012a1..000000000 --- a/stackbrew/create_db.py +++ /dev/null @@ -1,6 +0,0 @@ -import sys - -import lib.db as db - -data = db.DbManager(debug=True) -data.generate_tables() diff --git a/stackbrew/lib/__init__.py b/stackbrew/lib/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/stackbrew/lib/db.py b/stackbrew/lib/db.py deleted file mode 100644 index 21227409e..000000000 --- a/stackbrew/lib/db.py +++ /dev/null @@ -1,123 +0,0 @@ -import datetime - -import sqlalchemy as sql - - -metadata = sql.MetaData() -summary = sql.Table( - 'summary', metadata, - sql.Column('id', sql.Integer, primary_key=True), - sql.Column('result', sql.Boolean), - sql.Column('build_date', sql.String) -) - -summary_item = sql.Table( - 'summary_item', metadata, - sql.Column('id', sql.Integer, primary_key=True), - sql.Column('repo_name', sql.String), - sql.Column('exception', sql.String), - sql.Column('commit_id', sql.String), - sql.Column('image_id', sql.String), - sql.Column('source_desc', sql.String), - sql.Column('tag', sql.String), - sql.Column('summary_id', None, sql.ForeignKey('summary.id')) -) - - -class SummaryV2(object): - def __init__(self, engine, summary_id, errorlogs=None): - self.summary_id = summary_id - self._engine = engine - self.errorlogs = errorlogs - - def handle_build_result(self, exc, repo, version, img_id, build_result): - c = self._engine.connect() - if exc and self.errorlogs: - if isinstance(build_result, list): - build_result = '\n'.join(build_result) - elif not isinstance(build_result, str): - build_result = str(build_result) - with open('{2}/{0}.{1}.error.log'.format(repo.name, version[1], self.errorlogs), 'w') as f: - f.write(build_result) - ins = summary_item.insert().values( - repo_name=repo.name, - exception=str(exc) if exc else None, - commit_id=version[1], - image_id=img_id, - source_desc=version[0], - tag=', '.join(repo.get_associated_tags(version)), - summary_id=self.summary_id - ) - c.execute(ins) - - -class DbManager(object): - def __init__(self, db='/opt/stackbrew/data.db', debug=False): - self._engine = sql.create_engine('sqlite:///' + db, echo=debug) - - def generate_tables(self): - metadata.create_all(self._engine) - - def insert_summary(self, s): - c = self._engine.connect() - summary_id = None - with c.begin(): - ins = summary.insert().values( - result=not s.exit_code(), - build_date=str(datetime.datetime.now())) - r = c.execute(ins) - summary_id = r.inserted_primary_key[0] - for item in s.items(): - ins = summary_item.insert().values( - repo_name=item.repository, - exception=item.exc, - commit_id=item.commit_id, - image_id=item.image_id, - source_desc=item.source, - tag=item.tag, - summary_id=summary_id - ) - c.execute(ins) - return summary_id - - def new_summary(self, errorlogs=None): - c = self._engine.connect() - ins = summary.insert().values( - result=True, build_date=str(datetime.datetime.now()) - ) - r = c.execute(ins) - summary_id = r.inserted_primary_key[0] - return SummaryV2(self._engine, summary_id, errorlogs) - - def latest_status(self): - c = self._engine.connect() - s = sql.select([summary]).order_by(summary.c.id.desc()).limit(1) - res = c.execute(s) - row = res.fetchone() - if row is not None: - return dict(row) - return None - - def get_summary(self, id): - c = self._engine.connect() - s = sql.select([summary_item]).where(summary_item.c.summary_id == id) - res = c.execute(s) - return [dict(row) for row in res] - - def get_latest_successful(self, repo, tag=None): - c = self._engine.connect() - tag = tag or 'latest' - s = sql.select([summary_item]).where( - summary_item.c.repo_name == repo - ).where( - summary_item.c.tag == tag - ).where( - summary_item.c.image_id is not None - ).order_by( - summary_item.c.summary_id.desc() - ).limit(1) - res = c.execute(s) - row = res.fetchone() - if row is not None: - return dict(row) - return None diff --git a/stackbrew/lib/periodic.py b/stackbrew/lib/periodic.py deleted file mode 100644 index 7fb27cb34..000000000 --- a/stackbrew/lib/periodic.py +++ /dev/null @@ -1,33 +0,0 @@ -import atexit -import os -import threading - -lockfiles = [] - - -def init_task(fn, period, lockfile='/opt/stackbrew/brw.lock', logger=None): - def periodic(logger): - if logger is not None: - logger.info('Periodic task started') - t = threading.Timer(period, periodic, [logger]) - t.daemon = True - t.start() - fn() - if os.path.exists(lockfile): - raise RuntimeError('Lockfile already present.') - open(lockfile, 'w').close() - lockfiles.append(lockfile) - t = threading.Timer(0, periodic, [logger]) - t.daemon = True - t.start() - - -def clear_lockfiles(lockfiles): - for lock in lockfiles: - os.remove(lock) - - -def on_exit(lockfiles): - clear_lockfiles(lockfiles) - -atexit.register(on_exit, lockfiles) diff --git a/stackbrew/lib/utils.py b/stackbrew/lib/utils.py deleted file mode 100644 index 8e8fca121..000000000 --- a/stackbrew/lib/utils.py +++ /dev/null @@ -1,37 +0,0 @@ -import json - - -def resp(app, data=None, code=200, headers=None): - if not headers: - headers = {} - if 'Content-Type' not in headers: - headers['Content-Type'] = 'application/json' - data = json.dumps(data) - return app.make_response((data, code, headers)) - - -def save_history(history, target='/opt/stackbrew/history.json'): - save = [] - for k in history.iterkeys(): - url, ref, dfile = k # unpack - save.append({ - 'url': url, - 'ref': ref, - 'dfile': dfile, - 'img': history[k] - }) - - with open(target, 'w') as f: - f.write(json.dumps(save)) - - -def load_history(target='/opt/stackbrew/history.json'): - history = {} - try: - with open(target, 'r') as f: - savefile = json.loads(f.read()) - for item in savefile: - history[(item['url'], item['ref'], item['dfile'])] = item['img'] - return history - except IOError: - return {} diff --git a/stackbrew/requirements.txt b/stackbrew/requirements.txt deleted file mode 100644 index 3707ee85d..000000000 --- a/stackbrew/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -Flask==0.9 -SQLAlchemy==0.8.2 -docker-py==0.5.0 diff --git a/stackbrew/wsgi.py b/stackbrew/wsgi.py deleted file mode 100755 index 72e0bd148..000000000 --- a/stackbrew/wsgi.py +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env python -import app as application \ No newline at end of file