diff --git a/src/.gitignore b/src/.gitignore index b087ff51f..f19d1c8ae 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -33,6 +33,7 @@ /util/bot/sde-linux64.tar.xz /util/bot/sde-win32 /util/bot/sde-win32.tar.xz +/util/bot/tools /util/bot/win_toolchain /util/bot/win_toolchain.json diff --git a/src/util/bot/DEPS b/src/util/bot/DEPS index f17ed25a3..f79ba47de 100644 --- a/src/util/bot/DEPS +++ b/src/util/bot/DEPS @@ -50,6 +50,10 @@ vars = { 'llvm_libc_revision': '17e581644f9a71be3eb30f468722ce866058f93a', 'ninja_version': 'version:2@1.12.1.chromium.4', + # Update to the latest revision of + # https://chromium.googlesource.com/chromium/src/tools/clang + 'tools_clang_revision': 'bf9a3411372f2d5eed8b3d27ee8bd8cf6c17135f', + # The Android NDK cannot be updated until https://crbug.com/boringssl/454 is fixed. # We rely on an older NDK to test building without NEON instructions as the baseline. 'android_ndk_revision': 'U0e8L6l52ySjBrUBB82Vdyhsg60vVMqH0ItTW3TRHAQC', @@ -145,7 +149,12 @@ deps = { }], 'condition': 'checkout_riscv64', 'dep_type': 'cipd', - } + }, + + 'boringssl/util/bot/tools/clang': { + 'url': Var('chromium_git') + '/chromium/src/tools/clang.git' + '@' + Var('tools_clang_revision'), + 'condition': 'checkout_clang', + }, } recursedeps = [ @@ -181,7 +190,8 @@ hooks = [ 'pattern': '.', 'condition': 'checkout_clang', 'action': [ 'python3', - 'boringssl/util/bot/update_clang.py', + 'boringssl/util/bot/tools/clang/scripts/update.py', + '--output-dir', 'boringssl/util/bot/llvm-build', ], }, { diff --git a/src/util/bot/UPDATING b/src/util/bot/UPDATING index 8fc0faf76..5a97e488e 100644 --- a/src/util/bot/UPDATING +++ b/src/util/bot/UPDATING @@ -10,10 +10,6 @@ To update to newer revisions, follow these instructions: DEPS: Update the variables as described in the comments. -update_clang.py: Set CLANG_REVISION and CLANG_SUB_REVISION to the values used in - Chromium, found at - https://chromium.googlesource.com/chromium/src/+/main/tools/clang/scripts/update.py - vs_toolchain.py: Update _GetDesiredVsToolchainHashes from Chromium, found at https://chromium.googlesource.com/chromium/src/+/main/build/vs_toolchain.py This may require taking other updates to that file. (Don't remove MSVC diff --git a/src/util/bot/update_clang.py b/src/util/bot/update_clang.py deleted file mode 100644 index b8bb0f33c..000000000 --- a/src/util/bot/update_clang.py +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""This script is used to download prebuilt clang binaries.""" - -from __future__ import division -from __future__ import print_function - -import os -import platform -import shutil -import subprocess -import stat -import sys -import tarfile -import tempfile -import time - -try: - # Python 3.0 or later - from urllib.error import HTTPError, URLError - from urllib.request import urlopen -except ImportError: - from urllib2 import urlopen, HTTPError, URLError - - -# CLANG_REVISION and CLANG_SUB_REVISION determine the build of clang -# to use. These should be synced with tools/clang/scripts/update.py in -# Chromium. -CLANG_REVISION = 'llvmorg-20-init-9764-gb81d8e90' -CLANG_SUB_REVISION = 7 - -PACKAGE_VERSION = '%s-%s' % (CLANG_REVISION, CLANG_SUB_REVISION) - -# Path constants. (All of these should be absolute paths.) -THIS_DIR = os.path.abspath(os.path.dirname(__file__)) -LLVM_BUILD_DIR = os.path.join(THIS_DIR, 'llvm-build') -STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision') - -# URL for pre-built binaries. -CDS_URL = os.environ.get('CDS_CLANG_BUCKET_OVERRIDE', - 'https://commondatastorage.googleapis.com/chromium-browser-clang') - - -def DownloadUrl(url, output_file): - """Download url into output_file.""" - CHUNK_SIZE = 4096 - TOTAL_DOTS = 10 - num_retries = 3 - retry_wait_s = 5 # Doubled at each retry. - - while True: - try: - sys.stdout.write('Downloading %s ' % url) - sys.stdout.flush() - response = urlopen(url) - total_size = int(response.headers.get('Content-Length').strip()) - bytes_done = 0 - dots_printed = 0 - while True: - chunk = response.read(CHUNK_SIZE) - if not chunk: - break - output_file.write(chunk) - bytes_done += len(chunk) - num_dots = TOTAL_DOTS * bytes_done // total_size - sys.stdout.write('.' * (num_dots - dots_printed)) - sys.stdout.flush() - dots_printed = num_dots - if bytes_done != total_size: - raise URLError("only got %d of %d bytes" % (bytes_done, total_size)) - print(' Done.') - return - except URLError as e: - sys.stdout.write('\n') - print(e) - if num_retries == 0 or isinstance(e, HTTPError) and e.code == 404: - raise e - num_retries -= 1 - print('Retrying in %d s ...' % retry_wait_s) - time.sleep(retry_wait_s) - retry_wait_s *= 2 - - -def EnsureDirExists(path): - if not os.path.exists(path): - print("Creating directory %s" % path) - os.makedirs(path) - - -def DownloadAndUnpack(url, output_dir): - with tempfile.TemporaryFile() as f: - DownloadUrl(url, f) - f.seek(0) - EnsureDirExists(output_dir) - tarfile.open(mode='r:*', fileobj=f).extractall(path=output_dir) - - -def ReadStampFile(path=STAMP_FILE): - """Return the contents of the stamp file, or '' if it doesn't exist.""" - try: - with open(path, 'r') as f: - return f.read().rstrip() - except IOError: - return '' - - -def WriteStampFile(s, path=STAMP_FILE): - """Write s to the stamp file.""" - EnsureDirExists(os.path.dirname(path)) - with open(path, 'w') as f: - f.write(s) - f.write('\n') - - -def RmTree(dir): - """Delete dir.""" - def ChmodAndRetry(func, path, _): - # Subversion can leave read-only files around. - if not os.access(path, os.W_OK): - os.chmod(path, stat.S_IWUSR) - return func(path) - raise - - shutil.rmtree(dir, onerror=ChmodAndRetry) - - -def CopyFile(src, dst): - """Copy a file from src to dst.""" - print("Copying %s to %s" % (src, dst)) - shutil.copy(src, dst) - - -def UpdateClang(): - cds_file = "clang-%s.tar.xz" % PACKAGE_VERSION - if sys.platform == 'win32' or sys.platform == 'cygwin': - cds_full_url = CDS_URL + '/Win/' + cds_file - elif sys.platform.startswith('linux'): - cds_full_url = CDS_URL + '/Linux_x64/' + cds_file - elif sys.platform == 'darwin': - if platform.machine() == 'arm64': - cds_full_url = CDS_URL + '/Mac_arm64/' + cds_file - else: - cds_full_url = CDS_URL + '/Mac/' + cds_file - else: - return 0 - - print('Updating Clang to %s...' % PACKAGE_VERSION) - - if ReadStampFile() == PACKAGE_VERSION: - print('Clang is already up to date.') - return 0 - - # Reset the stamp file in case the build is unsuccessful. - WriteStampFile('') - - print('Downloading prebuilt clang') - if os.path.exists(LLVM_BUILD_DIR): - RmTree(LLVM_BUILD_DIR) - try: - DownloadAndUnpack(cds_full_url, LLVM_BUILD_DIR) - print('clang %s unpacked' % PACKAGE_VERSION) - WriteStampFile(PACKAGE_VERSION) - return 0 - except URLError: - print('Failed to download prebuilt clang %s' % cds_file) - print('Exiting.') - return 1 - - -def main(): - return UpdateClang() - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/src/util/embed_test_data.go b/src/util/embed_test_data.go deleted file mode 100644 index da66694e6..000000000 --- a/src/util/embed_test_data.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2017 The BoringSSL Authors -// -// Permission to use, copy, modify, and/or distribute this software for any -// purpose with or without fee is hereby granted, provided that the above -// copyright notice and this permission notice appear in all copies. -// -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION -// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -//go:build ignore - -// embed_test_data generates a C++ source file which exports a function, -// GetTestData, which looks up the specified data files. -package main - -import ( - "bytes" - "flag" - "fmt" - "os" - "strings" -) - -var fileList = flag.String("file-list", "", "if not empty, the path to a file containing a newline-separated list of files, to work around Windows command-line limits") - -func quote(in []byte) string { - var lastWasHex bool - var buf bytes.Buffer - buf.WriteByte('"') - for _, b := range in { - var wasHex bool - switch b { - case '\a': - buf.WriteString(`\a`) - case '\b': - buf.WriteString(`\b`) - case '\f': - buf.WriteString(`\f`) - case '\n': - buf.WriteString(`\n`) - case '\r': - buf.WriteString(`\r`) - case '\t': - buf.WriteString(`\t`) - case '\v': - buf.WriteString(`\v`) - case '"': - buf.WriteString(`\"`) - case '\\': - buf.WriteString(`\\`) - default: - // Emit printable ASCII characters, [32, 126], as-is to minimize - // file size. However, if the previous character used a hex escape - // sequence, do not emit 0-9 and a-f as-is. C++ interprets "\x123" - // as a single (overflowing) escape sequence, rather than '\x12' - // followed by '3'. - isHexDigit := ('0' <= b && b <= '9') || ('a' <= b && b <= 'f') || ('A' <= b && b <= 'F') - if 32 <= b && b <= 126 && !(lastWasHex && isHexDigit) { - buf.WriteByte(b) - } else { - fmt.Fprintf(&buf, "\\x%02x", b) - wasHex = true - } - } - lastWasHex = wasHex - } - buf.WriteByte('"') - return buf.String() -} - -func main() { - flag.Parse() - - var files []string - if len(*fileList) != 0 { - data, err := os.ReadFile(*fileList) - if err != nil { - fmt.Fprintf(os.Stderr, "Error reading %s: %s.\n", *fileList, err) - os.Exit(1) - } - files = strings.FieldsFunc(string(data), func(r rune) bool { return r == '\r' || r == '\n' }) - } - - files = append(files, flag.Args()...) - - fmt.Printf(`/* Copyright 2017 The BoringSSL Authors - * - * Permission to use, copy, modify, and/or distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION - * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN - * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ - -/* This file is generated by: -`) - fmt.Printf(" * go run util/embed_test_data.go") - for _, arg := range files { - fmt.Printf(" \\\n * %s", arg) - } - fmt.Printf(" */\n") - - fmt.Printf(` -/* clang-format off */ - -#include -#include - -#include -#include - - -`) - - // MSVC limits the length of string constants, so we emit an array of - // them and concatenate at runtime. We could also use a single array - // literal, but this is less compact. - const chunkSize = 8192 - - for i, arg := range files { - data, err := os.ReadFile(arg) - if err != nil { - fmt.Fprintf(os.Stderr, "Error reading %s: %s.\n", arg, err) - os.Exit(1) - } - fmt.Printf("static const size_t kLen%d = %d;\n\n", i, len(data)) - - fmt.Printf("static const char *kData%d[] = {\n", i) - for len(data) > 0 { - chunk := chunkSize - if chunk > len(data) { - chunk = len(data) - } - fmt.Printf(" %s,\n", quote(data[:chunk])) - data = data[chunk:] - } - fmt.Printf("};\n") - } - - fmt.Printf(`static std::string AssembleString(const char **data, size_t len) { - std::string ret; - for (size_t i = 0; i < len; i += %d) { - size_t chunk = std::min(static_cast(%d), len - i); - ret.append(data[i / %d], chunk); - } - return ret; -} - -/* Silence -Wmissing-declarations. */ -std::string GetTestData(const char *path); - -std::string GetTestData(const char *path) { -`, chunkSize, chunkSize, chunkSize) - for i, arg := range files { - fmt.Printf(" if (strcmp(path, %s) == 0) {\n", quote([]byte(arg))) - fmt.Printf(" return AssembleString(kData%d, kLen%d);\n", i, i) - fmt.Printf(" }\n") - } - fmt.Printf(` fprintf(stderr, "File not embedded: %%s.\n", path); - abort(); -} -`) - -} diff --git a/src/util/generate_build_files.py b/src/util/generate_build_files.py index 30c583354..8152ec45e 100644 --- a/src/util/generate_build_files.py +++ b/src/util/generate_build_files.py @@ -24,7 +24,6 @@ import json PREFIX = None -EMBED_TEST_DATA = False def PathOf(x): @@ -620,13 +619,6 @@ def main(platforms): crypto_nasm = sorted(sources['bcm']['nasm'] + sources['crypto']['nasm'] + sources['test_support']['nasm']) - if EMBED_TEST_DATA: - with open('crypto_test_data.cc', 'w+') as out: - subprocess.check_call( - ['go', 'run', 'util/embed_test_data.go'] + sources['crypto_test']['data'], - cwd='src', - stdout=out) - files = { 'bcm_crypto': PrefixWithSrc(sources['bcm']['srcs']), 'crypto': PrefixWithSrc(crypto), @@ -680,14 +672,8 @@ if __name__ == '__main__': '|'.join(sorted(ALL_PLATFORMS.keys()))) parser.add_option('--prefix', dest='prefix', help='For Bazel, prepend argument to all source files') - parser.add_option( - '--embed_test_data', dest='embed_test_data', action='store_true', - help='Generates the legacy crypto_test_data.cc file. To use, build with' + - ' -DBORINGSSL_CUSTOM_GET_TEST_DATA and add this file to ' + - 'crypto_test.') options, args = parser.parse_args(sys.argv[1:]) PREFIX = options.prefix - EMBED_TEST_DATA = options.embed_test_data if not args: parser.print_help()