From 06410babf542327d0b80b3a23e0dd51a8b3fd57c Mon Sep 17 00:00:00 2001 From: David Benjamin Date: Tue, 7 Jan 2025 02:34:29 -0500 Subject: [PATCH 1/2] Remove embed_test_data.go I believe nothing uses this anymore. Bug: 42290412 Change-Id: I8c736570d89393b37efe8d53cef29f887a8762fa Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/74927 Commit-Queue: David Benjamin Auto-Submit: David Benjamin Reviewed-by: Bob Beck --- util/embed_test_data.go | 174 ----------------------------------- util/generate_build_files.py | 14 --- 2 files changed, 188 deletions(-) delete mode 100644 util/embed_test_data.go diff --git a/util/embed_test_data.go b/util/embed_test_data.go deleted file mode 100644 index da66694e6..000000000 --- a/util/embed_test_data.go +++ /dev/null @@ -1,174 +0,0 @@ -// Copyright 2017 The BoringSSL Authors -// -// Permission to use, copy, modify, and/or distribute this software for any -// purpose with or without fee is hereby granted, provided that the above -// copyright notice and this permission notice appear in all copies. -// -// THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -// WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -// MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY -// SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -// WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION -// OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN -// CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -//go:build ignore - -// embed_test_data generates a C++ source file which exports a function, -// GetTestData, which looks up the specified data files. -package main - -import ( - "bytes" - "flag" - "fmt" - "os" - "strings" -) - -var fileList = flag.String("file-list", "", "if not empty, the path to a file containing a newline-separated list of files, to work around Windows command-line limits") - -func quote(in []byte) string { - var lastWasHex bool - var buf bytes.Buffer - buf.WriteByte('"') - for _, b := range in { - var wasHex bool - switch b { - case '\a': - buf.WriteString(`\a`) - case '\b': - buf.WriteString(`\b`) - case '\f': - buf.WriteString(`\f`) - case '\n': - buf.WriteString(`\n`) - case '\r': - buf.WriteString(`\r`) - case '\t': - buf.WriteString(`\t`) - case '\v': - buf.WriteString(`\v`) - case '"': - buf.WriteString(`\"`) - case '\\': - buf.WriteString(`\\`) - default: - // Emit printable ASCII characters, [32, 126], as-is to minimize - // file size. However, if the previous character used a hex escape - // sequence, do not emit 0-9 and a-f as-is. C++ interprets "\x123" - // as a single (overflowing) escape sequence, rather than '\x12' - // followed by '3'. - isHexDigit := ('0' <= b && b <= '9') || ('a' <= b && b <= 'f') || ('A' <= b && b <= 'F') - if 32 <= b && b <= 126 && !(lastWasHex && isHexDigit) { - buf.WriteByte(b) - } else { - fmt.Fprintf(&buf, "\\x%02x", b) - wasHex = true - } - } - lastWasHex = wasHex - } - buf.WriteByte('"') - return buf.String() -} - -func main() { - flag.Parse() - - var files []string - if len(*fileList) != 0 { - data, err := os.ReadFile(*fileList) - if err != nil { - fmt.Fprintf(os.Stderr, "Error reading %s: %s.\n", *fileList, err) - os.Exit(1) - } - files = strings.FieldsFunc(string(data), func(r rune) bool { return r == '\r' || r == '\n' }) - } - - files = append(files, flag.Args()...) - - fmt.Printf(`/* Copyright 2017 The BoringSSL Authors - * - * Permission to use, copy, modify, and/or distribute this software for any - * purpose with or without fee is hereby granted, provided that the above - * copyright notice and this permission notice appear in all copies. - * - * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES - * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF - * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY - * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION - * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN - * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ - -/* This file is generated by: -`) - fmt.Printf(" * go run util/embed_test_data.go") - for _, arg := range files { - fmt.Printf(" \\\n * %s", arg) - } - fmt.Printf(" */\n") - - fmt.Printf(` -/* clang-format off */ - -#include -#include - -#include -#include - - -`) - - // MSVC limits the length of string constants, so we emit an array of - // them and concatenate at runtime. We could also use a single array - // literal, but this is less compact. - const chunkSize = 8192 - - for i, arg := range files { - data, err := os.ReadFile(arg) - if err != nil { - fmt.Fprintf(os.Stderr, "Error reading %s: %s.\n", arg, err) - os.Exit(1) - } - fmt.Printf("static const size_t kLen%d = %d;\n\n", i, len(data)) - - fmt.Printf("static const char *kData%d[] = {\n", i) - for len(data) > 0 { - chunk := chunkSize - if chunk > len(data) { - chunk = len(data) - } - fmt.Printf(" %s,\n", quote(data[:chunk])) - data = data[chunk:] - } - fmt.Printf("};\n") - } - - fmt.Printf(`static std::string AssembleString(const char **data, size_t len) { - std::string ret; - for (size_t i = 0; i < len; i += %d) { - size_t chunk = std::min(static_cast(%d), len - i); - ret.append(data[i / %d], chunk); - } - return ret; -} - -/* Silence -Wmissing-declarations. */ -std::string GetTestData(const char *path); - -std::string GetTestData(const char *path) { -`, chunkSize, chunkSize, chunkSize) - for i, arg := range files { - fmt.Printf(" if (strcmp(path, %s) == 0) {\n", quote([]byte(arg))) - fmt.Printf(" return AssembleString(kData%d, kLen%d);\n", i, i) - fmt.Printf(" }\n") - } - fmt.Printf(` fprintf(stderr, "File not embedded: %%s.\n", path); - abort(); -} -`) - -} diff --git a/util/generate_build_files.py b/util/generate_build_files.py index 30c583354..8152ec45e 100644 --- a/util/generate_build_files.py +++ b/util/generate_build_files.py @@ -24,7 +24,6 @@ import json PREFIX = None -EMBED_TEST_DATA = False def PathOf(x): @@ -620,13 +619,6 @@ def main(platforms): crypto_nasm = sorted(sources['bcm']['nasm'] + sources['crypto']['nasm'] + sources['test_support']['nasm']) - if EMBED_TEST_DATA: - with open('crypto_test_data.cc', 'w+') as out: - subprocess.check_call( - ['go', 'run', 'util/embed_test_data.go'] + sources['crypto_test']['data'], - cwd='src', - stdout=out) - files = { 'bcm_crypto': PrefixWithSrc(sources['bcm']['srcs']), 'crypto': PrefixWithSrc(crypto), @@ -680,14 +672,8 @@ if __name__ == '__main__': '|'.join(sorted(ALL_PLATFORMS.keys()))) parser.add_option('--prefix', dest='prefix', help='For Bazel, prepend argument to all source files') - parser.add_option( - '--embed_test_data', dest='embed_test_data', action='store_true', - help='Generates the legacy crypto_test_data.cc file. To use, build with' + - ' -DBORINGSSL_CUSTOM_GET_TEST_DATA and add this file to ' + - 'crypto_test.') options, args = parser.parse_args(sys.argv[1:]) PREFIX = options.prefix - EMBED_TEST_DATA = options.embed_test_data if not args: parser.print_help() From 305e5c570be4b142e8590d37dd9136b60f12b9bc Mon Sep 17 00:00:00 2001 From: David Benjamin Date: Tue, 7 Jan 2025 01:52:59 -0500 Subject: [PATCH 2/2] Unfork Chromium's Clang update script We can DEPS in individual directories and just run the actual script. This avoids needing to maintain our own copy, and also means we can potentially use Chromium's Rust build later. (The Rust update script pulls in the Clang one.) We should also be able to unfork vs_toolchain.py, but there's a minor headache around it ending up in the LUCI copy of depot_tools and not being cached. Avoiding that seems to require enough fuss that it might be easier to switch to the windows_sdk recipe module, even if that VS toolchain isn't updated as frequently. Change-Id: Icc4ca117d0187b616747711cb563f766cd3f9aba Reviewed-on: https://boringssl-review.googlesource.com/c/boringssl/+/74889 Commit-Queue: David Benjamin Reviewed-by: Bob Beck --- .gitignore | 1 + util/bot/DEPS | 14 ++- util/bot/UPDATING | 4 - util/bot/update_clang.py | 178 --------------------------------------- 4 files changed, 13 insertions(+), 184 deletions(-) delete mode 100644 util/bot/update_clang.py diff --git a/.gitignore b/.gitignore index b087ff51f..f19d1c8ae 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ /util/bot/sde-linux64.tar.xz /util/bot/sde-win32 /util/bot/sde-win32.tar.xz +/util/bot/tools /util/bot/win_toolchain /util/bot/win_toolchain.json diff --git a/util/bot/DEPS b/util/bot/DEPS index f17ed25a3..f79ba47de 100644 --- a/util/bot/DEPS +++ b/util/bot/DEPS @@ -50,6 +50,10 @@ vars = { 'llvm_libc_revision': '17e581644f9a71be3eb30f468722ce866058f93a', 'ninja_version': 'version:2@1.12.1.chromium.4', + # Update to the latest revision of + # https://chromium.googlesource.com/chromium/src/tools/clang + 'tools_clang_revision': 'bf9a3411372f2d5eed8b3d27ee8bd8cf6c17135f', + # The Android NDK cannot be updated until https://crbug.com/boringssl/454 is fixed. # We rely on an older NDK to test building without NEON instructions as the baseline. 'android_ndk_revision': 'U0e8L6l52ySjBrUBB82Vdyhsg60vVMqH0ItTW3TRHAQC', @@ -145,7 +149,12 @@ deps = { }], 'condition': 'checkout_riscv64', 'dep_type': 'cipd', - } + }, + + 'boringssl/util/bot/tools/clang': { + 'url': Var('chromium_git') + '/chromium/src/tools/clang.git' + '@' + Var('tools_clang_revision'), + 'condition': 'checkout_clang', + }, } recursedeps = [ @@ -181,7 +190,8 @@ hooks = [ 'pattern': '.', 'condition': 'checkout_clang', 'action': [ 'python3', - 'boringssl/util/bot/update_clang.py', + 'boringssl/util/bot/tools/clang/scripts/update.py', + '--output-dir', 'boringssl/util/bot/llvm-build', ], }, { diff --git a/util/bot/UPDATING b/util/bot/UPDATING index 8fc0faf76..5a97e488e 100644 --- a/util/bot/UPDATING +++ b/util/bot/UPDATING @@ -10,10 +10,6 @@ To update to newer revisions, follow these instructions: DEPS: Update the variables as described in the comments. -update_clang.py: Set CLANG_REVISION and CLANG_SUB_REVISION to the values used in - Chromium, found at - https://chromium.googlesource.com/chromium/src/+/main/tools/clang/scripts/update.py - vs_toolchain.py: Update _GetDesiredVsToolchainHashes from Chromium, found at https://chromium.googlesource.com/chromium/src/+/main/build/vs_toolchain.py This may require taking other updates to that file. (Don't remove MSVC diff --git a/util/bot/update_clang.py b/util/bot/update_clang.py deleted file mode 100644 index b8bb0f33c..000000000 --- a/util/bot/update_clang.py +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env python3 -# Copyright (c) 2012 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -"""This script is used to download prebuilt clang binaries.""" - -from __future__ import division -from __future__ import print_function - -import os -import platform -import shutil -import subprocess -import stat -import sys -import tarfile -import tempfile -import time - -try: - # Python 3.0 or later - from urllib.error import HTTPError, URLError - from urllib.request import urlopen -except ImportError: - from urllib2 import urlopen, HTTPError, URLError - - -# CLANG_REVISION and CLANG_SUB_REVISION determine the build of clang -# to use. These should be synced with tools/clang/scripts/update.py in -# Chromium. -CLANG_REVISION = 'llvmorg-20-init-9764-gb81d8e90' -CLANG_SUB_REVISION = 7 - -PACKAGE_VERSION = '%s-%s' % (CLANG_REVISION, CLANG_SUB_REVISION) - -# Path constants. (All of these should be absolute paths.) -THIS_DIR = os.path.abspath(os.path.dirname(__file__)) -LLVM_BUILD_DIR = os.path.join(THIS_DIR, 'llvm-build') -STAMP_FILE = os.path.join(LLVM_BUILD_DIR, 'cr_build_revision') - -# URL for pre-built binaries. -CDS_URL = os.environ.get('CDS_CLANG_BUCKET_OVERRIDE', - 'https://commondatastorage.googleapis.com/chromium-browser-clang') - - -def DownloadUrl(url, output_file): - """Download url into output_file.""" - CHUNK_SIZE = 4096 - TOTAL_DOTS = 10 - num_retries = 3 - retry_wait_s = 5 # Doubled at each retry. - - while True: - try: - sys.stdout.write('Downloading %s ' % url) - sys.stdout.flush() - response = urlopen(url) - total_size = int(response.headers.get('Content-Length').strip()) - bytes_done = 0 - dots_printed = 0 - while True: - chunk = response.read(CHUNK_SIZE) - if not chunk: - break - output_file.write(chunk) - bytes_done += len(chunk) - num_dots = TOTAL_DOTS * bytes_done // total_size - sys.stdout.write('.' * (num_dots - dots_printed)) - sys.stdout.flush() - dots_printed = num_dots - if bytes_done != total_size: - raise URLError("only got %d of %d bytes" % (bytes_done, total_size)) - print(' Done.') - return - except URLError as e: - sys.stdout.write('\n') - print(e) - if num_retries == 0 or isinstance(e, HTTPError) and e.code == 404: - raise e - num_retries -= 1 - print('Retrying in %d s ...' % retry_wait_s) - time.sleep(retry_wait_s) - retry_wait_s *= 2 - - -def EnsureDirExists(path): - if not os.path.exists(path): - print("Creating directory %s" % path) - os.makedirs(path) - - -def DownloadAndUnpack(url, output_dir): - with tempfile.TemporaryFile() as f: - DownloadUrl(url, f) - f.seek(0) - EnsureDirExists(output_dir) - tarfile.open(mode='r:*', fileobj=f).extractall(path=output_dir) - - -def ReadStampFile(path=STAMP_FILE): - """Return the contents of the stamp file, or '' if it doesn't exist.""" - try: - with open(path, 'r') as f: - return f.read().rstrip() - except IOError: - return '' - - -def WriteStampFile(s, path=STAMP_FILE): - """Write s to the stamp file.""" - EnsureDirExists(os.path.dirname(path)) - with open(path, 'w') as f: - f.write(s) - f.write('\n') - - -def RmTree(dir): - """Delete dir.""" - def ChmodAndRetry(func, path, _): - # Subversion can leave read-only files around. - if not os.access(path, os.W_OK): - os.chmod(path, stat.S_IWUSR) - return func(path) - raise - - shutil.rmtree(dir, onerror=ChmodAndRetry) - - -def CopyFile(src, dst): - """Copy a file from src to dst.""" - print("Copying %s to %s" % (src, dst)) - shutil.copy(src, dst) - - -def UpdateClang(): - cds_file = "clang-%s.tar.xz" % PACKAGE_VERSION - if sys.platform == 'win32' or sys.platform == 'cygwin': - cds_full_url = CDS_URL + '/Win/' + cds_file - elif sys.platform.startswith('linux'): - cds_full_url = CDS_URL + '/Linux_x64/' + cds_file - elif sys.platform == 'darwin': - if platform.machine() == 'arm64': - cds_full_url = CDS_URL + '/Mac_arm64/' + cds_file - else: - cds_full_url = CDS_URL + '/Mac/' + cds_file - else: - return 0 - - print('Updating Clang to %s...' % PACKAGE_VERSION) - - if ReadStampFile() == PACKAGE_VERSION: - print('Clang is already up to date.') - return 0 - - # Reset the stamp file in case the build is unsuccessful. - WriteStampFile('') - - print('Downloading prebuilt clang') - if os.path.exists(LLVM_BUILD_DIR): - RmTree(LLVM_BUILD_DIR) - try: - DownloadAndUnpack(cds_full_url, LLVM_BUILD_DIR) - print('clang %s unpacked' % PACKAGE_VERSION) - WriteStampFile(PACKAGE_VERSION) - return 0 - except URLError: - print('Failed to download prebuilt clang %s' % cds_file) - print('Exiting.') - return 1 - - -def main(): - return UpdateClang() - - -if __name__ == '__main__': - sys.exit(main())