|
|
|
@@ -1,845 +0,0 @@
|
|
|
|
|
diff -ur ceph-19.2.2/src/s3select.orig/include/encryption_internal_19.h ceph-19.2.2/src/s3select/include/encryption_internal_19.h
|
|
|
|
|
--- ceph-19.2.2/src/s3select.orig/include/encryption_internal_19.h 2025-07-07 13:17:09.262618668 -0400
|
|
|
|
|
+++ ceph-19.2.2/src/s3select/include/encryption_internal_19.h 2025-07-07 13:08:00.746355226 -0400
|
|
|
|
|
@@ -0,0 +1,114 @@
|
|
|
|
|
+// Licensed to the Apache Software Foundation (ASF) under one
|
|
|
|
|
+// or more contributor license agreements. See the NOTICE file
|
|
|
|
|
+// distributed with this work for additional information
|
|
|
|
|
+// regarding copyright ownership. The ASF licenses this file
|
|
|
|
|
+// to you under the Apache License, Version 2.0 (the
|
|
|
|
|
+// "License"); you may not use this file except in compliance
|
|
|
|
|
+// with the License. You may obtain a copy of the License at
|
|
|
|
|
+//
|
|
|
|
|
+// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
+//
|
|
|
|
|
+// Unless required by applicable law or agreed to in writing,
|
|
|
|
|
+// software distributed under the License is distributed on an
|
|
|
|
|
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
|
|
|
+// KIND, either express or implied. See the License for the
|
|
|
|
|
+// specific language governing permissions and limitations
|
|
|
|
|
+// under the License.
|
|
|
|
|
+
|
|
|
|
|
+#pragma once
|
|
|
|
|
+
|
|
|
|
|
+#include <memory>
|
|
|
|
|
+#include <string>
|
|
|
|
|
+#include <vector>
|
|
|
|
|
+
|
|
|
|
|
+#include "parquet/properties.h"
|
|
|
|
|
+#include "parquet/types.h"
|
|
|
|
|
+
|
|
|
|
|
+using parquet::ParquetCipher;
|
|
|
|
|
+
|
|
|
|
|
+namespace parquet {
|
|
|
|
|
+namespace encryption {
|
|
|
|
|
+
|
|
|
|
|
+constexpr int kGcmTagLength = 16;
|
|
|
|
|
+constexpr int kNonceLength = 12;
|
|
|
|
|
+
|
|
|
|
|
+// Module types
|
|
|
|
|
+constexpr int8_t kFooter = 0;
|
|
|
|
|
+constexpr int8_t kColumnMetaData = 1;
|
|
|
|
|
+constexpr int8_t kDataPage = 2;
|
|
|
|
|
+constexpr int8_t kDictionaryPage = 3;
|
|
|
|
|
+constexpr int8_t kDataPageHeader = 4;
|
|
|
|
|
+constexpr int8_t kDictionaryPageHeader = 5;
|
|
|
|
|
+constexpr int8_t kColumnIndex = 6;
|
|
|
|
|
+constexpr int8_t kOffsetIndex = 7;
|
|
|
|
|
+
|
|
|
|
|
+/// Performs AES encryption operations with GCM or CTR ciphers.
|
|
|
|
|
+class AesEncryptor {
|
|
|
|
|
+ public:
|
|
|
|
|
+ static AesEncryptor* Make(ParquetCipher::type alg_id, int key_len, bool metadata,
|
|
|
|
|
+ std::vector<AesEncryptor*>* all_encryptors);
|
|
|
|
|
+
|
|
|
|
|
+ ~AesEncryptor();
|
|
|
|
|
+
|
|
|
|
|
+ /// Size difference between plaintext and ciphertext, for this cipher.
|
|
|
|
|
+ int CiphertextSizeDelta();
|
|
|
|
|
+
|
|
|
|
|
+ /// Encrypts plaintext with the key and aad. Key length is passed only for validation.
|
|
|
|
|
+ /// If different from value in constructor, exception will be thrown.
|
|
|
|
|
+ int Encrypt(const uint8_t* plaintext, int plaintext_len, const uint8_t* key,
|
|
|
|
|
+ int key_len, const uint8_t* aad, int aad_len, uint8_t* ciphertext);
|
|
|
|
|
+
|
|
|
|
|
+ /// Encrypts plaintext footer, in order to compute footer signature (tag).
|
|
|
|
|
+ int SignedFooterEncrypt(const uint8_t* footer, int footer_len, const uint8_t* key,
|
|
|
|
|
+ int key_len, const uint8_t* aad, int aad_len,
|
|
|
|
|
+ const uint8_t* nonce, uint8_t* encrypted_footer);
|
|
|
|
|
+
|
|
|
|
|
+ void WipeOut();
|
|
|
|
|
+
|
|
|
|
|
+ private:
|
|
|
|
|
+ /// Can serve one key length only. Possible values: 16, 24, 32 bytes.
|
|
|
|
|
+ explicit AesEncryptor(ParquetCipher::type alg_id, int key_len, bool metadata);
|
|
|
|
|
+ // PIMPL Idiom
|
|
|
|
|
+ class AesEncryptorImpl;
|
|
|
|
|
+ std::unique_ptr<AesEncryptorImpl> impl_;
|
|
|
|
|
+};
|
|
|
|
|
+
|
|
|
|
|
+/// Performs AES decryption operations with GCM or CTR ciphers.
|
|
|
|
|
+class AesDecryptor {
|
|
|
|
|
+ public:
|
|
|
|
|
+ static AesDecryptor* Make(ParquetCipher::type alg_id, int key_len, bool metadata,
|
|
|
|
|
+ std::vector<AesDecryptor*>* all_decryptors);
|
|
|
|
|
+
|
|
|
|
|
+ ~AesDecryptor();
|
|
|
|
|
+ void WipeOut();
|
|
|
|
|
+
|
|
|
|
|
+ /// Size difference between plaintext and ciphertext, for this cipher.
|
|
|
|
|
+ int CiphertextSizeDelta();
|
|
|
|
|
+
|
|
|
|
|
+ /// Decrypts ciphertext with the key and aad. Key length is passed only for
|
|
|
|
|
+ /// validation. If different from value in constructor, exception will be thrown.
|
|
|
|
|
+ int Decrypt(const uint8_t* ciphertext, int ciphertext_len, const uint8_t* key,
|
|
|
|
|
+ int key_len, const uint8_t* aad, int aad_len, uint8_t* plaintext);
|
|
|
|
|
+
|
|
|
|
|
+ private:
|
|
|
|
|
+ /// Can serve one key length only. Possible values: 16, 24, 32 bytes.
|
|
|
|
|
+ explicit AesDecryptor(ParquetCipher::type alg_id, int key_len, bool metadata);
|
|
|
|
|
+ // PIMPL Idiom
|
|
|
|
|
+ class AesDecryptorImpl;
|
|
|
|
|
+ std::unique_ptr<AesDecryptorImpl> impl_;
|
|
|
|
|
+};
|
|
|
|
|
+
|
|
|
|
|
+std::string CreateModuleAad(const std::string& file_aad, int8_t module_type,
|
|
|
|
|
+ int16_t row_group_ordinal, int16_t column_ordinal,
|
|
|
|
|
+ int16_t page_ordinal);
|
|
|
|
|
+
|
|
|
|
|
+std::string CreateFooterAad(const std::string& aad_prefix_bytes);
|
|
|
|
|
+
|
|
|
|
|
+// Update last two bytes of page (or page header) module AAD
|
|
|
|
|
+void QuickUpdatePageAad(const std::string& AAD, int16_t new_page_ordinal);
|
|
|
|
|
+
|
|
|
|
|
+// Wraps OpenSSL RAND_bytes function
|
|
|
|
|
+void RandBytes(unsigned char* buf, int num);
|
|
|
|
|
+
|
|
|
|
|
+} // namespace encryption
|
|
|
|
|
+} // namespace parquet
|
|
|
|
|
diff -ur ceph-19.2.2/src/s3select.orig/include/encryption_internal_20.h ceph-19.2.2/src/s3select/include/encryption_internal_20.h
|
|
|
|
|
--- ceph-19.2.2/src/s3select.orig/include/encryption_internal_20.h 2025-07-07 13:17:09.262618668 -0400
|
|
|
|
|
+++ ceph-19.2.2/src/s3select/include/encryption_internal_20.h 2025-07-07 13:08:00.746536155 -0400
|
|
|
|
|
@@ -0,0 +1,141 @@
|
|
|
|
|
+// Licensed to the Apache Software Foundation (ASF) under one
|
|
|
|
|
+// or more contributor license agreements. See the NOTICE file
|
|
|
|
|
+// distributed with this work for additional information
|
|
|
|
|
+// regarding copyright ownership. The ASF licenses this file
|
|
|
|
|
+// to you under the Apache License, Version 2.0 (the
|
|
|
|
|
+// "License"); you may not use this file except in compliance
|
|
|
|
|
+// with the License. You may obtain a copy of the License at
|
|
|
|
|
+//
|
|
|
|
|
+// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
+//
|
|
|
|
|
+// Unless required by applicable law or agreed to in writing,
|
|
|
|
|
+// software distributed under the License is distributed on an
|
|
|
|
|
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
|
|
|
+// KIND, either express or implied. See the License for the
|
|
|
|
|
+// specific language governing permissions and limitations
|
|
|
|
|
+// under the License.
|
|
|
|
|
+
|
|
|
|
|
+#pragma once
|
|
|
|
|
+
|
|
|
|
|
+#include <memory>
|
|
|
|
|
+#include <string>
|
|
|
|
|
+#include <vector>
|
|
|
|
|
+
|
|
|
|
|
+#include "arrow/util/span.h"
|
|
|
|
|
+#include "parquet/properties.h"
|
|
|
|
|
+#include "parquet/types.h"
|
|
|
|
|
+
|
|
|
|
|
+using parquet::ParquetCipher;
|
|
|
|
|
+
|
|
|
|
|
+namespace parquet::encryption {
|
|
|
|
|
+
|
|
|
|
|
+constexpr int32_t kGcmTagLength = 16;
|
|
|
|
|
+constexpr int32_t kNonceLength = 12;
|
|
|
|
|
+
|
|
|
|
|
+// Module types
|
|
|
|
|
+constexpr int8_t kFooter = 0;
|
|
|
|
|
+constexpr int8_t kColumnMetaData = 1;
|
|
|
|
|
+constexpr int8_t kDataPage = 2;
|
|
|
|
|
+constexpr int8_t kDictionaryPage = 3;
|
|
|
|
|
+constexpr int8_t kDataPageHeader = 4;
|
|
|
|
|
+constexpr int8_t kDictionaryPageHeader = 5;
|
|
|
|
|
+constexpr int8_t kColumnIndex = 6;
|
|
|
|
|
+constexpr int8_t kOffsetIndex = 7;
|
|
|
|
|
+constexpr int8_t kBloomFilterHeader = 8;
|
|
|
|
|
+constexpr int8_t kBloomFilterBitset = 9;
|
|
|
|
|
+
|
|
|
|
|
+/// Performs AES encryption operations with GCM or CTR ciphers.
|
|
|
|
|
+class PARQUET_EXPORT AesEncryptor {
|
|
|
|
|
+ public:
|
|
|
|
|
+ /// Can serve one key length only. Possible values: 16, 24, 32 bytes.
|
|
|
|
|
+ /// If write_length is true, prepend ciphertext length to the ciphertext
|
|
|
|
|
+ explicit AesEncryptor(ParquetCipher::type alg_id, int32_t key_len, bool metadata,
|
|
|
|
|
+ bool write_length = true);
|
|
|
|
|
+
|
|
|
|
|
+ static std::unique_ptr<AesEncryptor> Make(ParquetCipher::type alg_id, int32_t key_len,
|
|
|
|
|
+ bool metadata, bool write_length = true);
|
|
|
|
|
+
|
|
|
|
|
+ ~AesEncryptor();
|
|
|
|
|
+
|
|
|
|
|
+ /// The size of the ciphertext, for this cipher and the specified plaintext length.
|
|
|
|
|
+ [[nodiscard]] int32_t CiphertextLength(int64_t plaintext_len) const;
|
|
|
|
|
+
|
|
|
|
|
+ /// Encrypts plaintext with the key and aad. Key length is passed only for validation.
|
|
|
|
|
+ /// If different from value in constructor, exception will be thrown.
|
|
|
|
|
+ int32_t Encrypt(::arrow::util::span<const uint8_t> plaintext,
|
|
|
|
|
+ ::arrow::util::span<const uint8_t> key,
|
|
|
|
|
+ ::arrow::util::span<const uint8_t> aad,
|
|
|
|
|
+ ::arrow::util::span<uint8_t> ciphertext);
|
|
|
|
|
+
|
|
|
|
|
+ /// Encrypts plaintext footer, in order to compute footer signature (tag).
|
|
|
|
|
+ int32_t SignedFooterEncrypt(::arrow::util::span<const uint8_t> footer,
|
|
|
|
|
+ ::arrow::util::span<const uint8_t> key,
|
|
|
|
|
+ ::arrow::util::span<const uint8_t> aad,
|
|
|
|
|
+ ::arrow::util::span<const uint8_t> nonce,
|
|
|
|
|
+ ::arrow::util::span<uint8_t> encrypted_footer);
|
|
|
|
|
+
|
|
|
|
|
+ private:
|
|
|
|
|
+ // PIMPL Idiom
|
|
|
|
|
+ class AesEncryptorImpl;
|
|
|
|
|
+ std::unique_ptr<AesEncryptorImpl> impl_;
|
|
|
|
|
+};
|
|
|
|
|
+
|
|
|
|
|
+/// Performs AES decryption operations with GCM or CTR ciphers.
|
|
|
|
|
+class PARQUET_EXPORT AesDecryptor {
|
|
|
|
|
+ public:
|
|
|
|
|
+ /// \brief Construct an AesDecryptor
|
|
|
|
|
+ ///
|
|
|
|
|
+ /// \param alg_id the encryption algorithm to use
|
|
|
|
|
+ /// \param key_len key length. Possible values: 16, 24, 32 bytes.
|
|
|
|
|
+ /// \param metadata if true then this is a metadata decryptor
|
|
|
|
|
+ /// \param contains_length if true, expect ciphertext length prepended to the ciphertext
|
|
|
|
|
+ explicit AesDecryptor(ParquetCipher::type alg_id, int32_t key_len, bool metadata,
|
|
|
|
|
+ bool contains_length = true);
|
|
|
|
|
+
|
|
|
|
|
+ static std::unique_ptr<AesDecryptor> Make(ParquetCipher::type alg_id, int32_t key_len,
|
|
|
|
|
+ bool metadata);
|
|
|
|
|
+
|
|
|
|
|
+ ~AesDecryptor();
|
|
|
|
|
+
|
|
|
|
|
+ /// The size of the plaintext, for this cipher and the specified ciphertext length.
|
|
|
|
|
+ [[nodiscard]] int32_t PlaintextLength(int32_t ciphertext_len) const;
|
|
|
|
|
+
|
|
|
|
|
+ /// The size of the ciphertext, for this cipher and the specified plaintext length.
|
|
|
|
|
+ [[nodiscard]] int32_t CiphertextLength(int32_t plaintext_len) const;
|
|
|
|
|
+
|
|
|
|
|
+ /// Decrypts ciphertext with the key and aad. Key length is passed only for
|
|
|
|
|
+ /// validation. If different from value in constructor, exception will be thrown.
|
|
|
|
|
+ /// The caller is responsible for ensuring that the plaintext buffer is at least as
|
|
|
|
|
+ /// large as PlaintextLength(ciphertext_len).
|
|
|
|
|
+ int32_t Decrypt(::arrow::util::span<const uint8_t> ciphertext,
|
|
|
|
|
+ ::arrow::util::span<const uint8_t> key,
|
|
|
|
|
+ ::arrow::util::span<const uint8_t> aad,
|
|
|
|
|
+ ::arrow::util::span<uint8_t> plaintext);
|
|
|
|
|
+
|
|
|
|
|
+ private:
|
|
|
|
|
+ // PIMPL Idiom
|
|
|
|
|
+ class AesDecryptorImpl;
|
|
|
|
|
+ std::unique_ptr<AesDecryptorImpl> impl_;
|
|
|
|
|
+};
|
|
|
|
|
+
|
|
|
|
|
+std::string CreateModuleAad(const std::string& file_aad, int8_t module_type,
|
|
|
|
|
+ int16_t row_group_ordinal, int16_t column_ordinal,
|
|
|
|
|
+ int32_t page_ordinal);
|
|
|
|
|
+
|
|
|
|
|
+std::string CreateFooterAad(const std::string& aad_prefix_bytes);
|
|
|
|
|
+
|
|
|
|
|
+// Update last two bytes of page (or page header) module AAD
|
|
|
|
|
+void QuickUpdatePageAad(int32_t new_page_ordinal, std::string* AAD);
|
|
|
|
|
+
|
|
|
|
|
+// Wraps OpenSSL RAND_bytes function
|
|
|
|
|
+void RandBytes(unsigned char* buf, size_t num);
|
|
|
|
|
+
|
|
|
|
|
+// Ensure OpenSSL is initialized.
|
|
|
|
|
+//
|
|
|
|
|
+// This is only necessary in specific situations since OpenSSL otherwise
|
|
|
|
|
+// initializes itself automatically. For example, under Valgrind, a memory
|
|
|
|
|
+// leak will be reported if OpenSSL is initialized for the first time from
|
|
|
|
|
+// a worker thread; calling this function from the main thread prevents this.
|
|
|
|
|
+void EnsureBackendInitialized();
|
|
|
|
|
+
|
|
|
|
|
+} // namespace parquet::encryption
|
|
|
|
|
diff -ur ceph-19.2.2/src/s3select.orig/include/encryption_internal.h ceph-19.2.2/src/s3select/include/encryption_internal.h
|
|
|
|
|
--- ceph-19.2.2/src/s3select.orig/include/encryption_internal.h 2024-03-26 19:23:12.000000000 -0400
|
|
|
|
|
+++ ceph-19.2.2/src/s3select/include/encryption_internal.h 2025-07-07 13:08:00.745633149 -0400
|
|
|
|
|
@@ -17,98 +17,8 @@
|
|
|
|
|
|
|
|
|
|
#pragma once
|
|
|
|
|
|
|
|
|
|
-#include <memory>
|
|
|
|
|
-#include <string>
|
|
|
|
|
-#include <vector>
|
|
|
|
|
-
|
|
|
|
|
-#include "parquet/properties.h"
|
|
|
|
|
-#include "parquet/types.h"
|
|
|
|
|
-
|
|
|
|
|
-using parquet::ParquetCipher;
|
|
|
|
|
-
|
|
|
|
|
-namespace parquet {
|
|
|
|
|
-namespace encryption {
|
|
|
|
|
-
|
|
|
|
|
-constexpr int kGcmTagLength = 16;
|
|
|
|
|
-constexpr int kNonceLength = 12;
|
|
|
|
|
-
|
|
|
|
|
-// Module types
|
|
|
|
|
-constexpr int8_t kFooter = 0;
|
|
|
|
|
-constexpr int8_t kColumnMetaData = 1;
|
|
|
|
|
-constexpr int8_t kDataPage = 2;
|
|
|
|
|
-constexpr int8_t kDictionaryPage = 3;
|
|
|
|
|
-constexpr int8_t kDataPageHeader = 4;
|
|
|
|
|
-constexpr int8_t kDictionaryPageHeader = 5;
|
|
|
|
|
-constexpr int8_t kColumnIndex = 6;
|
|
|
|
|
-constexpr int8_t kOffsetIndex = 7;
|
|
|
|
|
-
|
|
|
|
|
-/// Performs AES encryption operations with GCM or CTR ciphers.
|
|
|
|
|
-class AesEncryptor {
|
|
|
|
|
- public:
|
|
|
|
|
- static AesEncryptor* Make(ParquetCipher::type alg_id, int key_len, bool metadata,
|
|
|
|
|
- std::vector<AesEncryptor*>* all_encryptors);
|
|
|
|
|
-
|
|
|
|
|
- ~AesEncryptor();
|
|
|
|
|
-
|
|
|
|
|
- /// Size difference between plaintext and ciphertext, for this cipher.
|
|
|
|
|
- int CiphertextSizeDelta();
|
|
|
|
|
-
|
|
|
|
|
- /// Encrypts plaintext with the key and aad. Key length is passed only for validation.
|
|
|
|
|
- /// If different from value in constructor, exception will be thrown.
|
|
|
|
|
- int Encrypt(const uint8_t* plaintext, int plaintext_len, const uint8_t* key,
|
|
|
|
|
- int key_len, const uint8_t* aad, int aad_len, uint8_t* ciphertext);
|
|
|
|
|
-
|
|
|
|
|
- /// Encrypts plaintext footer, in order to compute footer signature (tag).
|
|
|
|
|
- int SignedFooterEncrypt(const uint8_t* footer, int footer_len, const uint8_t* key,
|
|
|
|
|
- int key_len, const uint8_t* aad, int aad_len,
|
|
|
|
|
- const uint8_t* nonce, uint8_t* encrypted_footer);
|
|
|
|
|
-
|
|
|
|
|
- void WipeOut();
|
|
|
|
|
-
|
|
|
|
|
- private:
|
|
|
|
|
- /// Can serve one key length only. Possible values: 16, 24, 32 bytes.
|
|
|
|
|
- explicit AesEncryptor(ParquetCipher::type alg_id, int key_len, bool metadata);
|
|
|
|
|
- // PIMPL Idiom
|
|
|
|
|
- class AesEncryptorImpl;
|
|
|
|
|
- std::unique_ptr<AesEncryptorImpl> impl_;
|
|
|
|
|
-};
|
|
|
|
|
-
|
|
|
|
|
-/// Performs AES decryption operations with GCM or CTR ciphers.
|
|
|
|
|
-class AesDecryptor {
|
|
|
|
|
- public:
|
|
|
|
|
- static AesDecryptor* Make(ParquetCipher::type alg_id, int key_len, bool metadata,
|
|
|
|
|
- std::vector<AesDecryptor*>* all_decryptors);
|
|
|
|
|
-
|
|
|
|
|
- ~AesDecryptor();
|
|
|
|
|
- void WipeOut();
|
|
|
|
|
-
|
|
|
|
|
- /// Size difference between plaintext and ciphertext, for this cipher.
|
|
|
|
|
- int CiphertextSizeDelta();
|
|
|
|
|
-
|
|
|
|
|
- /// Decrypts ciphertext with the key and aad. Key length is passed only for
|
|
|
|
|
- /// validation. If different from value in constructor, exception will be thrown.
|
|
|
|
|
- int Decrypt(const uint8_t* ciphertext, int ciphertext_len, const uint8_t* key,
|
|
|
|
|
- int key_len, const uint8_t* aad, int aad_len, uint8_t* plaintext);
|
|
|
|
|
-
|
|
|
|
|
- private:
|
|
|
|
|
- /// Can serve one key length only. Possible values: 16, 24, 32 bytes.
|
|
|
|
|
- explicit AesDecryptor(ParquetCipher::type alg_id, int key_len, bool metadata);
|
|
|
|
|
- // PIMPL Idiom
|
|
|
|
|
- class AesDecryptorImpl;
|
|
|
|
|
- std::unique_ptr<AesDecryptorImpl> impl_;
|
|
|
|
|
-};
|
|
|
|
|
-
|
|
|
|
|
-std::string CreateModuleAad(const std::string& file_aad, int8_t module_type,
|
|
|
|
|
- int16_t row_group_ordinal, int16_t column_ordinal,
|
|
|
|
|
- int16_t page_ordinal);
|
|
|
|
|
-
|
|
|
|
|
-std::string CreateFooterAad(const std::string& aad_prefix_bytes);
|
|
|
|
|
-
|
|
|
|
|
-// Update last two bytes of page (or page header) module AAD
|
|
|
|
|
-void QuickUpdatePageAad(const std::string& AAD, int16_t new_page_ordinal);
|
|
|
|
|
-
|
|
|
|
|
-// Wraps OpenSSL RAND_bytes function
|
|
|
|
|
-void RandBytes(unsigned char* buf, int num);
|
|
|
|
|
-
|
|
|
|
|
-} // namespace encryption
|
|
|
|
|
-} // namespace parquet
|
|
|
|
|
+#if ARROW_VERSION_MAJOR < 20
|
|
|
|
|
+#include "encryption_internal_19.h"
|
|
|
|
|
+#else
|
|
|
|
|
+#include "encryption_internal_20.h"
|
|
|
|
|
+#endif
|
|
|
|
|
diff -ur ceph-19.2.2/src/s3select.orig/include/internal_file_decryptor_19.h ceph-19.2.2/src/s3select/include/internal_file_decryptor_19.h
|
|
|
|
|
--- ceph-19.2.2/src/s3select.orig/include/internal_file_decryptor_19.h 2025-07-07 13:17:58.730837390 -0400
|
|
|
|
|
+++ ceph-19.2.2/src/s3select/include/internal_file_decryptor_19.h 2025-07-07 13:08:00.746893764 -0400
|
|
|
|
|
@@ -0,0 +1,121 @@
|
|
|
|
|
+// Licensed to the Apache Software Foundation (ASF) under one
|
|
|
|
|
+// or more contributor license agreements. See the NOTICE file
|
|
|
|
|
+// distributed with this work for additional information
|
|
|
|
|
+// regarding copyright ownership. The ASF licenses this file
|
|
|
|
|
+// to you under the Apache License, Version 2.0 (the
|
|
|
|
|
+// "License"); you may not use this file except in compliance
|
|
|
|
|
+// with the License. You may obtain a copy of the License at
|
|
|
|
|
+//
|
|
|
|
|
+// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
+//
|
|
|
|
|
+// Unless required by applicable law or agreed to in writing,
|
|
|
|
|
+// software distributed under the License is distributed on an
|
|
|
|
|
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
|
|
|
+// KIND, either express or implied. See the License for the
|
|
|
|
|
+// specific language governing permissions and limitations
|
|
|
|
|
+// under the License.
|
|
|
|
|
+
|
|
|
|
|
+#pragma once
|
|
|
|
|
+
|
|
|
|
|
+#include <map>
|
|
|
|
|
+#include <memory>
|
|
|
|
|
+#include <string>
|
|
|
|
|
+#include <vector>
|
|
|
|
|
+
|
|
|
|
|
+#include "parquet/schema.h"
|
|
|
|
|
+
|
|
|
|
|
+namespace parquet {
|
|
|
|
|
+
|
|
|
|
|
+namespace encryption {
|
|
|
|
|
+class AesDecryptor;
|
|
|
|
|
+class AesEncryptor;
|
|
|
|
|
+} // namespace encryption
|
|
|
|
|
+
|
|
|
|
|
+class FileDecryptionProperties;
|
|
|
|
|
+
|
|
|
|
|
+class PARQUET_EXPORT Decryptor {
|
|
|
|
|
+ public:
|
|
|
|
|
+ Decryptor(encryption::AesDecryptor* decryptor, const std::string& key,
|
|
|
|
|
+ const std::string& file_aad, const std::string& aad,
|
|
|
|
|
+ ::arrow::MemoryPool* pool);
|
|
|
|
|
+
|
|
|
|
|
+ const std::string& file_aad() const { return file_aad_; }
|
|
|
|
|
+ void UpdateAad(const std::string& aad) { aad_ = aad; }
|
|
|
|
|
+ ::arrow::MemoryPool* pool() { return pool_; }
|
|
|
|
|
+
|
|
|
|
|
+ int CiphertextSizeDelta();
|
|
|
|
|
+ int Decrypt(const uint8_t* ciphertext, int ciphertext_len, uint8_t* plaintext);
|
|
|
|
|
+
|
|
|
|
|
+ private:
|
|
|
|
|
+ encryption::AesDecryptor* aes_decryptor_;
|
|
|
|
|
+ std::string key_;
|
|
|
|
|
+ std::string file_aad_;
|
|
|
|
|
+ std::string aad_;
|
|
|
|
|
+ ::arrow::MemoryPool* pool_;
|
|
|
|
|
+};
|
|
|
|
|
+
|
|
|
|
|
+class InternalFileDecryptor {
|
|
|
|
|
+ public:
|
|
|
|
|
+ explicit InternalFileDecryptor(FileDecryptionProperties* properties,
|
|
|
|
|
+ const std::string& file_aad,
|
|
|
|
|
+ ParquetCipher::type algorithm,
|
|
|
|
|
+ const std::string& footer_key_metadata,
|
|
|
|
|
+ ::arrow::MemoryPool* pool);
|
|
|
|
|
+
|
|
|
|
|
+ std::string& file_aad() { return file_aad_; }
|
|
|
|
|
+
|
|
|
|
|
+ std::string GetFooterKey();
|
|
|
|
|
+
|
|
|
|
|
+ ParquetCipher::type algorithm() { return algorithm_; }
|
|
|
|
|
+
|
|
|
|
|
+ std::string& footer_key_metadata() { return footer_key_metadata_; }
|
|
|
|
|
+
|
|
|
|
|
+ FileDecryptionProperties* properties() { return properties_; }
|
|
|
|
|
+
|
|
|
|
|
+ void WipeOutDecryptionKeys();
|
|
|
|
|
+
|
|
|
|
|
+ ::arrow::MemoryPool* pool() { return pool_; }
|
|
|
|
|
+
|
|
|
|
|
+ std::shared_ptr<Decryptor> GetFooterDecryptor();
|
|
|
|
|
+ std::shared_ptr<Decryptor> GetFooterDecryptorForColumnMeta(const std::string& aad = "");
|
|
|
|
|
+ std::shared_ptr<Decryptor> GetFooterDecryptorForColumnData(const std::string& aad = "");
|
|
|
|
|
+ std::shared_ptr<Decryptor> GetColumnMetaDecryptor(
|
|
|
|
|
+ const std::string& column_path, const std::string& column_key_metadata,
|
|
|
|
|
+ const std::string& aad = "");
|
|
|
|
|
+ std::shared_ptr<Decryptor> GetColumnDataDecryptor(
|
|
|
|
|
+ const std::string& column_path, const std::string& column_key_metadata,
|
|
|
|
|
+ const std::string& aad = "");
|
|
|
|
|
+
|
|
|
|
|
+ private:
|
|
|
|
|
+ FileDecryptionProperties* properties_;
|
|
|
|
|
+ // Concatenation of aad_prefix (if exists) and aad_file_unique
|
|
|
|
|
+ std::string file_aad_;
|
|
|
|
|
+ std::map<std::string, std::shared_ptr<Decryptor>> column_data_map_;
|
|
|
|
|
+ std::map<std::string, std::shared_ptr<Decryptor>> column_metadata_map_;
|
|
|
|
|
+
|
|
|
|
|
+ std::shared_ptr<Decryptor> footer_metadata_decryptor_;
|
|
|
|
|
+ std::shared_ptr<Decryptor> footer_data_decryptor_;
|
|
|
|
|
+ ParquetCipher::type algorithm_;
|
|
|
|
|
+ std::string footer_key_metadata_;
|
|
|
|
|
+ std::vector<encryption::AesDecryptor*> all_decryptors_;
|
|
|
|
|
+
|
|
|
|
|
+ /// Key must be 16, 24 or 32 bytes in length. Thus there could be up to three
|
|
|
|
|
+ // types of meta_decryptors and data_decryptors.
|
|
|
|
|
+ std::unique_ptr<encryption::AesDecryptor> meta_decryptor_[3];
|
|
|
|
|
+ std::unique_ptr<encryption::AesDecryptor> data_decryptor_[3];
|
|
|
|
|
+
|
|
|
|
|
+ ::arrow::MemoryPool* pool_;
|
|
|
|
|
+
|
|
|
|
|
+ std::shared_ptr<Decryptor> GetFooterDecryptor(const std::string& aad, bool metadata);
|
|
|
|
|
+ std::shared_ptr<Decryptor> GetColumnDecryptor(const std::string& column_path,
|
|
|
|
|
+ const std::string& column_key_metadata,
|
|
|
|
|
+ const std::string& aad,
|
|
|
|
|
+ bool metadata = false);
|
|
|
|
|
+
|
|
|
|
|
+ encryption::AesDecryptor* GetMetaAesDecryptor(size_t key_size);
|
|
|
|
|
+ encryption::AesDecryptor* GetDataAesDecryptor(size_t key_size);
|
|
|
|
|
+
|
|
|
|
|
+ int MapKeyLenToDecryptorArrayIndex(int key_len);
|
|
|
|
|
+};
|
|
|
|
|
+
|
|
|
|
|
+} // namespace parquet
|
|
|
|
|
diff -ur ceph-19.2.2/src/s3select.orig/include/internal_file_decryptor_20.h ceph-19.2.2/src/s3select/include/internal_file_decryptor_20.h
|
|
|
|
|
--- ceph-19.2.2/src/s3select.orig/include/internal_file_decryptor_20.h 2025-07-07 13:17:58.730837390 -0400
|
|
|
|
|
+++ ceph-19.2.2/src/s3select/include/internal_file_decryptor_20.h 2025-07-07 13:08:00.747082281 -0400
|
|
|
|
|
@@ -0,0 +1,148 @@
|
|
|
|
|
+// Licensed to the Apache Software Foundation (ASF) under one
|
|
|
|
|
+// or more contributor license agreements. See the NOTICE file
|
|
|
|
|
+// distributed with this work for additional information
|
|
|
|
|
+// regarding copyright ownership. The ASF licenses this file
|
|
|
|
|
+// to you under the Apache License, Version 2.0 (the
|
|
|
|
|
+// "License"); you may not use this file except in compliance
|
|
|
|
|
+// with the License. You may obtain a copy of the License at
|
|
|
|
|
+//
|
|
|
|
|
+// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
+//
|
|
|
|
|
+// Unless required by applicable law or agreed to in writing,
|
|
|
|
|
+// software distributed under the License is distributed on an
|
|
|
|
|
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
|
|
|
+// KIND, either express or implied. See the License for the
|
|
|
|
|
+// specific language governing permissions and limitations
|
|
|
|
|
+// under the License.
|
|
|
|
|
+
|
|
|
|
|
+#pragma once
|
|
|
|
|
+
|
|
|
|
|
+#include <memory>
|
|
|
|
|
+#include <mutex>
|
|
|
|
|
+#include <string>
|
|
|
|
|
+#include <vector>
|
|
|
|
|
+
|
|
|
|
|
+#include "parquet/schema.h"
|
|
|
|
|
+
|
|
|
|
|
+namespace parquet {
|
|
|
|
|
+
|
|
|
|
|
+namespace encryption {
|
|
|
|
|
+class AesDecryptor;
|
|
|
|
|
+class AesEncryptor;
|
|
|
|
|
+} // namespace encryption
|
|
|
|
|
+
|
|
|
|
|
+class ColumnCryptoMetaData;
|
|
|
|
|
+class FileDecryptionProperties;
|
|
|
|
|
+
|
|
|
|
|
+// An object handling decryption using well-known encryption parameters
|
|
|
|
|
+//
|
|
|
|
|
+// CAUTION: Decryptor objects are not thread-safe.
|
|
|
|
|
+class PARQUET_EXPORT Decryptor {
|
|
|
|
|
+ public:
|
|
|
|
|
+ Decryptor(std::unique_ptr<encryption::AesDecryptor> decryptor, const std::string& key,
|
|
|
|
|
+ const std::string& file_aad, const std::string& aad,
|
|
|
|
|
+ ::arrow::MemoryPool* pool);
|
|
|
|
|
+ ~Decryptor();
|
|
|
|
|
+
|
|
|
|
|
+ const std::string& file_aad() const { return file_aad_; }
|
|
|
|
|
+ void UpdateAad(const std::string& aad) { aad_ = aad; }
|
|
|
|
|
+ ::arrow::MemoryPool* pool() { return pool_; }
|
|
|
|
|
+
|
|
|
|
|
+ [[nodiscard]] int32_t PlaintextLength(int32_t ciphertext_len) const;
|
|
|
|
|
+ [[nodiscard]] int32_t CiphertextLength(int32_t plaintext_len) const;
|
|
|
|
|
+ int32_t Decrypt(::arrow::util::span<const uint8_t> ciphertext,
|
|
|
|
|
+ ::arrow::util::span<uint8_t> plaintext);
|
|
|
|
|
+
|
|
|
|
|
+ private:
|
|
|
|
|
+ std::unique_ptr<encryption::AesDecryptor> aes_decryptor_;
|
|
|
|
|
+ std::string key_;
|
|
|
|
|
+ std::string file_aad_;
|
|
|
|
|
+ std::string aad_;
|
|
|
|
|
+ ::arrow::MemoryPool* pool_;
|
|
|
|
|
+};
|
|
|
|
|
+
|
|
|
|
|
+class InternalFileDecryptor {
|
|
|
|
|
+ public:
|
|
|
|
|
+ explicit InternalFileDecryptor(std::shared_ptr<FileDecryptionProperties> properties,
|
|
|
|
|
+ const std::string& file_aad,
|
|
|
|
|
+ ParquetCipher::type algorithm,
|
|
|
|
|
+ const std::string& footer_key_metadata,
|
|
|
|
|
+ ::arrow::MemoryPool* pool);
|
|
|
|
|
+
|
|
|
|
|
+ const std::string& file_aad() const { return file_aad_; }
|
|
|
|
|
+
|
|
|
|
|
+ std::string GetFooterKey();
|
|
|
|
|
+
|
|
|
|
|
+ ParquetCipher::type algorithm() const { return algorithm_; }
|
|
|
|
|
+
|
|
|
|
|
+ const std::string& footer_key_metadata() const { return footer_key_metadata_; }
|
|
|
|
|
+
|
|
|
|
|
+ const std::shared_ptr<FileDecryptionProperties>& properties() const {
|
|
|
|
|
+ return properties_;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ ::arrow::MemoryPool* pool() const { return pool_; }
|
|
|
|
|
+
|
|
|
|
|
+ // Get a Decryptor instance for the Parquet footer
|
|
|
|
|
+ std::unique_ptr<Decryptor> GetFooterDecryptor();
|
|
|
|
|
+
|
|
|
|
|
+ // Get a Decryptor instance for column chunk metadata.
|
|
|
|
|
+ std::unique_ptr<Decryptor> GetColumnMetaDecryptor(
|
|
|
|
|
+ const std::string& column_path, const std::string& column_key_metadata,
|
|
|
|
|
+ const std::string& aad = "") {
|
|
|
|
|
+ return GetColumnDecryptor(column_path, column_key_metadata, aad, /*metadata=*/true);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ // Get a Decryptor instance for column chunk data.
|
|
|
|
|
+ std::unique_ptr<Decryptor> GetColumnDataDecryptor(
|
|
|
|
|
+ const std::string& column_path, const std::string& column_key_metadata,
|
|
|
|
|
+ const std::string& aad = "") {
|
|
|
|
|
+ return GetColumnDecryptor(column_path, column_key_metadata, aad, /*metadata=*/false);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ // Get a Decryptor factory for column chunk metadata.
|
|
|
|
|
+ //
|
|
|
|
|
+ // This is typically useful if multi-threaded decryption is expected.
|
|
|
|
|
+ // This is a static function as it accepts a null `InternalFileDecryptor*`
|
|
|
|
|
+ // argument if the column is not encrypted.
|
|
|
|
|
+ static std::function<std::unique_ptr<Decryptor>()> GetColumnMetaDecryptorFactory(
|
|
|
|
|
+ InternalFileDecryptor*, const ColumnCryptoMetaData* crypto_metadata,
|
|
|
|
|
+ const std::string& aad = "");
|
|
|
|
|
+ // Get a Decryptor factory for column chunk data.
|
|
|
|
|
+ //
|
|
|
|
|
+ // This is typically useful if multi-threaded decryption is expected.
|
|
|
|
|
+ // This is a static function as it accepts a null `InternalFileDecryptor*`
|
|
|
|
|
+ // argument if the column is not encrypted.
|
|
|
|
|
+ static std::function<std::unique_ptr<Decryptor>()> GetColumnDataDecryptorFactory(
|
|
|
|
|
+ InternalFileDecryptor*, const ColumnCryptoMetaData* crypto_metadata,
|
|
|
|
|
+ const std::string& aad = "");
|
|
|
|
|
+
|
|
|
|
|
+ private:
|
|
|
|
|
+ std::shared_ptr<FileDecryptionProperties> properties_;
|
|
|
|
|
+ // Concatenation of aad_prefix (if exists) and aad_file_unique
|
|
|
|
|
+ std::string file_aad_;
|
|
|
|
|
+ ParquetCipher::type algorithm_;
|
|
|
|
|
+ std::string footer_key_metadata_;
|
|
|
|
|
+ ::arrow::MemoryPool* pool_;
|
|
|
|
|
+
|
|
|
|
|
+ // Protects footer_key_ updates
|
|
|
|
|
+ std::mutex mutex_;
|
|
|
|
|
+ std::string footer_key_;
|
|
|
|
|
+
|
|
|
|
|
+ std::string GetColumnKey(const std::string& column_path,
|
|
|
|
|
+ const std::string& column_key_metadata);
|
|
|
|
|
+
|
|
|
|
|
+ std::unique_ptr<Decryptor> GetFooterDecryptor(const std::string& aad, bool metadata);
|
|
|
|
|
+
|
|
|
|
|
+ std::unique_ptr<Decryptor> GetColumnDecryptor(const std::string& column_path,
|
|
|
|
|
+ const std::string& column_key_metadata,
|
|
|
|
|
+ const std::string& aad, bool metadata);
|
|
|
|
|
+
|
|
|
|
|
+ std::function<std::unique_ptr<Decryptor>()> GetColumnDecryptorFactory(
|
|
|
|
|
+ const ColumnCryptoMetaData* crypto_metadata, const std::string& aad, bool metadata);
|
|
|
|
|
+};
|
|
|
|
|
+
|
|
|
|
|
+void UpdateDecryptor(Decryptor* decryptor, int16_t row_group_ordinal,
|
|
|
|
|
+ int16_t column_ordinal, int8_t module_type);
|
|
|
|
|
+
|
|
|
|
|
+} // namespace parquet
|
|
|
|
|
diff -ur ceph-19.2.2/src/s3select.orig/include/internal_file_decryptor.h ceph-19.2.2/src/s3select/include/internal_file_decryptor.h
|
|
|
|
|
--- ceph-19.2.2/src/s3select.orig/include/internal_file_decryptor.h 2024-03-26 19:23:12.000000000 -0400
|
|
|
|
|
+++ ceph-19.2.2/src/s3select/include/internal_file_decryptor.h 2025-07-07 13:08:00.746720961 -0400
|
|
|
|
|
@@ -17,105 +17,8 @@
|
|
|
|
|
|
|
|
|
|
#pragma once
|
|
|
|
|
|
|
|
|
|
-#include <map>
|
|
|
|
|
-#include <memory>
|
|
|
|
|
-#include <string>
|
|
|
|
|
-#include <vector>
|
|
|
|
|
-
|
|
|
|
|
-#include "parquet/schema.h"
|
|
|
|
|
-
|
|
|
|
|
-namespace parquet {
|
|
|
|
|
-
|
|
|
|
|
-namespace encryption {
|
|
|
|
|
-class AesDecryptor;
|
|
|
|
|
-class AesEncryptor;
|
|
|
|
|
-} // namespace encryption
|
|
|
|
|
-
|
|
|
|
|
-class FileDecryptionProperties;
|
|
|
|
|
-
|
|
|
|
|
-class PARQUET_EXPORT Decryptor {
|
|
|
|
|
- public:
|
|
|
|
|
- Decryptor(encryption::AesDecryptor* decryptor, const std::string& key,
|
|
|
|
|
- const std::string& file_aad, const std::string& aad,
|
|
|
|
|
- ::arrow::MemoryPool* pool);
|
|
|
|
|
-
|
|
|
|
|
- const std::string& file_aad() const { return file_aad_; }
|
|
|
|
|
- void UpdateAad(const std::string& aad) { aad_ = aad; }
|
|
|
|
|
- ::arrow::MemoryPool* pool() { return pool_; }
|
|
|
|
|
-
|
|
|
|
|
- int CiphertextSizeDelta();
|
|
|
|
|
- int Decrypt(const uint8_t* ciphertext, int ciphertext_len, uint8_t* plaintext);
|
|
|
|
|
-
|
|
|
|
|
- private:
|
|
|
|
|
- encryption::AesDecryptor* aes_decryptor_;
|
|
|
|
|
- std::string key_;
|
|
|
|
|
- std::string file_aad_;
|
|
|
|
|
- std::string aad_;
|
|
|
|
|
- ::arrow::MemoryPool* pool_;
|
|
|
|
|
-};
|
|
|
|
|
-
|
|
|
|
|
-class InternalFileDecryptor {
|
|
|
|
|
- public:
|
|
|
|
|
- explicit InternalFileDecryptor(FileDecryptionProperties* properties,
|
|
|
|
|
- const std::string& file_aad,
|
|
|
|
|
- ParquetCipher::type algorithm,
|
|
|
|
|
- const std::string& footer_key_metadata,
|
|
|
|
|
- ::arrow::MemoryPool* pool);
|
|
|
|
|
-
|
|
|
|
|
- std::string& file_aad() { return file_aad_; }
|
|
|
|
|
-
|
|
|
|
|
- std::string GetFooterKey();
|
|
|
|
|
-
|
|
|
|
|
- ParquetCipher::type algorithm() { return algorithm_; }
|
|
|
|
|
-
|
|
|
|
|
- std::string& footer_key_metadata() { return footer_key_metadata_; }
|
|
|
|
|
-
|
|
|
|
|
- FileDecryptionProperties* properties() { return properties_; }
|
|
|
|
|
-
|
|
|
|
|
- void WipeOutDecryptionKeys();
|
|
|
|
|
-
|
|
|
|
|
- ::arrow::MemoryPool* pool() { return pool_; }
|
|
|
|
|
-
|
|
|
|
|
- std::shared_ptr<Decryptor> GetFooterDecryptor();
|
|
|
|
|
- std::shared_ptr<Decryptor> GetFooterDecryptorForColumnMeta(const std::string& aad = "");
|
|
|
|
|
- std::shared_ptr<Decryptor> GetFooterDecryptorForColumnData(const std::string& aad = "");
|
|
|
|
|
- std::shared_ptr<Decryptor> GetColumnMetaDecryptor(
|
|
|
|
|
- const std::string& column_path, const std::string& column_key_metadata,
|
|
|
|
|
- const std::string& aad = "");
|
|
|
|
|
- std::shared_ptr<Decryptor> GetColumnDataDecryptor(
|
|
|
|
|
- const std::string& column_path, const std::string& column_key_metadata,
|
|
|
|
|
- const std::string& aad = "");
|
|
|
|
|
-
|
|
|
|
|
- private:
|
|
|
|
|
- FileDecryptionProperties* properties_;
|
|
|
|
|
- // Concatenation of aad_prefix (if exists) and aad_file_unique
|
|
|
|
|
- std::string file_aad_;
|
|
|
|
|
- std::map<std::string, std::shared_ptr<Decryptor>> column_data_map_;
|
|
|
|
|
- std::map<std::string, std::shared_ptr<Decryptor>> column_metadata_map_;
|
|
|
|
|
-
|
|
|
|
|
- std::shared_ptr<Decryptor> footer_metadata_decryptor_;
|
|
|
|
|
- std::shared_ptr<Decryptor> footer_data_decryptor_;
|
|
|
|
|
- ParquetCipher::type algorithm_;
|
|
|
|
|
- std::string footer_key_metadata_;
|
|
|
|
|
- std::vector<encryption::AesDecryptor*> all_decryptors_;
|
|
|
|
|
-
|
|
|
|
|
- /// Key must be 16, 24 or 32 bytes in length. Thus there could be up to three
|
|
|
|
|
- // types of meta_decryptors and data_decryptors.
|
|
|
|
|
- std::unique_ptr<encryption::AesDecryptor> meta_decryptor_[3];
|
|
|
|
|
- std::unique_ptr<encryption::AesDecryptor> data_decryptor_[3];
|
|
|
|
|
-
|
|
|
|
|
- ::arrow::MemoryPool* pool_;
|
|
|
|
|
-
|
|
|
|
|
- std::shared_ptr<Decryptor> GetFooterDecryptor(const std::string& aad, bool metadata);
|
|
|
|
|
- std::shared_ptr<Decryptor> GetColumnDecryptor(const std::string& column_path,
|
|
|
|
|
- const std::string& column_key_metadata,
|
|
|
|
|
- const std::string& aad,
|
|
|
|
|
- bool metadata = false);
|
|
|
|
|
-
|
|
|
|
|
- encryption::AesDecryptor* GetMetaAesDecryptor(size_t key_size);
|
|
|
|
|
- encryption::AesDecryptor* GetDataAesDecryptor(size_t key_size);
|
|
|
|
|
-
|
|
|
|
|
- int MapKeyLenToDecryptorArrayIndex(int key_len);
|
|
|
|
|
-};
|
|
|
|
|
-
|
|
|
|
|
-} // namespace parquet
|
|
|
|
|
+#if ARROW_VERSION_MAJOR < 20
|
|
|
|
|
+#include "internal_file_decryptor_19.h"
|
|
|
|
|
+#else
|
|
|
|
|
+#include "internal_file_decryptor_20.h"
|
|
|
|
|
+#endif
|
|
|
|
|
diff -ur ceph-19.2.2/src/s3select.orig/include/s3select_parquet_intrf.h ceph-19.2.2/src/s3select/include/s3select_parquet_intrf.h
|
|
|
|
|
--- ceph-19.2.2/src/s3select.orig/include/s3select_parquet_intrf.h 2024-03-26 19:23:12.000000000 -0400
|
|
|
|
|
+++ ceph-19.2.2/src/s3select/include/s3select_parquet_intrf.h 2025-07-07 13:08:00.747275733 -0400
|
|
|
|
|
@@ -983,6 +1002,7 @@
|
|
|
|
|
throw ParquetException("Encrypted files cannot contain more than 32767 row groups");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
+#if ARROW_VERSION_MAJOR < 20
|
|
|
|
|
// The column is encrypted
|
|
|
|
|
std::shared_ptr<::parquet::Decryptor> meta_decryptor;
|
|
|
|
|
std::shared_ptr<Decryptor> data_decryptor;
|
|
|
|
|
@@ -1016,6 +1036,25 @@
|
|
|
|
|
false,
|
|
|
|
|
#endif
|
|
|
|
|
properties_.memory_pool(), &ctx);
|
|
|
|
|
+#else
|
|
|
|
|
+ // Arrow 20+ version uses factory functions instead of shared_ptr for decryptors
|
|
|
|
|
+ std::function<std::unique_ptr<Decryptor>()> meta_decryptor_factory =
|
|
|
|
|
+ InternalFileDecryptor::GetColumnMetaDecryptorFactory(file_decryptor_.get(), crypto_metadata.get());
|
|
|
|
|
+ std::function<std::unique_ptr<Decryptor>()> data_decryptor_factory =
|
|
|
|
|
+ InternalFileDecryptor::GetColumnDataDecryptorFactory(file_decryptor_.get(), crypto_metadata.get());
|
|
|
|
|
+
|
|
|
|
|
+ const CryptoContext ctx {
|
|
|
|
|
+ col->has_dictionary_page(),
|
|
|
|
|
+ row_group_ordinal_,
|
|
|
|
|
+ static_cast<int16_t>(i),
|
|
|
|
|
+ meta_decryptor_factory,
|
|
|
|
|
+ data_decryptor_factory,
|
|
|
|
|
+ };
|
|
|
|
|
+
|
|
|
|
|
+ return PageReader::Open(stream, col->num_values(), col->compression(),
|
|
|
|
|
+ false,
|
|
|
|
|
+ properties_.memory_pool(), &ctx);
|
|
|
|
|
+#endif
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
private:
|
|
|
|
|
@@ -1052,7 +1091,9 @@
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void Close() override {
|
|
|
|
|
+#if ARROW_VERSION_MAJOR < 20
|
|
|
|
|
if (file_decryptor_) file_decryptor_->WipeOutDecryptionKeys();
|
|
|
|
|
+#endif
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
std::shared_ptr<RowGroupReader> GetRowGroup(int i) override {
|
|
|
|
|
@@ -1230,9 +1271,17 @@
|
|
|
|
|
// Handle AAD prefix
|
|
|
|
|
EncryptionAlgorithm algo = file_crypto_metadata->encryption_algorithm();
|
|
|
|
|
std::string file_aad = HandleAadPrefix(file_decryption_properties, algo);
|
|
|
|
|
+#if ARROW_VERSION_MAJOR < 20
|
|
|
|
|
file_decryptor_ = std::make_shared<::parquet::InternalFileDecryptor>(
|
|
|
|
|
file_decryption_properties, file_aad, algo.algorithm,
|
|
|
|
|
file_crypto_metadata->key_metadata(), properties_.memory_pool());
|
|
|
|
|
+#else
|
|
|
|
|
+ // Arrow 20+ takes a shared_ptr to FileDecryptionProperties
|
|
|
|
|
+ file_decryptor_ = std::make_shared<::parquet::InternalFileDecryptor>(
|
|
|
|
|
+ std::shared_ptr<FileDecryptionProperties>(file_decryption_properties),
|
|
|
|
|
+ file_aad, algo.algorithm,
|
|
|
|
|
+ file_crypto_metadata->key_metadata(), properties_.memory_pool());
|
|
|
|
|
+#endif
|
|
|
|
|
|
|
|
|
|
int64_t metadata_offset = source_size_ - kFooterSize - footer_len + crypto_metadata_len;
|
|
|
|
|
uint32_t metadata_len = footer_len - crypto_metadata_len;
|
|
|
|
|
@@ -1259,9 +1312,18 @@
|
|
|
|
|
EncryptionAlgorithm algo = file_metadata_->encryption_algorithm();
|
|
|
|
|
// Handle AAD prefix
|
|
|
|
|
std::string file_aad = HandleAadPrefix(file_decryption_properties, algo);
|
|
|
|
|
+#if ARROW_VERSION_MAJOR < 20
|
|
|
|
|
file_decryptor_ = std::make_shared<::parquet::InternalFileDecryptor>(
|
|
|
|
|
file_decryption_properties, file_aad, algo.algorithm,
|
|
|
|
|
file_metadata_->footer_signing_key_metadata(), properties_.memory_pool());
|
|
|
|
|
+#else
|
|
|
|
|
+ // Arrow 20+ takes a shared_ptr to FileDecryptionProperties
|
|
|
|
|
+ file_decryptor_ = std::make_shared<::parquet::InternalFileDecryptor>(
|
|
|
|
|
+ std::shared_ptr<FileDecryptionProperties>(file_decryption_properties),
|
|
|
|
|
+ file_aad, algo.algorithm,
|
|
|
|
|
+ file_metadata_->footer_signing_key_metadata(), properties_.memory_pool());
|
|
|
|
|
+ // In Arrow 20+, no need to set file_decryptor in metadata
|
|
|
|
|
+#endif
|
|
|
|
|
// set the InternalFileDecryptor in the metadata as well, as it's used
|
|
|
|
|
// for signature verification and for ColumnChunkMetaData creation.
|
|
|
|
|
#if GAL_set_file_decryptor_declare_private
|