Feature: AES CTR encryption (#6)

* mostly compiling version of env_encrypt_2.h ... one std::move error ... fix next

* cleans up build (must add export EXTRA_LDFLAGS=-lcrypto -lssl manually)

* hmm, somebody messed with SYNC_POINT defines ... and made debug unbuildable ... unit tests too

* Add the original encryption Env to the env_basic_test unit test suite.

* I lied.  Forced to clean up Java style code in env_encryption by moving declaration half to env_encryption.h to be able to unit test new OpenSSL encryption.  What a pain.

* Going back to original layout.  Rebuilt env_encryption.h/.cc into proper C++ declaration / definition split to allow proper inheritance (resuse).

* first batch of tests, other than one for file size, work with new encryption code that does not yet encrypt (infrastructure validation).

* ok, GetFileSize() corrected (slow but corrected).  Considering whether or not to do same for GetChildrenFileAttributes

* and add GetChildrenFileAttributes() update

* Activate AESBlockAccessCipherStream.  Remove dead code from original copying of env_encryption.cc.  Unit test works.

* change unique_ptr with deleter to traditional pointer code.  Circle build did not want to compile it.  And add some error checking to EncryptBlock()

* use EVP_MD_CTX_create/destroy instead of new/free.  This is openssl 1.0 syntax that is compatible in openssl 1.1

* move the definition of Sha1Description_t(std::string &) to .cc file in hopes of eliminating link issues in starrocks unit tests

* again move some AES stuff from .h to .cc

* add helper constructor to AesCtrKey_t (NOT TESTED). add IsValid() to Sha1Description and AesCtrKey_t

* attempt to make initialization easier with Sha1Description_t as const.

* const was a really bad idea

* need explicit copy constructor with move disabled

* removed delete of move constructor ... removal seems suspect

* add env_encrypt_2_test to builds.  Test Sha1Description_t.

* saving for safety.  first NIST AES case matches.  code is in hack state.  will clean and add other cases tomorrow.

* code clean up within EncryptBlock.  push to see if circle compiles

* add remaining NIST cases.

* added operator== for unit testing

* remove non-portable byteswap.h ... not using it anymore ... and breaks OSX build

* attempt include fix for osx

* make openssl dependency OS specific

* backport files used in Facebook/rocksdb PR

* rename our env_encrypt_2 to more rocksdb-like env_encrypt2

* create conditional build of EnvEncrypt2 based on flag ROCKSDB_OPENSSL_AES_CTR

* linux library loader code.  not integrated.  not yet supporting OSX

* Linux library load ready for libcrypto SHA1 and RAND functions (includes unit tests).  AES CTR functions next.

* Add remaining functions from libcrypto that are used in EncryptedEnv2

* slight change to have .dylib names instead of .so names on OSX build

* Create EncryptedEnv2::WriteKey_t and ReadKeys_t to simplify look of code.

* create EncryptedEnv2::Default() to help time static loading of libcrypto.

* clang-format applied

* remove conditional openssl from OSX build

* hmm, missed removing include files for openssl

* Revert "hmm, missed removing include files for openssl"

This reverts commit e22a1f6.

* Revert "remove conditional openssl from OSX build"

This reverts commit 4eef8d4.

* address PR comments from Alex

Co-authored-by: matthewvon <[email protected]>
Co-authored-by: MatthewVon <[email protected]>

CMakeLists.txt

@@ -522,6 +522,7 @@ set(SOURCES
         env/env.cc
         env/env_chroot.cc
         env/env_encryption.cc
+        env/env_encrypt2.cc
         env/env_hdfs.cc
         env/mock_env.cc
         memtable/alloc_tracker.cc
@@ -918,6 +919,7 @@ if(WITH_TESTS)
         db/write_callback_test.cc
         db/write_controller_test.cc
         env/env_basic_test.cc
+        env/env_encrypt2_test.cc
         env/env_test.cc
         env/mock_env_test.cc
         memtable/inlineskiplist_test.cc
@@ -956,6 +958,7 @@ if(WITH_TESTS)
         util/hash_test.cc
         util/heap_test.cc
         util/rate_limiter_test.cc
+        util/library_loader_test.cc
         util/repeatable_thread_test.cc
         util/slice_transform_test.cc
         util/timer_queue_test.cc

Makefile

@@ -419,8 +419,10 @@ TESTS = \
 	coding_test \
 	inlineskiplist_test \
 	env_basic_test \
+	env_encrypt2_test \
 	env_test \
 	hash_test \
+	library_loader_test \
 	thread_local_test \
 	rate_limiter_test \
 	perf_context_test \
@@ -1135,6 +1137,9 @@ coding_test: util/coding_test.o $(LIBOBJECTS) $(TESTHARNESS)
 hash_test: util/hash_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
+library_loader_test: util/library_loader_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 option_change_migration_test: utilities/option_change_migration/option_change_migration_test.o db/db_test_util.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
@@ -1306,6 +1311,9 @@ sim_cache_test: utilities/simulator_cache/sim_cache_test.o db/db_test_util.o $(L
 spatial_db_test: utilities/spatialdb/spatial_db_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 
+env_encrypt2_test: env/env_encrypt2_test.o $(LIBOBJECTS) $(TESTHARNESS)
+	$(AM_LINK)
+
 env_mirror_test: utilities/env_mirror_test.o $(LIBOBJECTS) $(TESTHARNESS)
 	$(AM_LINK)
 

TARGETS

@@ -139,6 +139,7 @@ cpp_library(
         "env/env.cc",
         "env/env_chroot.cc",
         "env/env_encryption.cc",
+        "env/env_encrypt2.cc",
         "env/env_hdfs.cc",
         "env/env_posix.cc",
         "env/io_posix.cc",
@@ -227,6 +228,7 @@ cpp_library(
         "util/filter_policy.cc",
         "util/hash.cc",
         "util/jemalloc_nodump_allocator.cc",
+        "util/library_loader.cc",
         "util/log_buffer.cc",
         "util/murmurhash.cc",
         "util/random.cc",
@@ -702,6 +704,11 @@ ROCKS_TESTS = [
         "env/env_basic_test.cc",
         "serial",
     ],
+    [
+        "env_encrypt2_test",
+        "env/env_encrypt2_test.cc",
+        "serial",
+    ],
     [
         "env_test",
         "env/env_test.cc",
@@ -812,6 +819,11 @@ ROCKS_TESTS = [
         "tools/ldb_cmd_test.cc",
         "serial",
     ],
+    [
+        "library_loader_test",
+        "util/library_loader_test.cc",
+        "serial",
+    ],
     [
         "listener_test",
         "db/listener_test.cc",

env/BUILD

@@ -19,6 +19,7 @@ PLATFORM_HDRS = select({
 COMMON_SRCS = [
     "env.cc",
     "env_encryption.cc",
+    "env_encrypt2.cc",
 ]
 
 PLATFORM_SRCS = select({
@@ -100,6 +101,17 @@ constrained_test(
     ],
 )
 
+constrained_test(
+    name = "env_encrypt2_test",
+    srcs = ["env_encrypt2_test.cc"],
+    deps = [
+        ":env",
+        ":test_utils",
+        "//util",
+        "//util:test_utils",
+    ],
+)
+
 constrained_test(
     name = "env_test",
     srcs = ["env_test.cc"],

env/env_basic_test.cc

@@ -1,17 +1,19 @@
 // Copyright (c) 2011 The LevelDB Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file. See the AUTHORS file for names of contributors.
+//
+// Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
 
+#include <algorithm>
 #include <memory>
 #include <string>
 #include <vector>
-#include <algorithm>
 
 #include "env/mock_env.h"
 #include "rocksdb/env.h"
+#include "rocksdb/env_encryption.h"
 #include "rocksdb/utilities/object_registry.h"
 #include "util/testharness.h"
-
 namespace rocksdb {
 
 // Normalizes trivial differences across Envs such that these test cases can
@@ -21,8 +23,8 @@ class NormalizingEnvWrapper : public EnvWrapper {
   explicit NormalizingEnvWrapper(Env* base) : EnvWrapper(base) {}
 
   // Removes . and .. from directory listing
-  virtual Status GetChildren(const std::string& dir,
-                             std::vector<std::string>* result) override {
+  Status GetChildren(const std::string& dir,
+                     std::vector<std::string>* result) override {
     Status status = EnvWrapper::GetChildren(dir, result);
     if (status.ok()) {
       result->erase(std::remove_if(result->begin(), result->end(),
@@ -35,7 +37,7 @@ class NormalizingEnvWrapper : public EnvWrapper {
   }
 
   // Removes . and .. from directory listing
-  virtual Status GetChildrenFileAttributes(
+  Status GetChildrenFileAttributes(
       const std::string& dir, std::vector<FileAttributes>* result) override {
     Status status = EnvWrapper::GetChildrenFileAttributes(dir, result);
     if (status.ok()) {
@@ -60,11 +62,9 @@ class EnvBasicTestWithParam : public testing::Test,
     test_dir_ = test::PerThreadDBPath(env_, "env_basic_test");
   }
 
-  void SetUp() {
-    env_->CreateDirIfMissing(test_dir_);
-  }
+  void SetUp() override { env_->CreateDirIfMissing(test_dir_); }
 
-  void TearDown() {
+  void TearDown() override {
     std::vector<std::string> files;
     env_->GetChildren(test_dir_, &files);
     for (const auto& file : files) {
@@ -90,6 +90,19 @@ INSTANTIATE_TEST_CASE_P(EnvDefault, EnvMoreTestWithParam,
 static std::unique_ptr<Env> mock_env(new MockEnv(Env::Default()));
 INSTANTIATE_TEST_CASE_P(MockEnv, EnvBasicTestWithParam,
                         ::testing::Values(mock_env.get()));
+
+// next statements run env test against default encryption code.
+static ROT13BlockCipher encrypt_block_rot13(32);
+
+static CTREncryptionProvider encrypt_provider_ctr(encrypt_block_rot13);
+
+static std::unique_ptr<Env> encrypt_env(new NormalizingEnvWrapper(
+    NewEncryptedEnv(Env::Default(), &encrypt_provider_ctr)));
+INSTANTIATE_TEST_CASE_P(EncryptedEnv, EnvBasicTestWithParam,
+                        ::testing::Values(encrypt_env.get()));
+INSTANTIATE_TEST_CASE_P(EncryptedEnv, EnvMoreTestWithParam,
+                        ::testing::Values(encrypt_env.get()));
+
 #ifndef ROCKSDB_LITE
 static std::unique_ptr<Env> mem_env(NewMemEnv(Env::Default()));
 INSTANTIATE_TEST_CASE_P(MemEnv, EnvBasicTestWithParam,
@@ -111,6 +124,7 @@ std::vector<Env*> GetCustomEnvs() {
     const char* uri = getenv("TEST_ENV_URI");
     if (uri != nullptr) {
       custom_env = NewCustomObject<Env>(uri, &custom_env_guard);
+//      Env::LoadEnv(uri, &custom_env);
     }
   }