Enabled memory tier config API in cachebench

Author: vinser52

cachelib/cachebench/cache/Cache-inl.h

@@ -80,6 +80,18 @@ Cache<Allocator>::Cache(const CacheConfig& config,
 
   allocatorConfig_.setCacheSize(config_.cacheSizeMB * (MB));
 
+  if (!cacheDir.empty()) {
+    allocatorConfig_.cacheDir = cacheDir;
+  }
+
+  if (config_.usePosixShm) {
+    allocatorConfig_.usePosixForShm();
+  }
+
+  if (!config_.memoryTierConfigs.empty()) {
+    allocatorConfig_.configureMemoryTiers(config_.memoryTierConfigs);
+  }
+
   auto cleanupGuard = folly::makeGuard([&] {
     if (!nvmCacheFilePath_.empty()) {
       util::removePath(nvmCacheFilePath_);
@@ -222,8 +234,7 @@ Cache<Allocator>::Cache(const CacheConfig& config,
   allocatorConfig_.cacheName = "cachebench";
 
   bool isRecovered = false;
-  if (!cacheDir.empty()) {
-    allocatorConfig_.cacheDir = cacheDir;
+  if (!allocatorConfig_.cacheDir.empty()) {
     try {
       cache_ = std::make_unique<Allocator>(Allocator::SharedMemAttach,
                                            allocatorConfig_);

cachelib/cachebench/test_configs/simple_tiers_test.json

@@ -0,0 +1,36 @@
+// @nolint instantiates a small cache and runs a quick run of basic operations.
+{
+    "cache_config" : {
+      "cacheSizeMB" : 512,
+      "usePosixShm" : false,
+      "cacheDir" : "/tmp/mem-tiers",
+      "memoryTiers" : [
+        {
+          "ratio": 1,
+          "memBindNodes": 0
+        }
+      ],
+      "poolRebalanceIntervalSec" : 1,
+      "moveOnSlabRelease" : false,
+  
+      "numPools" : 2,
+      "poolSizes" : [0.3, 0.7]
+    },
+    "test_config" : {
+        "numOps" : 100000,
+        "numThreads" : 32,
+        "numKeys" : 1000000,
+  
+        "keySizeRange" : [1, 8, 64],
+        "keySizeRangeProbability" : [0.3, 0.7],
+  
+        "valSizeRange" : [1, 32, 10240, 409200],
+        "valSizeRangeProbability" : [0.1, 0.2, 0.7],
+  
+        "getRatio" : 0.15,
+        "setRatio" : 0.8,
+        "delRatio" : 0.05,
+        "keyPoolDistribution": [0.4, 0.6],
+        "opPoolDistribution" : [0.5, 0.5]
+    }
+  }

cachelib/cachebench/util/CacheConfig.cpp

@@ -83,6 +83,13 @@ CacheConfig::CacheConfig(const folly::dynamic& configJson) {
   JSONSetVal(configJson, deviceMaxWriteSize);
 
   JSONSetVal(configJson, memoryOnlyTTL);
+  
+  JSONSetVal(configJson, usePosixShm);
+  if (configJson.count("memoryTiers")) {
+    for (auto& it : configJson["memoryTiers"]) {
+      memoryTierConfigs.push_back(MemoryTierConfig(it).getMemoryTierCacheConfig());
+    }
+  }
 
   JSONSetVal(configJson, useTraceTimeStamp);
   JSONSetVal(configJson, printNvmCounters);
@@ -95,7 +102,7 @@ CacheConfig::CacheConfig(const folly::dynamic& configJson) {
   // if you added new fields to the configuration, update the JSONSetVal
   // to make them available for the json configs and increment the size
   // below
-  checkCorrectSize<CacheConfig, 664>();
+  checkCorrectSize<CacheConfig, 696>();
 
   if (numPools != poolSizes.size()) {
     throw std::invalid_argument(folly::sformat(
@@ -124,6 +131,55 @@ std::shared_ptr<RebalanceStrategy> CacheConfig::getRebalanceStrategy() const {
         RandomStrategy::Config{static_cast<unsigned int>(rebalanceMinSlabs)});
   }
 }
+
+MemoryTierConfig::MemoryTierConfig(const folly::dynamic& configJson) {
+  JSONSetVal(configJson, ratio);
+  JSONSetVal(configJson, memBindNodes);
+
+  checkCorrectSize<MemoryTierConfig, 40>();
+}
+
+std::vector<size_t> MemoryTierConfig::parseNumaNodes() {
+  std::vector<size_t> numaNodes;
+
+  std::vector<folly::StringPiece> tokens;
+  folly::split(",", memBindNodes, tokens, true /*ignore empty*/);
+  for (const auto &token : tokens) {
+    if (token.startsWith("!")) {
+      throw std::invalid_argument(folly::sformat(
+        "invalid NUMA nodes binding in memory tier config: {} "
+        "inverse !N or !N-N is not supported "
+        "nodes may be specified as N,N,N or N-N or N,N-N or N-N,N-N and so forth.",
+        token));
+    }
+    else if (token.startsWith("+")) {
+      throw std::invalid_argument(folly::sformat(
+        "invalid NUMA nodes binding in memory tier config: {} "
+        "relative nodes are not supported. "
+        "nodes may be specified as N,N,N or N-N or N,N-N or N-N,N-N and so forth.",
+        token));
+    }
+    else if (token.contains("-")) {
+      size_t begin, end;
+      if(folly::split("-", token, begin, end) && begin < end) {
+        while(begin <=end) {
+          numaNodes.push_back(begin++);
+        }
+      } else {
+        throw std::invalid_argument(folly::sformat(
+        "invalid NUMA nodes binding in memory tier config: {} "
+        "Invalid range format. "
+        "nodes may be specified as N,N,N or N-N or N,N-N or N-N,N-N and so forth.",
+        token));
+      }
+    }
+    else {
+      numaNodes.push_back(folly::to<size_t>(token));
+    }
+  }
+
+  return numaNodes;
+}
 } // namespace cachebench
 } // namespace cachelib
 } // namespace facebook

cachelib/cachebench/util/CacheConfig.h

@@ -41,6 +41,31 @@ class CacheMonitorFactory {
   virtual std::unique_ptr<CacheMonitor> create(Lru2QAllocator& cache) = 0;
 };
 
+// Parse memory tiers configuration from JSON config
+struct MemoryTierConfig : public JSONConfig {
+  MemoryTierConfig() {}
+
+  explicit MemoryTierConfig(const folly::dynamic& configJson);
+  MemoryTierCacheConfig getMemoryTierCacheConfig() {
+    MemoryTierCacheConfig config = memoryTierCacheConfigFromSource();
+    config.setRatio(ratio);
+    config.setMemBind(parseNumaNodes());
+    return config;
+  }
+
+  // Specifies ratio of this memory tier to other tiers
+  size_t ratio{0};
+  // Allocate memory only from specified NUMA nodes
+  std::string memBindNodes{""};
+
+private:
+  MemoryTierCacheConfig memoryTierCacheConfigFromSource() {
+    return MemoryTierCacheConfig::fromShm();
+  }
+
+  std::vector<size_t> parseNumaNodes();
+};
+
 struct CacheConfig : public JSONConfig {
   // by defaullt, lru allocator. can be set to LRU-2Q.
   std::string allocator{"LRU"};
@@ -194,6 +219,12 @@ struct CacheConfig : public JSONConfig {
   // Not used when its value is 0.  In seconds.
   uint32_t memoryOnlyTTL{0};
 
+  // Use Posix Shm instead of SysVShm
+  bool usePosixShm{false};
+
+  // Memory tiers configs
+  std::vector<MemoryTierCacheConfig> memoryTierConfigs{};
+
   // If enabled, we will use the timestamps from the trace file in the ticker
   // so that the cachebench will observe time based on timestamps from the trace
   // instead of the system time.