Merge pull request #953 from swiftlang/t/actioncache

[llbuild3] Cleanup and Implement action cache support

Author: dmbryson

LICENSE.txt

@@ -222,6 +222,11 @@ licenses, and/or restrictions:
 
 Program             Directory
 -------             ---------
-Google Test         llbuild/utils/unittest/googletest
-LLVM                llbuild/lib/llvm, llbuild/include/llvm
+BLAKE3              thirdparty/BLAKE3
 CMake               cmake/modules/FindSQLite3.cmake
+LLVM                llbuild/lib/llvm, llbuild/include/llvm
+Google Protobuf     thirdparty/protobuf
+Google Test         llbuild/utils/unittest/googletest
+Swift Protobuf      thridparty/swift-protobuf
+result.hpp          products/llbuild3/Result.hpp
+

Makefile

@@ -1,4 +1,4 @@
-# Helper makefile for generating Tritium protobuf files
+# Helper makefile for generating llbuild3 protobuf files
 
 .PHONY: generate
 generate: generate-protos
@@ -15,7 +15,7 @@ generate-protos: proto-toolchain
 		--swift_out=src \
 		--swift_opt=Visibility=Public \
 		--swift_opt=ProtoPathModuleMappings=src/module_map.asciipb \
-		$$(find src -name \*.proto)
+		$$(find src -name \*.proto -not -name \*Internal\*)
 
 .PHONY: proto-toolchain
 proto-toolchain:

README.tritium.md renamed to README.llbuild3.md

@@ -1,4 +1,4 @@
-# Tritium — llbuild3
+# llbuild3
 
 ## Objectives
 
@@ -18,7 +18,7 @@
 
 ## API Overview — Ergonomics
 
-Tritium will incorporate concepts from llbuild1, llbuild2, and llbuild2fx.  The constructs are intended to enable a separation of the items that can be produced (Artifacts) from the computations that can produce them (Rules/Tasks), addressing an expression challenge that exists today in llbuild2fx rule definition.
+llbuild3 will incorporate concepts from llbuild1, llbuild2, and llbuild2fx.  The constructs are intended to enable a separation of the items that can be produced (Artifacts) from the computations that can produce them (Rules/Tasks), addressing an expression challenge that exists today in llbuild2fx rule definition.
 
 ### Dynamic Graph Definition
 
@@ -53,7 +53,7 @@ Tritium will incorporate concepts from llbuild1, llbuild2, and llbuild2fx.  The
 * Build
     * Outstanding, in-progress request to produce a specific artifact from the overall build namespace
 
-![Tritium Classes](./docs/tritium.jpg)
+![llbuild3 Classes](./docs/llbuild3.jpg)
 
 ### Engine Components
 
@@ -111,8 +111,8 @@ Next steps
 Code Structure
 
 * Framework Headers
-    * products/Tritium
+    * products/llbuild3
 * Framework Sources
-    * src/tritium
+    * src/llbuild3
 * Unit Tests
-    * unittests/TritiumCoreTests
+    * unittests/llbuild3CoreTests

Xcode/Configs/TritiumCommon.xcconfig renamed to Xcode/Configs/llbuild3Common.xcconfig

@@ -10,8 +10,8 @@
 //
 //===----------------------------------------------------------------------===//
 
-TRITIUM_INSTALL_BASE_PATH = $(TRITIUM_INSTALL_DIR:default=$(LOCAL_LIBRARY_DIR))
-TRITIUM_FRAMEWORKS_INSTALL_DIR = $(TRITIUM_INSTALL_BASE_PATH)/Frameworks
+LLBUILD3_INSTALL_BASE_PATH = $(LLBUILD3_INSTALL_DIR:default=$(LOCAL_LIBRARY_DIR))
+LLBUILD3_FRAMEWORKS_INSTALL_DIR = $(LLBUILD3_INSTALL_BASE_PATH)/Frameworks
 
 // Set the default product name.
 PRODUCT_NAME = $(TARGET_NAME)
@@ -69,4 +69,4 @@ CODE_SIGN_ENTITLEMENTS = $(LLBUILD_CODE_SIGN_ENTITLEMENTS__producttype_eq_$(PROD
 LLBUILD_CODE_SIGN_ENTITLEMENTS__producttype_eq_com_apple_product_type_tool = $(LLBUILD_TOOL_CODE_SIGN_ENTITLEMENTS)
 LLBUILD_TOOL_CODE_SIGN_ENTITLEMENTS =
 
-#include? "TritiumOverrides.xcconfig"
+#include? "llbuild3Overrides.xcconfig"

Xcode/Configs/TritiumDebug.xcconfig renamed to Xcode/Configs/llbuild3Debug.xcconfig

@@ -11,7 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 // Shared build settings.
-#include "TritiumCommon.xcconfig"
+#include "llbuild3Common.xcconfig"
 
 // Only build the active architecture for Debug.
 ONLY_ACTIVE_ARCH = YES

Xcode/Configs/TritiumRelease.xcconfig renamed to Xcode/Configs/llbuild3Release.xcconfig

@@ -11,7 +11,7 @@
 //===----------------------------------------------------------------------===//
 
 // Shared build settings.
-#include "TritiumCommon.xcconfig"
+#include "llbuild3Common.xcconfig"
 
 // Prefer binary Info.plists for Release.
 INFOPLIST_OUTPUT_FORMAT = binary

Xcode/Tritium.xctestplan renamed to Xcode/llbuild3.xctestplan

@@ -16,7 +16,7 @@
       "target" : {
         "containerPath" : "container:llbuild.xcodeproj",
         "identifier" : "4072B5DD2C51BE5100B68221",
-        "name" : "TritiumTests"
+        "name" : "llbuild3Tests"
       }
     }
   ],

docs/tritium.jpg renamed to docs/llbuild3.jpg

@@ -16,8 +16,8 @@
             <BuildableReference
                BuildableIdentifier = "primary"
                BlueprintIdentifier = "4072B5D62C51BE5100B68221"
-               BuildableName = "Tritium.framework"
-               BlueprintName = "Tritium"
+               BuildableName = "llbuild3.framework"
+               BlueprintName = "llbuild3"
                ReferencedContainer = "container:llbuild.xcodeproj">
             </BuildableReference>
          </BuildActionEntry>
@@ -30,8 +30,8 @@
             <BuildableReference
                BuildableIdentifier = "primary"
                BlueprintIdentifier = "4072B5DD2C51BE5100B68221"
-               BuildableName = "TritiumTests.xctest"
-               BlueprintName = "TritiumTests"
+               BuildableName = "llbuild3Tests.xctest"
+               BlueprintName = "llbuild3Tests"
                ReferencedContainer = "container:llbuild.xcodeproj">
             </BuildableReference>
          </BuildActionEntry>
@@ -44,7 +44,7 @@
       shouldUseLaunchSchemeArgsEnv = "YES">
       <TestPlans>
          <TestPlanReference
-            reference = "container:Xcode/Tritium.xctestplan"
+            reference = "container:Xcode/llbuild3.xctestplan"
             default = "YES">
          </TestPlanReference>
       </TestPlans>
@@ -71,8 +71,8 @@
          <BuildableReference
             BuildableIdentifier = "primary"
             BlueprintIdentifier = "4072B5D62C51BE5100B68221"
-            BuildableName = "Tritium.framework"
-            BlueprintName = "Tritium"
+            BuildableName = "llbuild3.framework"
+            BlueprintName = "llbuild3"
             ReferencedContainer = "container:llbuild.xcodeproj">
          </BuildableReference>
       </MacroExpansion>

llbuild.xcodeproj/project.pbxproj

@@ -20,8 +20,8 @@
             <BuildableReference
                BuildableIdentifier = "primary"
                BlueprintIdentifier = "4072B5DD2C51BE5100B68221"
-               BuildableName = "TritiumTests.xctest"
-               BlueprintName = "TritiumTests"
+               BuildableName = "llbuild3Tests.xctest"
+               BlueprintName = "llbuild3Tests"
                ReferencedContainer = "container:llbuild.xcodeproj">
             </BuildableReference>
          </TestableReference>

llbuild.xcodeproj/xcshareddata/xcschemes/Tritium.xcscheme renamed to llbuild.xcodeproj/xcshareddata/xcschemes/llbuild3.xcscheme

@@ -16,8 +16,8 @@
             <BuildableReference
                BuildableIdentifier = "primary"
                BlueprintIdentifier = "4072B4DF2C51B3A000B68221"
-               BuildableName = "libTritiumThirdParty.a"
-               BlueprintName = "TritiumThirdParty"
+               BuildableName = "libllbuild3ThirdParty.a"
+               BlueprintName = "llbuild3ThirdParty"
                ReferencedContainer = "container:llbuild.xcodeproj">
             </BuildableReference>
          </BuildActionEntry>
@@ -52,8 +52,8 @@
          <BuildableReference
             BuildableIdentifier = "primary"
             BlueprintIdentifier = "4072B4DF2C51B3A000B68221"
-            BuildableName = "libTritiumThirdParty.a"
-            BlueprintName = "TritiumThirdParty"
+            BuildableName = "libllbuild3ThirdParty.a"
+            BlueprintName = "llbuild3ThirdParty"
             ReferencedContainer = "container:llbuild.xcodeproj">
          </BuildableReference>
       </MacroExpansion>

llbuild.xcodeproj/xcshareddata/xcschemes/TritiumTests.xcscheme renamed to llbuild.xcodeproj/xcshareddata/xcschemes/llbuild3Tests.xcscheme

@@ -10,12 +10,12 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef TRITIUM_CORE_ERRORS_H
-#define TRITIUM_CORE_ERRORS_H
+#ifndef LLBUILD3_CORE_ERRORS_H
+#define LLBUILD3_CORE_ERRORS_H
 
 #include <cstdint>
 
-namespace tritium {
+namespace llbuild3 {
 namespace core {
 
 enum EngineError: uint64_t {

llbuild.xcodeproj/xcshareddata/xcschemes/TritiumThirdParty.xcscheme renamed to llbuild.xcodeproj/xcshareddata/xcschemes/llbuild3ThirdParty.xcscheme

@@ -32,12 +32,12 @@
 #ifndef RESULT_RESULT_HPP
 #define RESULT_RESULT_HPP
 
-// BEGIN: Tritium additions
-#include <Tritium/Visibility.hpp>
-#define RESULT_NAMESPACE tritium
+// BEGIN: llbuild3 additions
+#include <llbuild3/Visibility.hpp>
+#define RESULT_NAMESPACE llbuild3
 #define RESULT_EXCEPTIONS_DISABLED
 #define RESULT_DISABLE_EXCEPTIONS
-// END: Tritium additions
+// END: llbuild3 additions
 
 #include <cstddef>      // std::size_t
 #include <type_traits>  // std::enable_if, std::is_constructible, etc
@@ -303,7 +303,7 @@ inline namespace bitwizeshift {
     explicit in_place_error_t() = default;
   };
 
-  TRITIUM_EXPORT constexpr auto in_place_error = in_place_error_t{};
+  LLBUILD3_EXPORT constexpr auto in_place_error = in_place_error_t{};
 
   //===========================================================================
   // forward-declarations
@@ -5998,8 +5998,8 @@ auto RESULT_NS_IMPL::swap(result<void,E>& lhs, result<void,E>& rhs)
 #undef RESULT_NODISCARD
 #undef RESULT_WARN_UNUSED
 
-// BEGIN: Tritium additions
+// BEGIN: llbuild3 additions
 #undef RESULT_NAMESPACE
-// END: Tritium additions
+// END: llbuild3 additions
 
 #endif /* RESULT_RESULT_HPP */

products/Tritium/Tritium.h

@@ -10,19 +10,19 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef TRITIUM_CORE_SWIFTADAPTORS_H
-#define TRITIUM_CORE_SWIFTADAPTORS_H
+#ifndef LLBUILD3_CORE_SWIFTADAPTORS_H
+#define LLBUILD3_CORE_SWIFTADAPTORS_H
 
 #include <memory>
 #include <optional>
 #include <string>
 #include <vector>
 
-#include <Tritium/Visibility.hpp>
-#include <Tritium/Result.hpp>
+#include <llbuild3/Visibility.hpp>
+#include <llbuild3/Result.hpp>
 
 
-namespace tritium {
+namespace llbuild3 {
 namespace core {
 
 class Build;
@@ -31,6 +31,8 @@ struct EngineConfig;
 
 // Serialized Protobuf Objects
 typedef std::string ArtifactPB;
+typedef std::string CacheKeyPB;
+typedef std::string CacheValuePB;
 typedef std::string ErrorPB;
 typedef std::string LabelPB;
 typedef std::string SignaturePB;
@@ -66,17 +68,18 @@ class ExtTaskInterface {
 public:
   ExtTaskInterface(void* impl, uint64_t ctx) : impl(impl), ctx(ctx) { }
 
-  TRITIUM_EXPORT ErrorPB registerRuleProvider(const ExtRuleProvider provider);
+  LLBUILD3_EXPORT ErrorPB registerRuleProvider(const ExtRuleProvider provider);
 
-  TRITIUM_EXPORT result<uint64_t, ErrorPB> requestArtifact(const LabelPB label);
-  TRITIUM_EXPORT result<uint64_t, ErrorPB> requestRule(const LabelPB label);
-  TRITIUM_EXPORT result<uint64_t, ErrorPB> requestAction();
+  LLBUILD3_EXPORT result<uint64_t, ErrorPB> requestArtifact(const LabelPB label);
+  LLBUILD3_EXPORT result<uint64_t, ErrorPB> requestRule(const LabelPB label);
+  LLBUILD3_EXPORT result<uint64_t, ErrorPB> requestAction();
 };
 
 struct ExtTask {
   void* ctx;
 
   LabelPB name;
+  SignaturePB signature;
 
   bool isInit = false;
 
@@ -106,25 +109,44 @@ class BuildRef {
 public:
   BuildRef(std::shared_ptr<Build> build) : build(build) { }
 
-  TRITIUM_EXPORT void cancel();
-  TRITIUM_EXPORT void addCompletionHandler(void* ctx, void (*handler)(void*, result<ArtifactPB, ErrorPB>*));
+  LLBUILD3_EXPORT void cancel();
+  LLBUILD3_EXPORT void addCompletionHandler(void* ctx, void (*handler)(void*, result<ArtifactPB, ErrorPB>*));
 };
 
+class CASDatabase;
+typedef std::shared_ptr<CASDatabase> CASDatabaseRef;
+LLBUILD3_EXPORT CASDatabaseRef makeInMemoryCASDatabase();
+
+struct ExtActionCache {
+  void* ctx;
+
+  // FIXME: cleanup context
+
+  void (*getFn)(void* ctx, CacheKeyPB key, std::function<void (CacheValuePB, ErrorPB)>);
+  void (*updateFn)(void*, CacheKeyPB key, CacheValuePB value);
+};
+
+class ActionCache;
+typedef std::shared_ptr<ActionCache> ActionCacheRef;
+LLBUILD3_EXPORT ActionCacheRef makeExtActionCache(ExtActionCache extCache);
+LLBUILD3_EXPORT ActionCacheRef makeInMemoryActionCache();
+
+
 struct ExtEngineConfig {
   std::optional<LabelPB> initRule;
 
-  TRITIUM_EXPORT inline void setInitRule(LabelPB ir) { initRule = ir; }
+  LLBUILD3_EXPORT inline void setInitRule(LabelPB ir) { initRule = ir; }
 };
 
 class EngineRef {
   std::shared_ptr<Engine> engine;
 public:
   EngineRef(std::shared_ptr<Engine> engine) : engine(engine) { }
 
-  TRITIUM_EXPORT BuildRef build(const LabelPB artifact);
+  LLBUILD3_EXPORT BuildRef build(const LabelPB artifact);
 };
 
-TRITIUM_EXPORT EngineRef makeEngine(ExtEngineConfig config, /* action cache, ... */ const ExtRuleProvider provider);
+LLBUILD3_EXPORT EngineRef makeEngine(ExtEngineConfig config, CASDatabaseRef casdb, ActionCacheRef cache, const ExtRuleProvider provider);
 
 }
 }

products/Tritium/Errors.hpp renamed to products/llbuild3/Errors.hpp

@@ -10,17 +10,17 @@
 //
 //===----------------------------------------------------------------------===//
 
-#ifndef TRITIUM_VISIBILITY_H
-#define TRITIUM_VISIBILITY_H
+#ifndef LLBUILD3_VISIBILITY_H
+#define LLBUILD3_VISIBILITY_H
 
 #if defined(__ELF__) || (defined(__APPLE__) && defined(__MACH__))
-#define TRITIUM_EXPORT __attribute__((__visibility__("default")))
+#define LLBUILD3_EXPORT __attribute__((__visibility__("default")))
 #else
 // asume PE/COFF
 #if defined(_WINDLL)
-#define TRITIUM_EXPORT __declspec(dllexport)
+#define LLBUILD3_EXPORT __declspec(dllexport)
 #else
-#define TRITIUM_EXPORT
+#define LLBUILD3_EXPORT
 #endif
 #endif