Implement -Xembed-bitcode and -Xembed-bitcode-marker

FAQ.md

@@ -50,6 +50,25 @@ framework("MyCustomFramework") {
 
 </div>
 
+### Q: How do I enable bitcode for my Kotlin framework?
+
+A: Use either `-Xembed-bitcode` or `-Xembed-bitcode-marker` compiler option
+or matching Gradle DSL statement, i.e.
+
+<div class="sample" markdown="1" theme="idea" mode="groovy">
+
+```groovy
+framework("MyCustomFramework") {
+    extraOpts '-Xembed-bitcode' // for release binaries
+    // or '-Xembed-bitcode-marker' for debug binaries
+}
+```
+
+These options have nearly the same effect as clang's `-fembed-bitcode`/`-fembed-bitcode-marker`
+and swiftc's `-embed-bitcode`/`-embed-bitcode-marker`.
+
+</div>
+
 ### Q: Why do I see `InvalidMutabilityException`?
 
 A: It likely happens, because you are trying to mutate a frozen object. An object can transfer to the

backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2Native.kt

@@ -14,17 +14,14 @@ import org.jetbrains.kotlin.cli.common.CLITool
 import org.jetbrains.kotlin.cli.common.CommonCompilerPerformanceManager
 import org.jetbrains.kotlin.cli.common.ExitCode
 import org.jetbrains.kotlin.cli.common.config.addKotlinSourceRoot
-import org.jetbrains.kotlin.cli.common.config.addKotlinSourceRoots
 import org.jetbrains.kotlin.cli.common.config.kotlinSourceRoots
-import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.ERROR
-import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.WARNING
+import org.jetbrains.kotlin.cli.common.messages.CompilerMessageSeverity.*
 import org.jetbrains.kotlin.cli.jvm.compiler.EnvironmentConfigFiles
 import org.jetbrains.kotlin.cli.jvm.compiler.KotlinCoreEnvironment
 import org.jetbrains.kotlin.cli.jvm.plugins.PluginCliParser
 import org.jetbrains.kotlin.config.CommonConfigurationKeys
 import org.jetbrains.kotlin.config.CompilerConfiguration
 import org.jetbrains.kotlin.config.Services
-import org.jetbrains.kotlin.konan.KonanAbiVersion
 import org.jetbrains.kotlin.konan.KonanVersion
 import org.jetbrains.kotlin.konan.file.File
 import org.jetbrains.kotlin.konan.target.CompilerOutputKind
@@ -190,6 +187,8 @@ class K2Native : CLICompiler<K2NativeCompilerArguments>() {
                     })
                 if (arguments.friendModules != null)
                     put(FRIEND_MODULES, arguments.friendModules!!.split(File.pathSeparator).filterNot(String::isEmpty))
+
+                put(BITCODE_EMBEDDING_MODE, selectBitcodeEmbeddingMode(this, arguments, outputKind))
             }
         }
     }
@@ -214,5 +213,45 @@ class K2Native : CLICompiler<K2NativeCompilerArguments>() {
         }
     }
 }
+
+private fun selectBitcodeEmbeddingMode(
+        configuration: CompilerConfiguration,
+        arguments: K2NativeCompilerArguments,
+        outputKind: CompilerOutputKind
+): BitcodeEmbedding.Mode {
+
+    if (outputKind != CompilerOutputKind.FRAMEWORK) {
+        return BitcodeEmbedding.Mode.NONE.also {
+            val flag = when {
+                arguments.embedBitcodeMarker -> EMBED_BITCODE_MARKER_FLAG
+                arguments.embedBitcode -> EMBED_BITCODE_FLAG
+                else -> return@also
+            }
+
+            configuration.report(
+                    STRONG_WARNING,
+                    "'$flag' is only supported when producing frameworks, " +
+                            "but the compiler is producing ${outputKind.name.toLowerCase()}"
+            )
+        }
+    }
+
+    return when {
+        arguments.embedBitcodeMarker -> {
+            if (arguments.embedBitcode) {
+                configuration.report(
+                        STRONG_WARNING,
+                        "'$EMBED_BITCODE_FLAG' is ignored because '$EMBED_BITCODE_MARKER_FLAG' is specified"
+                )
+            }
+            BitcodeEmbedding.Mode.MARKER
+        }
+        arguments.embedBitcode -> {
+            BitcodeEmbedding.Mode.FULL
+        }
+        else -> BitcodeEmbedding.Mode.NONE
+    }
+}
+
 fun main(args: Array<String>) = K2Native.main(args)
 

backend.native/cli.bc/src/org/jetbrains/kotlin/cli/bc/K2NativeCompilerArguments.kt

@@ -94,6 +94,12 @@ class K2NativeCompilerArguments : CommonCompilerArguments() {
     @Argument(value = "-Xdisable", deprecatedName = "--disable", valueDescription = "<Phase>", description = "Disable backend phase")
     var disablePhases: Array<String>? = null
 
+    @Argument(value = EMBED_BITCODE_FLAG, description = "Embed LLVM IR bitcode as data")
+    var embedBitcode: Boolean = false
+
+    @Argument(value = EMBED_BITCODE_MARKER_FLAG, description = "Embed placeholder LLVM IR data as a marker")
+    var embedBitcodeMarker: Boolean = false
+
     @Argument(value = "-Xenable", deprecatedName = "--enable", valueDescription = "<Phase>", description = "Enable backend phase")
     var enablePhases: Array<String>? = null
 
@@ -153,3 +159,5 @@ class K2NativeCompilerArguments : CommonCompilerArguments() {
             }
 }
 
+const val EMBED_BITCODE_FLAG = "-Xembed-bitcode"
+const val EMBED_BITCODE_MARKER_FLAG = "-Xembed-bitcode-marker"

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/BitcodeEmbedding.kt

@@ -0,0 +1,41 @@
+package org.jetbrains.kotlin.backend.konan
+
+import org.jetbrains.kotlin.backend.konan.llvm.Int8
+import org.jetbrains.kotlin.backend.konan.llvm.Llvm
+import org.jetbrains.kotlin.konan.target.CompilerOutputKind
+
+object BitcodeEmbedding {
+
+    enum class Mode {
+        NONE, MARKER, FULL
+    }
+
+    internal fun getLinkerOptions(config: KonanConfig): List<String> = when (config.bitcodeEmbeddingMode) {
+        Mode.NONE -> emptyList()
+        Mode.MARKER -> listOf("-bitcode_bundle", "-bitcode_process_mode", "marker")
+        Mode.FULL -> listOf("-bitcode_bundle")
+    }
+
+    private val KonanConfig.bitcodeEmbeddingMode get() = configuration.get(KonanConfigKeys.BITCODE_EMBEDDING_MODE)!!.also {
+        require(it == Mode.NONE || this.produce == CompilerOutputKind.FRAMEWORK) {
+            "${it.name.toLowerCase()} bitcode embedding mode is not supported when producing ${this.produce.name.toLowerCase()}"
+        }
+    }
+
+    internal fun processModule(llvm: Llvm) = when (llvm.context.config.bitcodeEmbeddingMode) {
+        Mode.NONE -> {}
+        Mode.MARKER -> {
+            addEmptyMarker(llvm, "konan_llvm_bitcode", "__LLVM,__bitcode")
+            addEmptyMarker(llvm, "konan_llvm_cmdline", "__LLVM,__cmdline")
+        }
+        Mode.FULL -> {
+            addEmptyMarker(llvm, "konan_llvm_asm", "__LLVM,__asm")
+        }
+    }
+
+    private fun addEmptyMarker(llvm: Llvm, name: String, section: String) {
+        val global = llvm.staticData.placeGlobal(name, Int8(0), isExported = false)
+        global.setSection(section)
+        llvm.usedGlobals += global.llvmGlobal
+    }
+}

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/KonanConfigurationKeys.kt

@@ -20,6 +20,8 @@ class KonanConfigKeys {
                 = CompilerConfigurationKey.create("add debug information")
         val DISABLED_PHASES: CompilerConfigurationKey<List<String>> 
                 = CompilerConfigurationKey.create("disable backend phases")
+        val BITCODE_EMBEDDING_MODE: CompilerConfigurationKey<BitcodeEmbedding.Mode>
+                = CompilerConfigurationKey.create("bitcode embedding mode")
         val ENABLE_ASSERTIONS: CompilerConfigurationKey<Boolean>
                 = CompilerConfigurationKey.create("enable runtime assertions in generated code")
         val ENABLED_PHASES: CompilerConfigurationKey<List<String>> 

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/LinkStage.kt

@@ -183,6 +183,7 @@ internal class LinkStage(val context: Context, val phaser: PhaseManager) {
                     libraries = linker.targetLibffi + linker.linkStaticLibraries(includedBinaries),
                     linkerArgs = entryPointSelector +
                             asLinkerArgs(config.getNotNull(KonanConfigKeys.LINKER_ARGS)) +
+                            BitcodeEmbedding.getLinkerOptions(context.config) +
                             libraryProvidedLinkerFlags + frameworkLinkerArgs,
                     optimize = optimize, debug = debug, kind = linkerOutput,
                     outputDsymBundle = context.config.outputFile + ".dSYM").forEach {

backend.native/compiler/ir/backend.native/src/org/jetbrains/kotlin/backend/konan/llvm/IrToBitcode.kt

@@ -383,6 +383,8 @@ internal class CodeGeneratorVisitor(val context: Context, val lifetimes: Map<IrE
 
         codegen.objCDataGenerator?.finishModule()
 
+        BitcodeEmbedding.processModule(context.llvm)
+
         appendLlvmUsed("llvm.used", context.llvm.usedFunctions + context.llvm.usedGlobals)
         appendLlvmUsed("llvm.compiler.used", context.llvm.compilerUsedGlobals)
         appendStaticInitializers()

backend.native/tests/build.gradle

@@ -2993,13 +2993,16 @@ if (isMac() && project.testTarget != 'wasm32') {
                 if (!useCustomDist) {
                     dependsOn ":${target}CrossDistRuntime", ':commonDistRuntime', ':distCompiler'
                 }
+
+                extraOpts "-Xembed-bitcode-marker"
             }
         }
         swiftSources = ['framework/values/values.swift']
     }
 
     task testStdlibFramework(type: FrameworkTest) {
         frameworkName = 'Stdlib'
+        fullBitcode = true
         konanArtifacts {
             framework(frameworkName, targets: [ target ]) {
                 srcDir 'framework/stdlib'
@@ -3008,6 +3011,8 @@ if (isMac() && project.testTarget != 'wasm32') {
                 if (!useCustomDist) {
                     dependsOn ":${target}CrossDistRuntime", ':commonDistRuntime', ':distCompiler'
                 }
+
+                extraOpts "-Xembed-bitcode"
             }
         }
         swiftSources = ['framework/stdlib/stdlib.swift']

buildSrc/plugins/src/main/kotlin/org/jetbrains/kotlin/FrameworkTest.kt

@@ -31,6 +31,9 @@ open class FrameworkTest : DefaultTask() {
     @Input
     lateinit var frameworkName: String
 
+    @Input
+    var fullBitcode: Boolean = false
+
     private val testOutput: String by lazy {
         project.file(project.property("testOutputFramework")!!).absolutePath
     }
@@ -109,7 +112,8 @@ open class FrameworkTest : DefaultTask() {
         }
 
         val args = listOf("-sdk", configs.absoluteTargetSysRoot, "-target", swiftTarget) +
-                options + "-o" + output.toString() + sources
+                options + "-o" + output.toString() + sources +
+                if (fullBitcode) listOf("-embed-bitcode", "-Xlinker", "-bitcode_verify") else listOf("-embed-bitcode-marker")
 
         val (stdOut, stdErr, exitCode) = runProcess(executor = localExecutor(project), executable = compiler, args = args)
 

konan/konan.properties

@@ -39,7 +39,7 @@ targetToolchain.macos_x64 = target-toolchain-7-macos_x64
 
 arch.macos_x64 = x86_64
 targetSysRoot.macos_x64 = target-sysroot-7-macos_x64
-libffiDir.macos_x64 = libffi-3.2.1-2-darwin-macos
+libffiDir.macos_x64 = libffi-3.2.1-3-darwin-macos
 llvmLtoFlags.macos_x64 =
 llvmLtoOptFlags.macos_x64 = -O3 -function-sections
 llvmLtoNooptFlags.macos_x64 = -O1
@@ -53,7 +53,7 @@ osVersionMinFlagClang.macos_x64 = -mmacosx-version-min
 osVersionMin.macos_x64 = 10.11
 entrySelector.macos_x64 = -alias _Konan_main _main
 dependencies.macos_x64 = \
-    libffi-3.2.1-2-darwin-macos \
+    libffi-3.2.1-3-darwin-macos \
     clang-llvm-6.0.1-darwin-macos
 
 target-sysroot-7-macos_x64.default = \
@@ -65,7 +65,7 @@ target-toolchain-7-macos_x64.default = \
 # Apple's 32-bit iOS.
 targetToolchain.macos_x64-ios_arm32 = target-toolchain-7-macos_x64
 dependencies.macos_x64-ios_arm32 = \
-  libffi-3.2.1-2-darwin-ios \
+  libffi-3.2.1-3-darwin-ios \
   clang-llvm-6.0.1-darwin-macos
 
 target-sysroot-7-ios_arm32.default = \
@@ -75,7 +75,7 @@ arch.ios_arm32 = armv7
 entrySelector.ios_arm32 = -alias _Konan_main _main
 # Shared with 64-bit version.
 targetSysRoot.ios_arm32 = target-sysroot-7-ios_arm64
-libffiDir.ios_arm32 = libffi-3.2.1-2-darwin-ios
+libffiDir.ios_arm32 = libffi-3.2.1-3-darwin-ios
 llvmLtoFlags.ios_arm32 =
 llvmLtoOptFlags.ios_arm32 = -O3 -function-sections
 linkerNoDebugFlags.ios_arm32 = -S
@@ -91,7 +91,7 @@ osVersionMin.ios_arm32 = 9.0
 # Apple's 64-bit iOS.
 targetToolchain.macos_x64-ios_arm64 = target-toolchain-7-macos_x64
 dependencies.macos_x64-ios_arm64 = \
-  libffi-3.2.1-2-darwin-ios \
+  libffi-3.2.1-3-darwin-ios \
   clang-llvm-6.0.1-darwin-macos
 
 target-sysroot-7-ios_arm64.default = \
@@ -100,7 +100,7 @@ target-sysroot-7-ios_arm64.default = \
 arch.ios_arm64 = arm64
 entrySelector.ios_arm64 = -alias _Konan_main _main
 targetSysRoot.ios_arm64 = target-sysroot-7-ios_arm64
-libffiDir.ios_arm64 = libffi-3.2.1-2-darwin-ios
+libffiDir.ios_arm64 = libffi-3.2.1-3-darwin-ios
 llvmLtoFlags.ios_arm64 =
 llvmLtoOptFlags.ios_arm64 = -O3 -function-sections
 linkerNoDebugFlags.ios_arm64 = -S
@@ -116,7 +116,7 @@ osVersionMin.ios_arm64 = 9.0
 #  Apple's iOS simulator.
 targetToolchain.macos_x64-ios_x64 = target-toolchain-7-macos_x64
 dependencies.macos_x64-ios_x64 = \
-  libffi-3.2.1-1-darwin-ios_sim \
+  libffi-3.2.1-2-darwin-ios_sim \
   clang-llvm-6.0.1-darwin-macos
 
 target-sysroot-7-ios_x64.default = \
@@ -125,7 +125,7 @@ target-sysroot-7-ios_x64.default = \
 arch.ios_x64 = x86_64
 entrySelector.ios_x64 = -alias _Konan_main _main
 targetSysRoot.ios_x64 = target-sysroot-7-ios_x64
-libffiDir.ios_x64 = libffi-3.2.1-1-darwin-ios_sim
+libffiDir.ios_x64 = libffi-3.2.1-2-darwin-ios_sim
 llvmLtoFlags.ios_x64 =
 llvmLtoOptFlags.ios_x64 = -O3 -function-sections
 llvmLtoNooptFlags.ios_x64 = -O1