Merge branch 'master' into formatHeader

Author: AndreiKingsley

CONTRIBUTING.md

@@ -112,6 +112,9 @@ This library is built with Gradle.
   making local publishing faster: `./gradlew publishToMavenLocal -PskipKodex`.
   This, however, publishes the library with "broken" KDocs, 
   so it's only meant for faster iterations during development.
+* The parameter `-PincludeCoreLibrariesJson` makes the build include the `libraries.json` file in the `:core` module.
+  This file allows loading dataframe-jupyter when dataframe-core is present on its own in a Kotlin Notebook.
+  Usually only done when publishing.
 
 You can import this project into IDEA, but you have to delegate the build actions
 to Gradle (in Preferences -> Build, Execution, Deployment -> Build Tools -> Gradle -> Runner)

build.gradle.kts

@@ -223,6 +223,23 @@ allprojects {
         } catch (_: UnknownDomainObjectException) {
             logger.warn("Could not set buildConfig on :${this.name}")
         }
+
+        // Adds the instrumentedJars configuration/artifact to all sub-projects with a `jar` task
+        // This allows other modules to depend on the output of this task, aka the compiled jar of that module
+        // Used in :plugins:dataframe-gradle-plugin integration tests and in :samples for compiler plugin support
+        try {
+            val instrumentedJars: Configuration by configurations.creating {
+                isCanBeConsumed = true
+                isCanBeResolved = false
+            }
+            artifacts {
+                add("instrumentedJars", tasks.jar.get().archiveFile) {
+                    builtBy(tasks.jar)
+                }
+            }
+        } catch (_: Exception) {
+            logger.warn("Could not set instrumentedJars on :${this.name}")
+        }
     }
 }
 

core/build.gradle.kts

@@ -3,7 +3,7 @@ import com.google.devtools.ksp.gradle.KspTaskJvm
 import io.github.devcrocod.korro.KorroTask
 import nl.jolanrensen.kodex.gradle.creatingRunKodexTask
 import org.gradle.jvm.tasks.Jar
-import org.gradle.kotlin.dsl.withType
+import org.intellij.lang.annotations.Language
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
 
 plugins {
@@ -251,6 +251,53 @@ val changeJarTask by tasks.registering {
     }
 }
 
+// generateLibrariesJson makes sure a META-INF/kotlin-jupyter-libraries/libraries.json file is generated
+// This file allows loading dataframe-jupyter when dataframe-core is present on its own in a Kotlin Notebook.
+val generatedJupyterResourcesDir = layout.buildDirectory.dir("generated/jupyter")
+val generateLibrariesJson by tasks.registering {
+    val outDir = generatedJupyterResourcesDir.get().asFile.resolve("META-INF/kotlin-jupyter-libraries")
+    val outFile = outDir.resolve("libraries.json")
+    outputs.file(outFile)
+    inputs.property("version", project.version)
+
+    doLast {
+        outDir.mkdirs()
+        @Language("json")
+        val content =
+            """
+            {
+              "descriptors": [
+                {
+                  "init": [
+                    "USE { dependencies(\"org.jetbrains.kotlinx:dataframe-jupyter:${project.version}\") }"
+                  ]
+                }
+              ]
+            }
+            """.trimIndent()
+
+        outFile.delete()
+        outFile.writeText(content)
+        logger.lifecycle("generated META-INF/kotlin-jupyter-libraries/libraries.json for :core")
+    }
+}
+
+// If `includeCoreLibrariesJson` is set, modify the processResources task such that it includes
+// a META-INF libraries.json file.
+// This file allows loading dataframe-jupyter when dataframe-core is present on its own in a Kotlin Notebook.
+// This is usually only done when publishing.
+tasks.processResources {
+    if (project.hasProperty("includeCoreLibrariesJson")) {
+        dependsOn(generateLibrariesJson)
+        from(generatedJupyterResourcesDir) {
+            into("") // keep META-INF/... structure as generated
+        }
+        doLast {
+            logger.lifecycle("$this includes generated META-INF/kotlin-jupyter-libraries/libraries.json")
+        }
+    }
+}
+
 // if `processKDocsMain` runs, the Jar tasks must run after it so the generated-sources are there
 tasks.withType<Jar> {
     mustRunAfter(changeJarTask, tasks.generateKeywordsSrc, processKDocsMain)
@@ -386,17 +433,6 @@ kotlinPublications {
     }
 }
 
-val instrumentedJars: Configuration by configurations.creating {
-    isCanBeConsumed = true
-    isCanBeResolved = false
-}
-
-artifacts {
-    add("instrumentedJars", tasks.jar.get().archiveFile) {
-        builtBy(tasks.jar)
-    }
-}
-
 // Disable and enable if updating plugin breaks the build
 dataframes {
     schema {

dataframe-csv/build.gradle.kts

@@ -160,14 +160,3 @@ kotlinPublications {
 kotlin {
     explicitApi()
 }
-
-val instrumentedJars: Configuration by configurations.creating {
-    isCanBeConsumed = true
-    isCanBeResolved = false
-}
-
-artifacts {
-    add("instrumentedJars", tasks.jar.get().archiveFile) {
-        builtBy(tasks.jar)
-    }
-}

dataframe-json/build.gradle.kts

@@ -59,17 +59,6 @@ sourceSets {
     }
 }
 
-val instrumentedJars: Configuration by configurations.creating {
-    isCanBeConsumed = true
-    isCanBeResolved = false
-}
-
-artifacts {
-    add("instrumentedJars", tasks.jar.get().archiveFile) {
-        builtBy(tasks.jar)
-    }
-}
-
 kotlinPublications {
     publication {
         publicationName = "dataframeJson"

dataframe-jupyter/src/main/kotlin/org/jetbrains/kotlinx/dataframe/jupyter/Integration.kt

@@ -56,7 +56,7 @@ import kotlin.reflect.KType
 import kotlin.reflect.full.isSubtypeOf
 
 /** Users will get an error if their Kotlin Jupyter kernel is older than this version. */
-private const val MIN_KERNEL_VERSION = "0.11.0.198"
+private const val MIN_KERNEL_VERSION = "0.15.0.606"
 
 internal val newDataSchemas = mutableListOf<KClass<*>>()
 

docs/StardustDocs/resources/api/tail/notebook_test_tail_1.html

@@ -459,6 +459,7 @@
 /*<!--*/
 call_DataFrame(function() { DataFrame.addTable({ cols: [{ name: "<span title=\"firstName: String\">firstName</span>", children: [], rightAlign: false, values: ["Alice","Bob","Charlie","Charlie","Bob","Alice","Charlie"] }, 
 { name: "<span title=\"lastName: String\">lastName</span>", children: [], rightAlign: false, values: ["Cooper","Dylan","Daniels","Chaplin","Marley","Wolf","Byrd"] }, 
+{ name: "<span title=\"name: DataRow<*>\">name</span>", children: [0, 1], rightAlign: false, values: ["<span class=\"formatted\" title=\"firstName: Alice\nlastName: Cooper\"><span class=\"structural\">{ </span><span class=\"structural\">firstName: </span>Alice<span class=\"structural\">, </span><span class=\"structural\">lastName: </span>Cooper<span class=\"structural\"> }</span></span>","<span class=\"formatted\" title=\"firstName: Bob\nlastName: Dylan\"><span class=\"structural\">{ </span><span class=\"structural\">firstName: </span>Bob<span class=\"structural\">, </span><span class=\"structural\">lastName: </span>Dylan<span class=\"structural\"> }</span></span>","<span class=\"formatted\" title=\"firstName: Charlie\nlastName: Daniels\"><span class=\"structural\">{ </span><span class=\"structural\">firstName: </span>Charlie<span class=\"structural\">, </span><span class=\"structural\">lastName: </span>Dan<span class=\"structural\">...</span><span class=\"structural\"> }</span></span>","<span class=\"formatted\" title=\"firstName: Charlie\nlastName: Chaplin\"><span class=\"structural\">{ </span><span class=\"structural\">firstName: </span>Charlie<span class=\"structural\">, </span><span class=\"structural\">lastName: </span>Cha<span class=\"structural\">...</span><span class=\"structural\"> }</span></span>","<span class=\"formatted\" title=\"firstName: Bob\nlastName: Marley\"><span class=\"structural\">{ </span><span class=\"structural\">firstName: </span>Bob<span class=\"structural\">, </span><span class=\"structural\">lastName: </span>Marley<span class=\"structural\"> }</span></span>","<span class=\"formatted\" title=\"firstName: Alice\nlastName: Wolf\"><span class=\"structural\">{ </span><span class=\"structural\">firstName: </span>Alice<span class=\"structural\">, </span><span class=\"structural\">lastName: </span>Wolf<span class=\"structural\"> }</span></span>","<span class=\"formatted\" title=\"firstName: Charlie\nlastName: Byrd\"><span class=\"structural\">{ </span><span class=\"structural\">firstName: </span>Charlie<span class=\"structural\">, </span><span class=\"structural\">lastName: </span>Byrd<span class=\"structural\"> }</span></span>"] }, 
 { name: "<span title=\"age: Int\">age</span>", children: [], rightAlign: true, values: ["<span class=\"formatted\" title=\"\"><span class=\"numbers\">15</span></span>","<span class=\"formatted\" title=\"\"><span class=\"numbers\">45</span></span>","<span class=\"formatted\" title=\"\"><span class=\"numbers\">20</span></span>","<span class=\"formatted\" title=\"\"><span class=\"numbers\">40</span></span>","<span class=\"formatted\" title=\"\"><span class=\"numbers\">30</span></span>","<span class=\"formatted\" title=\"\"><span class=\"numbers\">20</span></span>","<span class=\"formatted\" title=\"\"><span class=\"numbers\">30</span></span>"] }, 
 { name: "<span title=\"city: String?\">city</span>", children: [], rightAlign: false, values: ["London","Dubai","Moscow","Milan","Tokyo","<span class=\"formatted\" title=\"\"><span class=\"null\">null</span></span>","Moscow"] }, 
 { name: "<span title=\"weight: Int?\">weight</span>", children: [], rightAlign: true, values: ["<span class=\"formatted\" title=\"\"><span class=\"numbers\">54</span></span>","<span class=\"formatted\" title=\"\"><span class=\"numbers\">87</span></span>","<span class=\"formatted\" title=\"\"><span class=\"null\">null</span></span>","<span class=\"formatted\" title=\"\"><span class=\"null\">null</span></span>","<span class=\"formatted\" title=\"\"><span class=\"numbers\">68</span></span>","<span class=\"formatted\" title=\"\"><span class=\"numbers\">55</span></span>","<span class=\"formatted\" title=\"\"><span class=\"numbers\">90</span></span>"] }, 

docs/StardustDocs/resources/api/tail/notebook_test_tail_2.html

@@ -459,6 +459,7 @@
 /*<!--*/
 call_DataFrame(function() { DataFrame.addTable({ cols: [{ name: "<span title=\"firstName: String\">firstName</span>", children: [], rightAlign: false, values: ["Charlie","Charlie","Bob","Alice","Charlie"] }, 
 { name: "<span title=\"lastName: String\">lastName</span>", children: [], rightAlign: false, values: ["Daniels","Chaplin","Marley","Wolf","Byrd"] }, 
+{ name: "<span title=\"name: DataRow<*>\">name</span>", children: [0, 1], rightAlign: false, values: ["<span class=\"formatted\" title=\"firstName: Charlie\nlastName: Daniels\"><span class=\"structural\">{ </span><span class=\"structural\">firstName: </span>Charlie<span class=\"structural\">, </span><span class=\"structural\">lastName: </span>Dan<span class=\"structural\">...</span><span class=\"structural\"> }</span></span>","<span class=\"formatted\" title=\"firstName: Charlie\nlastName: Chaplin\"><span class=\"structural\">{ </span><span class=\"structural\">firstName: </span>Charlie<span class=\"structural\">, </span><span class=\"structural\">lastName: </span>Cha<span class=\"structural\">...</span><span class=\"structural\"> }</span></span>","<span class=\"formatted\" title=\"firstName: Bob\nlastName: Marley\"><span class=\"structural\">{ </span><span class=\"structural\">firstName: </span>Bob<span class=\"structural\">, </span><span class=\"structural\">lastName: </span>Marley<span class=\"structural\"> }</span></span>","<span class=\"formatted\" title=\"firstName: Alice\nlastName: Wolf\"><span class=\"structural\">{ </span><span class=\"structural\">firstName: </span>Alice<span class=\"structural\">, </span><span class=\"structural\">lastName: </span>Wolf<span class=\"structural\"> }</span></span>","<span class=\"formatted\" title=\"firstName: Charlie\nlastName: Byrd\"><span class=\"structural\">{ </span><span class=\"structural\">firstName: </span>Charlie<span class=\"structural\">, </span><span class=\"structural\">lastName: </span>Byrd<span class=\"structural\"> }</span></span>"] }, 
 { name: "<span title=\"age: Int\">age</span>", children: [], rightAlign: true, values: ["<span class=\"formatted\" title=\"\"><span class=\"numbers\">20</span></span>","<span class=\"formatted\" title=\"\"><span class=\"numbers\">40</span></span>","<span class=\"formatted\" title=\"\"><span class=\"numbers\">30</span></span>","<span class=\"formatted\" title=\"\"><span class=\"numbers\">20</span></span>","<span class=\"formatted\" title=\"\"><span class=\"numbers\">30</span></span>"] }, 
 { name: "<span title=\"city: String?\">city</span>", children: [], rightAlign: false, values: ["Moscow","Milan","Tokyo","<span class=\"formatted\" title=\"\"><span class=\"null\">null</span></span>","Moscow"] }, 
 { name: "<span title=\"weight: Int?\">weight</span>", children: [], rightAlign: true, values: ["<span class=\"formatted\" title=\"\"><span class=\"null\">null</span></span>","<span class=\"formatted\" title=\"\"><span class=\"null\">null</span></span>","<span class=\"formatted\" title=\"\"><span class=\"numbers\">68</span></span>","<span class=\"formatted\" title=\"\"><span class=\"numbers\">55</span></span>","<span class=\"formatted\" title=\"\"><span class=\"numbers\">90</span></span>"] }, 

docs/StardustDocs/resources/api/tail/notebook_test_tail_3.html

@@ -459,6 +459,7 @@
 /*<!--*/
 call_DataFrame(function() { DataFrame.addTable({ cols: [{ name: "<span title=\"firstName: String\">firstName</span>", children: [], rightAlign: false, values: ["Alice","Charlie"] }, 
 { name: "<span title=\"lastName: String\">lastName</span>", children: [], rightAlign: false, values: ["Wolf","Byrd"] }, 
+{ name: "<span title=\"name: DataRow<*>\">name</span>", children: [0, 1], rightAlign: false, values: ["<span class=\"formatted\" title=\"firstName: Alice\nlastName: Wolf\"><span class=\"structural\">{ </span><span class=\"structural\">firstName: </span>Alice<span class=\"structural\">, </span><span class=\"structural\">lastName: </span>Wolf<span class=\"structural\"> }</span></span>","<span class=\"formatted\" title=\"firstName: Charlie\nlastName: Byrd\"><span class=\"structural\">{ </span><span class=\"structural\">firstName: </span>Charlie<span class=\"structural\">, </span><span class=\"structural\">lastName: </span>Byrd<span class=\"structural\"> }</span></span>"] }, 
 { name: "<span title=\"age: Int\">age</span>", children: [], rightAlign: true, values: ["<span class=\"formatted\" title=\"\"><span class=\"numbers\">20</span></span>","<span class=\"formatted\" title=\"\"><span class=\"numbers\">30</span></span>"] }, 
 { name: "<span title=\"city: String?\">city</span>", children: [], rightAlign: false, values: ["<span class=\"formatted\" title=\"\"><span class=\"null\">null</span></span>","Moscow"] }, 
 { name: "<span title=\"weight: Int\">weight</span>", children: [], rightAlign: true, values: ["<span class=\"formatted\" title=\"\"><span class=\"numbers\">55</span></span>","<span class=\"formatted\" title=\"\"><span class=\"numbers\">90</span></span>"] }, 

docs/StardustDocs/topics/dataSources/ApacheArrow.md

@@ -25,7 +25,7 @@ and in [`%use dataframe`](SetupKotlinNotebook.md#integrate-kotlin-dataframe) for
 > {style="warning"}
 
 > Structured (nested) Arrow types such as Struct are not supported yet in Kotlin DataFrame.
-> See an issue: [Add inner / Struct type support in Arrow](https://github.com/Kotlin/dataframe/issues/536)
+> See the issue: [Add inner / Struct type support in Arrow](https://github.com/Kotlin/dataframe/issues/536)
 > {style="warning"}
 
 ## Read