Examples separate projects

.github/workflows/archive-examples.yml

@@ -5,9 +5,8 @@
       branches:
         - master
       paths:
-        - 'examples/android-example/**'
-        - 'examples/kotlin-dataframe-plugin-gradle-example/**'
-        - 'examples/kotlin-dataframe-plugin-maven-example/**'
+        - 'examples/projects/**'
+        - '!examples/projects/dev/**'
     workflow_dispatch:
 
   jobs:
@@ -21,15 +20,16 @@
         - name: Create archives in temp directory
           run: |
             mkdir -p /tmp/archives
-
-            cd examples/android-example
-            zip -r /tmp/archives/android-example.zip . -x "*.git*" "build/*" ".gradle/*" "*/build/*"
-
-            cd ../kotlin-dataframe-plugin-gradle-example
-            zip -r /tmp/archives/kotlin-dataframe-plugin-gradle-example.zip . -x "*.git*" "build/*" ".gradle/*" "*/build/*" ".idea/*"
-
-            cd ../kotlin-dataframe-plugin-maven-example
-            zip -r /tmp/archives/kotlin-dataframe-plugin-maven-example.zip . -x "*.git*" "target/*" ".idea/*"
+            cd examples/projects
+            for dir in */ ; do
+              dirname="${dir%/}"
+              if [ "$dirname" != "dev" ]; then
+                echo "Archiving $dirname"
+                cd "$dirname"
+                zip -r "/tmp/archives/${dirname}.zip" . -x "*.git*" "build/*" ".gradle/*" "*/build/*" ".idea/*" "target/*"
+                cd ..
+              fi
+            done
 
         - name: Checkout example-projects-archives branch
           run: |
@@ -58,21 +58,22 @@
             Automatically generated archives of example projects.
 
             ## Available archives:
+            EOF
 
-            ### Android Example
-            Example of using Kotlin DataFrame in an Android application.
-
-            **Download:** [android-example.zip](https://github.com/Kotlin/dataframe/raw/example-projects-archives/android-example.zip)
-
-            ### Kotlin DataFrame Plugin Gradle Example
-            Example of using the compiler plugin for Kotlin DataFrame in Gradle project.
+            for zipfile in *.zip; do
+              if [ "$zipfile" = "*.zip" ]; then continue; fi
+              project_name="${zipfile%.zip}"
+              title=$(echo "$project_name" | sed -r 's/-/ /g' | awk '{for(i=1;i<=NF;i++)sub(/./,toupper(substr($i,1,1)),$i)}1')
+
+              cat >> README.md << EOF
 
-            **Download:** [kotlin-dataframe-plugin-gradle-example.zip](https://github.com/Kotlin/dataframe/raw/example-projects-archives/kotlin-dataframe-plugin-gradle-example.zip)
+            ### $title
 
-            ### Kotlin DataFrame Plugin Maven Example
-            Example of using the compiler plugin for Kotlin DataFrame in Gradle project.
+            **Download:** [$zipfile](https://github.com/Kotlin/dataframe/raw/example-projects-archives/$zipfile)
+            EOF
+            done
 
-            **Download:** [kotlin-dataframe-plugin-maven-example.zip](https://github.com/Kotlin/dataframe/raw/example-projects-archives/kotlin-dataframe-plugin-maven-example.zip)
+            cat >> README.md << EOF
 
             ---
 

.github/workflows/generated-sources-master.yml

@@ -21,13 +21,13 @@ jobs:
 
       - name: Run Gradle task
         timeout-minutes: 60
-        run: ./gradlew processKDocsMain korro
+        run: ./gradlew processKDocsMain korro syncExampleFolders
 
       - name: Commit changes
         run: |
           git config --global user.name 'github-actions[bot]'
           git config --global user.email 'github-actions[bot]@users.noreply.github.com'
-          git add './core/generated-sources' './dataframe-csv/generated-sources' './docs/StardustDocs/resources/snippets' './docs/StardustDocs/topics'
+          git add './core/generated-sources' './dataframe-csv/generated-sources' './docs/StardustDocs/resources/snippets' './docs/StardustDocs/topics' './examples/projects'
           git diff --staged --quiet || git commit -m "Automated commit of generated code"
           git push
         env:

.github/workflows/generated-sources.yml

@@ -39,18 +39,18 @@ jobs:
 
       - name: Run Gradle task
         timeout-minutes: 60
-        run: ./gradlew processKDocsMain korro
+        run: ./gradlew processKDocsMain korro syncExampleFolders
 
       - name: Check for changes in generated sources
         id: git-diff
-        run: echo "changed=$(if git diff --quiet './core/generated-sources' './dataframe-csv/generated-sources' './docs/StardustDocs/resources/snippets' './docs/StardustDocs/topics'; then echo 'false'; else echo 'true'; fi)" >> $GITHUB_OUTPUT
+        run: echo "changed=$(if git diff --quiet './core/generated-sources' './dataframe-csv/generated-sources' './docs/StardustDocs/resources/snippets' './docs/StardustDocs/topics' './examples/projects'; then echo 'false'; else echo 'true'; fi)" >> $GITHUB_OUTPUT
 
       - name: Commit and push if changes
         id: git-commit
         if: steps.git-diff.outputs.changed == 'true'
         run: |
           git checkout -b generated-sources/docs-update-${{ github.run_number }}
-          git add './core/generated-sources' './dataframe-csv/generated-sources' './docs/StardustDocs/resources/snippets' './docs/StardustDocs/topics'
+          git add './core/generated-sources' './dataframe-csv/generated-sources' './docs/StardustDocs/resources/snippets' './docs/StardustDocs/topics' './examples/projects'
           git commit -m "Update generated sources with recent changes"
           git push origin generated-sources/docs-update-${{ github.run_number }}
           echo "commit=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT

README.md

@@ -141,8 +141,8 @@ for detailed setup instructions (including Groovy DSL).
 
 * You can also check the [Custom Gradle Configuration](https://kotlin.github.io/dataframe/setupcustomgradle.html) if you don't need certain formats as dependencies.
 * For Android projects, see [Setup Kotlin DataFrame on Android](https://kotlin.github.io/dataframe/setupandroid.html).
-* See [IDEA Gradle example projects](examples/idea-examples) 
-and [the Gradle project with the Kotlin DataFrame Compiler plugin](examples/kotlin-dataframe-plugin-gradle-example).
+* See [IDEA Gradle example projects](examples) 
+and [the Gradle project with the Kotlin DataFrame Compiler plugin](examples/projects/kotlin-dataframe-plugin-gradle-example).
 
 Refer to the
 [Setup Kotlin DataFrame in Kotlin Notebook](https://kotlin.github.io/dataframe/setupkotlinnotebook.html)
@@ -174,14 +174,14 @@ Make sure that you have `mavenCentral` in the list of repositories:
 Refer to
 [Get started with Kotlin DataFrame on Maven](https://kotlin.github.io/dataframe/setupmaven.html).
 
-* See [the Maven project with the Kotlin DataFrame Compiler plugin](examples/kotlin-dataframe-plugin-gradle-example).
+* See [the Maven project with the Kotlin DataFrame Compiler plugin](examples/projects/kotlin-dataframe-plugin-gradle-example).
 
 
 ## Code example
 
 This example of Kotlin DataFrame code with
 the [Compiler Plugin](https://kotlin.github.io/dataframe/compiler-plugin.html) enabled.
-See [the full project](https://github.com/Kotlin/dataframe/tree/master/examples/kotlin-dataframe-plugin-gradle-example).
+See [the full project](https://github.com/Kotlin/dataframe/tree/master/examples/projects/kotlin-dataframe-plugin-gradle-example).
 See also
 [this example in Kotlin Notebook](https://github.com/Kotlin/dataframe/tree/master/examples/notebooks/readme_example.ipynb).
 

RELEASE_CHECK_LIST.md

@@ -5,38 +5,39 @@
 3. Update documentation on Kotlin site **RC**
    - The overview [page](https://kotlinlang.org/docs/data-analysis-overview.html)
    - The Data Analysis subchapter, for example, the [page](https://kotlinlang.org/docs/data-analysis-work-with-data-sources.html)
-3. Update tutorials according to the latest code changes.
-4. Update README.MD according last code changes:
+4. Update tutorials according to the latest code changes.
+5. Update README.MD according last code changes:
    - update an artifact version.
    - update a Kotlin version.
    - update the [section](README.md#kotlin-kotlin-jupyter-openapi-arrow-and-jdk-versions) about library versions.
-5. Check the project version in the file `gradle.properties` (i.e. it's 0.10.0 when trying to release version 0.10.0).
+6. Check the project version in the file `gradle.properties` (i.e. it's 0.10.0 when trying to release version 0.10.0).
    - For major releases: update a project version in the file [`v.list`](https://github.com/Kotlin/dataframe/blame/master/docs/StardustDocs/v.list)
    - For major releases: update a project version in the file [`main.yml`](https://github.com/Kotlin/dataframe/blob/master/.github/workflows/main.yml)
    - For major releases: update a project version in the file [`project.ihp`](https://github.com/Kotlin/dataframe/blob/master/docs/StardustDocs/project.ihp)
-6. Update `libs.versions.toml` file if required, run `./gradlew dependencyUpdates` to check for updates. **RC**
-7. Create and checkout the release branch. **RC**
-8. Make last commit with release tag (_v0.1.1_ for example) to the release branch. **RC**
-9. Run tests and build artifacts on TC for the commit with the release tag. **RC**
-10. Deploy artifacts on Maven Central via the `Publish` task (**directly and without pre-run dependencies**) on TC based on the commit with the release tag. **RC**
-11. Check artifacts' availability on [MavenCentral](https://mvnrepository.com/artifact/org.jetbrains.kotlinx/dataframe). **RC**
-12. Check [Gradle Plugin portal availability](https://plugins.gradle.org/plugin/org.jetbrains.kotlinx.dataframe/) (usually it takes 12 hours). **RC**
-13. Update a bootstrap dependency version in the `libs.versions.toml` file (only after the plugin's publication). **RC**
-14. Do final testing: **RC**
+7. Update `libs.versions.toml` file if required, run `./gradlew dependencyUpdates` to check for updates. **RC**
+8. Create and checkout the release branch. **RC**
+9. Make last commit with release tag (_v0.1.1_ for example) to the release branch. **RC**
+10. Run tests and build artifacts on TC for the commit with the release tag. **RC**
+11. Deploy artifacts on Maven Central via the `Publish` task (**directly and without pre-run dependencies**) on TC based on the commit with the release tag. **RC**
+12. Check artifacts' availability on [MavenCentral](https://mvnrepository.com/artifact/org.jetbrains.kotlinx/dataframe). **RC**
+13. Check [Gradle Plugin portal availability](https://plugins.gradle.org/plugin/org.jetbrains.kotlinx.dataframe/) (usually it takes 12 hours). **RC**
+14. Update a bootstrap dependency version in the `libs.versions.toml` file (only after the plugin's publication). **RC**
+15. Do final testing: **RC**
     - Check on Datalore with a test project (TODO: add link).
     - Check for Android with a test project (TODO: add link).
     - Check for ServerSide with a test project (TODO: add link).
-15. Publish Documentation from [GitHub Action](https://github.com/Kotlin/dataframe/actions/workflows/main.yml)
-16. Prepare and publish the Release Notes.
-17. Create a Release from the release tag on GitHub.
-18. Update a KDF version in the [Kotlin Jupyter Descriptor](https://github.com/Kotlin/kotlin-jupyter-libraries/blob/master/dataframe.json). Now the Renovate bot does this.
-19. Update the DataFrame version in the `gradle.properties` file for the next release cycle (i.e. 0.10.0 -> 0.11.0)
-20. Update deprecated functions in [deprecationMessages.kt](/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/util/deprecationMessages.kt)
+16. Publish Documentation from [GitHub Action](https://github.com/Kotlin/dataframe/actions/workflows/main.yml)
+17. Prepare and publish the Release Notes.
+18. Create a Release from the release tag on GitHub.
+19. Update a KDF version in the [Kotlin Jupyter Descriptor](https://github.com/Kotlin/kotlin-jupyter-libraries/blob/master/dataframe.json). Now the Renovate bot does this.
+20. Update the DataFrame version in the `gradle.properties` file for the next release cycle (i.e. 0.10.0 -> 0.11.0)
+21. Update deprecated functions in [deprecationMessages.kt](/core/src/main/kotlin/org/jetbrains/kotlinx/dataframe/util/deprecationMessages.kt)
     such that 
     - `Level.WARNING` messages are changed to `Level.ERROR`
     - `Level.ERROR` messages and their functions are removed.
     - Update regions in the file accordingly.
-21. Update Notebook examples, both in the project and on Datalore.
+22. Update Notebook examples, both in the project and on Datalore.
     - Dev notebooks may contain updated code for release Notebook examples.
+23. Promote dev project examples by running `gradlew promoteDevExamples`.
 
 (Activities that need to be done for **R**elease **C**andidate releases are marked as such)

build-logic/README.md

@@ -9,15 +9,50 @@ acting as [Convention Plugins](https://docs.gradle.org/current/userguide/impleme
 
 ### Plugins:
 - `dfbuild.base`: common build logic for all projects.
+
 - `dfbuild.kotlinJvmCommon`: common build logic for all Kotlin JVM projects.
     - Includes `dfbuild.base` and `dfbuild.ktlint`.
     - Sets `explicitApi()`, opt-ins, and the toolchain version.
     - Sets up the `instrumentedJars` configuration and task. 
     - This should not be used directly, as a JVM target version needs to be picked.
       Use `dfbuild.kotlinJvm8` (preferred) or `dfbuild.kotlinJvm11` instead.
+
 - `dfbuild.kotlinJvm8`: See `dfbuild.kotlinJvmCommon`.
+
 - `dfbuild.kotlinJvm11`: See `dfbuild.kotlinJvmCommon`.
+
 - `dfbuild.ktlint`: Sets up our linter plugin. Included by default in `dfbuild.kotlinJvmCommon`.
+
 - `dfbuild.buildConfig`: Generates build config compile-time constants,
    like `BuildConfig.VERSION` and `BuildConfig.DEBUG`.
    Is NOT included by default, but can be combined with `dfbuild.kotlinJvm<X>`.
+
+- `dfbuild.buildExampleProjects`: Generates tasks that sync versions and
+    build all example projects in the `/examples/projects` and `/examples/projects/dev` directories.
+    - Only apply this plugin to the root project!
+    - Projects inside `/examples/projects` are built using the latest release version of DataFrame.
+      They are meant to be downloadable as separate projects by users.
+    - Projects inside `/examples/projects/dev` are built using sources of DataFrame.
+      They should be kept up to date as sources change.
+    - Available tasks:
+        - `sync<NameOfFolder(Dev)>`: Syncs and overwrites build settings and versions.
+          Copies and overwrites versions from the main `libs.versions.toml`, `gradle.properties`, etc.
+        - `syncAllExampleFolders`: Syncs all example projects. Automatically called on `assemble`.
+        - `promoteDevExamples`: Promotes the `/examples/projects/dev` example projects to `/examples/project`.
+          To be called after a release. Automatically calls `syncAllExampleFolders` afterward.
+        - `generate<NameOfFolder(Dev)>Test`: Generates a Junit test class for the given folder in
+          `/build/generated/testBuildingExamples/src/test/kotlin`.
+        - `generateAllExampleFoldersTests`: Cleans and generates Junit test classes for all example folders.
+        - `runBuild<NameOfFolder(Dev)>`: Executes the generated Junit test class for the given folder and
+          runs the build. This automatically syncs and generates the test classes beforehand.
+            - `publishLocal` is called automatically when dealing with a dev Maven test.
+            - Android tests (folders with "android" in their name) require the `android.sdk.dir` gradle property to be set.
+            Else, the test will be skipped.
+        - `runBuild<Tag>ExampleFolders`: Runs a subset of builds. `Tag` can be `Dev`, `Gradle`, `Android`, etc.
+        - `runBuildAllExampleFolders`: Runs all example project builds.
+           Automatically called on `test` when the Gradle property `kotlin.dataframe.debug=true`.
+    - Creates the `testBuildingExamples` source set which contains the 
+      [TestBuildingExampleProjects base class](src/testBuildingExamples/kotlin/dfbuild/buildExampleProjects/TestBuildingExampleProjects.kt),
+      and the generated tests in `/build/generated/testBuildingExamples`,
+      plus the right dependencies to run them. `:build-logic` is one of the dependencies as well,
+      allowing the tests to call helper functions from it.

build-logic/build.gradle.kts

@@ -6,10 +6,16 @@ plugins {
     // Convention plugins are located in `src/main/kotlin`, with the file extension `.gradle.kts`,
     // and are applied in the project's `build.gradle.kts` files as required.
     `kotlin-dsl`
+    alias(libs.plugins.ktlint.gradle)
 }
 
 kotlin {
-    jvmToolchain(21)
+    jvmToolchain(libs.versions.gradle.jdk.get().toInt())
+}
+
+// rules are set up through .editorconfig
+ktlint {
+    version = libs.versions.ktlint.asProvider()
 }
 
 dependencies {