Add encryption support through SQLite3MultipleCiphers

.github/workflows/prebuild_assets.yml

@@ -26,7 +26,7 @@ jobs:
         id: cache_prebuild
         with:
           path: internal/prebuild-binaries/build/output
-          key: sqlite-build-${{ hashFiles('internal/prebuild-binaries', 'plugins/build-plugin') }}
+          key: sqlite-build-${{ hashFiles('internal/prebuild-binaries/build.gradle.kts', 'plugins/build-plugin/src') }}
 
       - name: Validate Gradle Wrapper
         if: steps.cache_prebuild.outputs.cache-hit != 'true'
@@ -50,10 +50,18 @@ jobs:
       - name: Set up XCode
         if: steps.cache_prebuild.outputs.cache-hit != 'true'
         uses: maxim-lobanov/setup-xcode@v1
+      - name: Download build tools
+        if: steps.cache_prebuild.outputs.cache-hit != 'true'
+        run: |
+          brew install lld
+          cd internal/prebuild-binaries
+          ./download_glibc.sh
+          ./download_llvm_mingw.sh
+
       - name: Compile SQLite with Gradle
         if: steps.cache_prebuild.outputs.cache-hit != 'true'
         run: |
-          ./gradlew --scan internal:prebuild-binaries:compileNative
+          ./gradlew --scan internal:prebuild-binaries:compileAll
         shell: bash
       - uses: actions/upload-artifact@v5
         id: upload

README.md

@@ -41,6 +41,9 @@ and API documentation [here](https://powersync-ja.github.io/powersync-kotlin/).
     connector provides the connection between your application backend and the PowerSync managed database. It is used to:
       1. Retrieve a token to connect to the PowerSync service.
       2. Apply local changes on your backend application server (and from there, to your backend database).
+- [sqlite3multipleciphers](./sqlite3multipleciphers/)
+
+    - A SQLite driver implementation based on SQLite3MultipleCiphers.
 
 ## Demo Apps / Example Projects
 

build.gradle.kts

@@ -73,6 +73,7 @@ dependencies {
     dokka(project(":integrations:room"))
     dokka(project(":integrations:sqldelight"))
     dokka(project(":integrations:supabase"))
+    dokka(projects.sqlite3multipleciphers)
 }
 
 dokka {

common/build.gradle.kts

@@ -137,6 +137,7 @@ kotlin {
                 optIn("kotlinx.cinterop.ExperimentalForeignApi")
                 optIn("kotlin.time.ExperimentalTime")
                 optIn("kotlin.experimental.ExperimentalObjCRefinement")
+                optIn("com.powersync.PowerSyncInternal")
             }
         }
 
@@ -239,8 +240,6 @@ android {
                 .toInt()
         consumerProguardFiles("proguard-rules.pro")
     }
-
-    ndkVersion = "27.1.12297006"
 }
 
 tasks.named<ProcessResources>(kotlin.jvm().compilations["main"].processResourcesTaskName) {

common/src/commonMain/kotlin/com/powersync/PowerSyncInternal.kt

@@ -0,0 +1,12 @@
+package com.powersync
+
+@RequiresOptIn(message = "This API should not be used outside of PowerSync SDK packages")
+@Retention(AnnotationRetention.BINARY)
+@Target(
+    AnnotationTarget.CLASS,
+    AnnotationTarget.FUNCTION,
+    AnnotationTarget.CONSTRUCTOR,
+    AnnotationTarget.PROPERTY,
+    AnnotationTarget.VALUE_PARAMETER,
+)
+public annotation class PowerSyncInternal

common/src/jvmMain/kotlin/com/powersync/ConnectionFactory.jvm.kt

@@ -5,4 +5,4 @@ import com.powersync.db.runWrapped
 @Throws(PowerSyncException::class)
 public actual fun resolvePowerSyncLoadableExtensionPath(): String? = runWrapped { powersyncExtension }
 
-private val powersyncExtension: String by lazy { extractLib("powersync") }
+private val powersyncExtension: String by lazy { extractLib(BuildConfig::class, "powersync") }

common/src/jvmMain/kotlin/com/powersync/ExtractLib.kt

@@ -2,10 +2,13 @@ package com.powersync
 
 import java.io.File
 import java.util.UUID
+import kotlin.reflect.KClass
 
-private class R
-
-internal fun extractLib(fileName: String): String {
+@PowerSyncInternal
+public fun extractLib(
+    reference: KClass<*>,
+    fileName: String,
+): String {
     val os = System.getProperty("os.name").lowercase()
     val (prefix, extension) =
         when {
@@ -34,7 +37,7 @@ internal fun extractLib(fileName: String): String {
 
     val resourcePath = "/$prefix${fileName}_$arch.$extension"
 
-    (R::class.java.getResourceAsStream(resourcePath) ?: error("Resource $resourcePath not found")).use { input ->
+    (reference.java.getResourceAsStream(resourcePath) ?: error("Resource $resourcePath not found")).use { input ->
         file.outputStream().use { output -> input.copyTo(output) }
     }
 

core-tests-android/build.gradle.kts

@@ -8,6 +8,7 @@ plugins {
 
 dependencies {
     implementation(projects.core)
+    implementation(projects.sqlite3multipleciphers)
     implementation(libs.androidx.core)
     implementation(libs.androidx.appcompat)
     implementation(libs.androidx.material)

core-tests-android/src/androidTest/java/com/powersync/EncryptedDatabaseTest.kt

@@ -0,0 +1,57 @@
+package com.powersync
+
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import androidx.test.platform.app.InstrumentationRegistry
+import androidx.sqlite.SQLiteException
+import androidx.sqlite.execSQL
+import app.cash.turbine.turbineScope
+import com.powersync.db.schema.Schema
+import com.powersync.encryption.AndroidEncryptedDatabaseFactory
+import com.powersync.encryption.Key
+import com.powersync.testutils.UserRow
+import kotlinx.coroutines.*
+import kotlinx.coroutines.runBlocking
+import kotlinx.coroutines.test.runTest
+import org.junit.After
+import org.junit.Assert.*
+import org.junit.Before
+import org.junit.Test
+import org.junit.runner.RunWith
+
+@RunWith(AndroidJUnit4::class)
+class EncryptedDatabaseTest {
+
+    @Test
+    fun testEncryptedDatabase()  =
+        runTest {
+            val context = InstrumentationRegistry.getInstrumentation().targetContext
+
+            val database = PowerSyncDatabase(
+                factory = AndroidEncryptedDatabaseFactory(
+                    context,
+                    Key.Passphrase("mykey")
+                ),
+                schema = Schema(UserRow.table),
+                dbFilename = "encrypted_test",
+            )
+
+            assertEquals("chacha20", database.get("PRAGMA cipher") { it.getString(0)!! })
+
+            database.execute(
+                "INSERT INTO users (id, name, email) VALUES (uuid(), ?, ?)",
+                listOf("Test", "test@example.org"),
+            )
+            database.close()
+
+            val unencryptedFactory = DatabaseDriverFactory(context)
+            val unencrypted = unencryptedFactory.openConnection("encrypted_test", null, false)
+
+            try {
+                unencrypted.execSQL("SELECT * FROM sqlite_schema")
+                throw IllegalStateException("Was able to read schema from encrypted database without supplying a key")
+            } catch (_: SQLiteException) {
+                // Expected
+            }
+            unencrypted.close()
+        }
+}

core/build.gradle.kts

@@ -106,8 +106,6 @@ android {
                 .toInt()
         consumerProguardFiles("proguard-rules.pro")
     }
-
-    ndkVersion = "27.1.12297006"
 }
 
 // We want to build with recent JDKs, but need to make sure we support Java 8. https://jakewharton.com/build-on-latest-java-test-through-lowest-java/

internal/prebuild-binaries/README.md

@@ -7,9 +7,16 @@ Specifically, this builds:
 
 1. SQLite as a static library for iOS/macOS/watchOS/tvOS (+ simulators).
 2. SQLite3MultipleCiphers as a static library for iOS/macOS/watchOS/tvOS (+ simulators).
+3. SQLite3MultipleCiphers plus JNI wrappers as a dynamic library for Windows, macOS and Linux.
 
 We don't want to build these assets on every build since they're included in a `cinterops` definition file, meaning that
 they would have to be built during Gradle sync, which slows down that process.
 
 Instead, we use a cache for GitHub actions to only recompile these when necessary. During the main build, we then use
 a custom property to download assets instead of recompiling.
+
+This build is currently configured to run on macOS hosts only. Cross-compiling requires additional dependencies:
+
+1. To target Windows, we use [LLVM-mingw](https://github.com/mstorsjo/llvm-mingw), which can be downloaded with the
+   `download_llvm_mingw.sh`.
+2. To target Linux, we use clang. The `download_glibc.sh` file downloads necessary glibc headers and object files.

internal/prebuild-binaries/build.gradle.kts

@@ -1,9 +1,13 @@
 import com.powersync.compile.ClangCompile
 import com.powersync.compile.UnzipSqlite
 import de.undercouch.gradle.tasks.download.Download
+import kotlin.io.path.Path
 import org.gradle.kotlin.dsl.register
 import org.jetbrains.kotlin.konan.target.KonanTarget
 import com.powersync.compile.CreateStaticLibrary
+import com.powersync.compile.JniLibraryCompile
+import com.powersync.compile.JniTarget
+import kotlin.io.path.absolutePathString
 
 plugins {
     alias(libs.plugins.downloadPlugin)
@@ -14,6 +18,13 @@ val sqlite3BaseVersion = "3.51.1"
 val sqlite3ReleaseYear = "2025"
 val sqlite3ExpandedVersion = "3510100"
 
+data class CompiledAsset(
+    val output: Provider<RegularFileProperty>,
+    val fullName: String,
+)
+
+val xCodeInstallation = ClangCompile.resolveXcode(providers)
+
 val downloadSQLiteSources by tasks.registering(Download::class) {
     val zipFileName = "sqlite-amalgamation-$sqlite3ExpandedVersion.zip"
     src("https://www.sqlite.org/$sqlite3ReleaseYear/$zipFileName")
@@ -51,6 +62,55 @@ val unzipSqlite3MultipleCipherSources by tasks.registering(UnzipSqlite::class) {
     )
 }
 
+val prepareAndroidBuild by tasks.registering(Copy::class) {
+    from(unzipSqlite3MultipleCipherSources.flatMap { it.destination }) {
+        include(
+            "sqlite3mc_amalgamation.c",
+            "sqlite3mc_amalgamation.h",
+            "sqlite3.h"
+        )
+    }
+
+    from("jni/CMakeLists.txt")
+    from("jni/sqlite_bindings.cpp")
+    into(layout.buildDirectory.dir("android"))
+}
+
+fun compileJni(target: JniTarget): CompiledAsset {
+    val name = target.filename("sqlite3mc_jni")
+
+    val task = tasks.register<JniLibraryCompile>("compile${target.name}") {
+        this.target.set(target)
+        inputFiles.from(
+            "jni/sqlite_bindings.cpp",
+            unzipSqlite3MultipleCipherSources.flatMap { it.destination.file("sqlite3mc_amalgamation.c") }
+        )
+        include.set(unzipSqlite3MultipleCipherSources.flatMap { it.destination })
+        sharedLibrary.set(layout.buildDirectory.file("jni/$name"))
+
+        when (target) {
+            JniTarget.LINUX_X64, JniTarget.LINUX_ARM -> {}
+            JniTarget.WINDOWS_X64, JniTarget.WINDOWS_ARM -> {
+                // For Windows, we compile with LLVM MinGW: https://github.com/mstorsjo/llvm-mingw
+                val clang = layout.buildDirectory.file("llvm-mingw/bin/clang").map { it.asFile.path }
+                clangPath.set(clang)
+            }
+            JniTarget.MACOS_X64, JniTarget.MACOS_ARM -> {
+                // on macOS: Compile with xcode tools
+                toolchain.set(xCodeInstallation.map {
+                    val xcode = Path(it)
+                    xcode.resolve("Toolchains/XcodeDefault.xctoolchain/usr/bin").absolutePathString()
+                })
+            }
+        }
+    }
+
+    return CompiledAsset(
+        output = task.map { it.sharedLibrary },
+        fullName = name
+    )
+}
+
 fun compileSqliteForKotlinNativeOnApple(library: String, abi: String): TaskProvider<CreateStaticLibrary> {
     val name = "$library$abi"
     val outputDir = layout.buildDirectory.dir("c/$abi")
@@ -66,7 +126,7 @@ fun compileSqliteForKotlinNativeOnApple(library: String, abi: String): TaskProvi
         }
 
         inputs.dir(sourceTask.map { it.destination })
-        include.set(unzipSQLiteSources.flatMap { it.destination })
+        include.set(sourceTask.flatMap { it.destination })
         inputFile.set(sourceTask.flatMap { it.destination.file(filename) })
 
         konanTarget.set(abi)
@@ -82,12 +142,7 @@ fun compileSqliteForKotlinNativeOnApple(library: String, abi: String): TaskProvi
     return createStaticLibrary
 }
 
-data class CompiledAsset(
-    val output: Provider<RegularFileProperty>,
-    val fullName: String,
-)
-
-val compileTasks = buildList {
+val kotlinNativeCompileTasks = buildList {
     val targets = KonanTarget.predefinedTargets.values.filter { it.family.isAppleFamily }.map { it.name }.toList()
     for (library in listOf("sqlite3", "sqlite3mc")) {
         for (abi in targets) {
@@ -101,27 +156,61 @@ val compileTasks = buildList {
 }
 
 val compileNative by tasks.registering(Copy::class) {
-    into(project.layout.buildDirectory.dir("output"))
+    into(project.layout.buildDirectory.dir("output/static"))
 
-    for (task in compileTasks) {
+    for (task in kotlinNativeCompileTasks) {
         from(task.output) {
             rename { task.fullName }
         }
     }
 }
 
+val jniCompileTasks: Map<JniTarget, CompiledAsset> = buildMap {
+    for (target in JniTarget.entries) {
+        put(target, compileJni(target))
+    }
+}
+
+val compileJni by tasks.registering(Copy::class) {
+    into(project.layout.buildDirectory.dir("output/jni"))
+
+    for (task in jniCompileTasks.values) {
+        from(task.output) {
+            rename { task.fullName }
+        }
+    }
+}
+
+val compileAll by tasks.registering {
+    dependsOn(compileNative)
+    dependsOn(compileJni)
+}
+
 val hasPrebuiltAssets = providers.gradleProperty("hasPrebuiltAssets").map { it.toBooleanStrict() }
 
 val nativeSqliteConfiguration by configurations.creating {
     isCanBeResolved = false
 }
+val jniSqlite3McConfiguration by configurations.creating {
+    isCanBeResolved = false
+}
+val androidBuildSourceConfiguration by configurations.creating {
+    // We share the downloaded sqlite3mc sources with the sqlite3multipleciphers project, which uses it to
+    // setup a cmake-based NDK build. Since these work on all platforms and only run when needed, there's no
+    // need to prebuild them.
+    isCanBeResolved = false
+}
 
 artifacts {
     if (hasPrebuiltAssets.getOrElse(false)) {
         // In CI builds, we set hasPrebuiltAssets=true. In that case, contents of build/output have been downloaded from
         // cache and don't need to be rebuilt.
-        add(nativeSqliteConfiguration.name, layout.buildDirectory.dir("output"))
+        add(nativeSqliteConfiguration.name, layout.buildDirectory.dir("output/static"))
+        add(jniSqlite3McConfiguration.name, layout.buildDirectory.dir("output/jni"))
     } else {
         add(nativeSqliteConfiguration.name, compileNative)
+        add(jniSqlite3McConfiguration.name, compileJni)
     }
+
+    add(androidBuildSourceConfiguration.name, prepareAndroidBuild)
 }

internal/prebuild-binaries/download_glibc.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+mkdir -p build/sysroot
+cd build/sysroot
+
+function download_package() {
+  curl -L $1 | tar --extract --gzip
+}
+
+download_package https://archlinux.org/packages/core/x86_64/glibc/download/
+download_package https://archlinux.org/packages/core/x86_64/linux-api-headers/download/
+download_package https://archlinux.org/packages/core/x86_64/gcc/download/
+download_package https://archlinux.org/packages/core/x86_64/gcc-libs/download/
+
+download_package https://archlinux.org/packages/extra/any/aarch64-linux-gnu-glibc/download/
+download_package https://archlinux.org/packages/extra/any/aarch64-linux-gnu-linux-api-headers/download/
+download_package https://archlinux.org/packages/extra/x86_64/aarch64-linux-gnu-gcc/download/