Use clang driver to compile sqlite3 to wasm

Author: simolus3

sqlite3/README.md

@@ -175,7 +175,7 @@ With wasi in `/usr/share/wasi-sysroot` and the default clang compiler having the
 required builtins, you can setup the build with:
 
 ```
-cmake -S assets/wasm -B .dart_tool/sqlite3_build --toolchain toolchain.cmake
+cmake -S assets/wasm -B .dart_tool/sqlite3_build
 ```
 
 ##### macOS
@@ -196,7 +196,7 @@ Replace `clang/18` with the correct directory if you're using a different versio
 Then, set up the build with
 
 ```
-cmake -Dwasi_sysroot=/opt/wasi-sysroot -Dclang=/opt/homebrew/opt/llvm/bin/clang -S assets/wasm -B .dart_tool/sqlite3_build --toolchain toolchain.cmake
+cmake -Dwasi_sysroot=/opt/wasi-sysroot -Dclang=/opt/homebrew/opt/llvm/bin/clang -S assets/wasm -B .dart_tool/sqlite3_build
 ```
 
 #### Building

sqlite3/assets/wasm/CMakeLists.txt

@@ -1,65 +1,97 @@
-cmake_minimum_required(VERSION 3.14)
+cmake_minimum_required(VERSION 3.24)
+
 set(PROJECT_NAME "sqlite3_web")
 project(${PROJECT_NAME} LANGUAGES C)
 
+set(triple wasm32-unknown-wasi)
+set(wasi_sysroot "/usr/share/wasi-sysroot" CACHE PATH "Path to wasi sysroot")
+set(clang "clang" CACHE FILEPATH "Path to wasm-capable clang executable")
+
 include(FetchContent)
-if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.24.0")
-  # cmake 3.24.0 added the `DOWNLOAD_EXTRACT_TIMESTAMP` and prints an ugly warning when
-  # the default is used, so override it to the recommended behavior.
-  # We can't really ask users to use a cmake that recent, so there's this if here.
-  FetchContent_Declare(
+
+FetchContent_Declare(
     sqlite3
     # NOTE: When changing this, also update `test/wasm/sqlite3_test.dart`
     URL https://sqlite.org/2024/sqlite-autoconf-3460000.tar.gz
     DOWNLOAD_EXTRACT_TIMESTAMP NEW
+)
+
+FetchContent_MakeAvailable(sqlite3)
+
+file(DOWNLOAD https://raw.githubusercontent.com/sqlite/sqlite/master/src/test_vfstrace.c "${CMAKE_BINARY_DIR}/vfstrace.c")
+
+# Generate symbols we need to export from the sqlite3.wasm build
+add_custom_command(
+    OUTPUT required_symbols.txt
+    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../../
+    COMMAND dart run tool/wasm_symbols.dart ${CMAKE_CURRENT_BINARY_DIR}/required_symbols.txt
+    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/../../tool/wasm_symbols.dart
+    VERBATIM
+)
+add_custom_target(required_symbols DEPENDS required_symbols.txt)
+
+macro(base_sqlite3_target name debug)
+  set(clang_output ${name}.clang.wasm)
+  set(init_output ${name}.init.wasm)
+  set(output ${init_output})
+
+  set(sources
+    ${CMAKE_CURRENT_SOURCE_DIR}/os_web.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/helpers.c
+    ${sqlite3_SOURCE_DIR}/sqlite3.c
   )
-else()
-  FetchContent_Declare(
-    sqlite3
-    # NOTE: When changing this, also update `test/wasm/sqlite3_test.dart`
-    URL https://sqlite.org/2024/sqlite-autoconf-3460000.tar.gz
+  set(flags -Wall -Wextra -Wno-unused-parameter -Wno-unused-function)
+
+  if(${debug})
+    list(APPEND sources "${CMAKE_BINARY_DIR}/vfstrace.c")
+    list(APPEND flags "-g" "-DDEBUG")
+  else()
+    list(APPEND flags "-Oz" "-DNDEBUG" "-flto")
+  endif()
+
+  add_custom_command(
+    OUTPUT ${clang_output}
+    COMMAND ${clang} --target=${triple} -std=c23
+      ${flags}
+      -o ${clang_output}
+      -I ${PROJECT_SOURCE_DIR} -I ${sqlite3_SOURCE_DIR}
+      -D_HAVE_SQLITE_CONFIG_H
+      -mcpu=generic
+      -mexec-model=reactor
+      -fno-stack-protector -fno-stack-clash-protection
+      -Wl,--import-memory
+      --sysroot /usr/share/wasi-sysroot
+      ${sources}
+      @${CMAKE_CURRENT_BINARY_DIR}/required_symbols.txt
+    DEPENDS ${sources} required_symbols
+    VERBATIM
   )
-endif()
 
-FetchContent_MakeAvailable(sqlite3)
+  add_custom_command(
+    OUTPUT ${init_output}
+    COMMAND wasm-ctor-eval -c _initialize ${clang_output} -o ${init_output}
+    VERBATIM
+    DEPENDS ${clang_output}
+  )
 
-set(wasm_visibility "__attribute__((visibility(\"default\")))")
+  if(NOT ${debug})
+    set(output ${name}.wasm)
 
-macro(base_sqlite3_target name)
-    add_executable(${name}
-        "${sqlite3_SOURCE_DIR}/sqlite3.c"
-        os_web.c
-        helpers.c
+    add_custom_command(
+        OUTPUT ${output}
+        COMMAND wasm-opt --strip --strip-producers -c -O4 ${init_output} -o ${output}
+        VERBATIM
+        DEPENDS ${init_output}
     )
+  endif()
 
-    target_link_options(${name} PRIVATE -mexec-model=reactor -mcpu=generic -Wl,--import-memory -Wl,--no-entry -Wl,--export-dynamic)
-    target_compile_options(${name} PRIVATE -std=c23 -mcpu=generic)
-    target_include_directories(${name} PRIVATE "${PROJECT_SOURCE_DIR}/")
-    target_include_directories(${name} PRIVATE ${sqlite3_SOURCE_DIR})
-    target_compile_definitions(${name} PRIVATE
-        _HAVE_SQLITE_CONFIG_H
-        SQLITE_API=${wasm_visibility}
-    )
-    set_property(TARGET ${name} PROPERTY INTERPROCEDURAL_OPTIMIZATION TRUE)
+  add_custom_target(${name} DEPENDS ${output})
 endmacro()
 
-base_sqlite3_target(sqlite3_debug)
-file(DOWNLOAD https://raw.githubusercontent.com/sqlite/sqlite/master/src/test_vfstrace.c "${CMAKE_BINARY_DIR}/vfstrace.c")
-target_sources(sqlite3_debug PRIVATE "${CMAKE_BINARY_DIR}/vfstrace.c")
-target_compile_options(sqlite3_debug PRIVATE -g)
-target_compile_definitions(sqlite3_debug PRIVATE SQLITE_ENABLE_VFSTRACE SQLITE_ENABLE_API_ARMOR)
-set_target_properties(sqlite3_debug PROPERTIES OUTPUT_NAME "sqlite3" SUFFIX ".debug.wasm")
-
-base_sqlite3_target(sqlite3_opt)
-target_compile_options(sqlite3_opt PRIVATE -Oz)
-set_target_properties(sqlite3_opt PROPERTIES OUTPUT_NAME "sqlite3" SUFFIX ".tmp.wasm")
-add_custom_command(TARGET sqlite3_opt POST_BUILD
-    WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../../
-    COMMAND dart run tool/wasm_dce.dart ${CMAKE_CURRENT_BINARY_DIR}/sqlite3.tmp.wasm ${CMAKE_CURRENT_BINARY_DIR}/sqlite3.dce.wasm
-    COMMAND wasm-opt ${CMAKE_CURRENT_BINARY_DIR}/sqlite3.dce.wasm -O4 -o ${CMAKE_CURRENT_BINARY_DIR}/sqlite3.wasm
-)
+base_sqlite3_target(sqlite3_debug true)
+base_sqlite3_target(sqlite3_opt false)
 
 add_custom_target(output)
-add_custom_command(TARGET output COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/sqlite3.wasm ${PROJECT_SOURCE_DIR}/../../example/web/sqlite3.wasm)
-add_custom_command(TARGET output COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/sqlite3.debug.wasm ${PROJECT_SOURCE_DIR}/../../example/web/sqlite3.debug.wasm)
+add_custom_command(TARGET output COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/sqlite3_opt.wasm ${PROJECT_SOURCE_DIR}/../../example/web/sqlite3.wasm)
+add_custom_command(TARGET output COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_BINARY_DIR}/sqlite3_debug.init.wasm ${PROJECT_SOURCE_DIR}/../../example/web/sqlite3.debug.wasm)
 add_dependencies(output sqlite3_debug sqlite3_opt)

sqlite3/assets/wasm/toolchain.cmake

@@ -7,9 +7,8 @@ fn main() {
         env::var("WASI_SYSROOT").unwrap_or_else(|_| "/usr/share/wasi-sysroot".to_string());
 
     let cmake_dir = Config::new("../../assets/wasm/")
-        .configure_arg("--toolchain")
-        .configure_arg(std::fs::canonicalize("../../assets/wasm/toolchain.cmake").unwrap())
         .define("wasi_sysroot", &sysroot)
+        .define("CMAKE_C_COMPILER_WORKS", "1")
         .build_target("sqlite3_opt_lib")
         .build_target("help") // We only need the sources
         .build();

sqlite3/example/custom_wasm_build/build.rs

@@ -1,45 +1,21 @@
-import 'dart:convert';
 import 'dart:io';
 
 import 'package:analyzer/dart/analysis/utilities.dart';
 import 'package:analyzer/dart/ast/visitor.dart';
 import 'package:analyzer/src/dart/ast/ast.dart';
 
-void main(List<String> args) async {
-  if (args.length != 2) {
-    print(
-        'Removes elements from a WASM file that are not accessed by the Dart bindings.');
-    print('Usage: dart run tool/wasm_dce.dart in.wasm out.wasm');
-    exit(1);
-  }
-
+/// Writes flags for clang that will `-Wl,--export` all symbols used by the
+/// `sqlite3` Dart package.
+void main(List<String> args) {
   final file = File('lib/src/wasm/wasm_interop.dart');
   final ast = parseString(content: file.readAsStringSync(), path: file.path);
 
   final finder = _FindUsedSymbols();
   ast.unit.accept(finder);
 
-  final list = File('.dart_tool/used_wasm_symbols.json');
-  list.writeAsStringSync(json.encode([
-    for (final entry in finder.symbols)
-      {
-        'name': '_$entry',
-        'export': entry,
-        'root': true,
-      }
-  ]));
-
-  final process = await Process.start(
-    'wasm-metadce',
-    [
-      args[0],
-      '--graph-file=${list.path}',
-      '--output',
-      args[1],
-    ],
-    mode: ProcessStartMode.inheritStdio,
-  );
-  exit(await process.exitCode);
+  final output = File(args[0]);
+  output.writeAsStringSync(
+      finder.symbols.map((e) => '-Wl,--export=$e').join(' '));
 }
 
 class _FindUsedSymbols extends RecursiveAstVisitor<void> {