feat: add LiveView Native support (#9)

* feat: initial lvn setup

* feat: add mix task for asset downloads

* fix: nx_iree as a lib compilation

* feat: add file download mix task

* feat: improve download script

* feat: add nx_iree lib and includes

* feat: update headers and add initial implementation of dynamic module call

* feat: working compilation for ipad

* feat: send bytecode and signature to device given handle_info message

* feat: add msg/reply for sending an execution to a liveview

* feat: add actual cpp bridge

* feat: connect lvn with iree

* fix: update iree

* fix: read data from buffer if data is not present

* fix: constructor clause

* feat: working MVP for on-device execution

* feat: add automatic output decoding

* feat: add camera function

* wip: add image processing example

* feat: first working on-device example

* feat: working noise-add example

* feat: working version with live preview

* feat: set fixed rotation angle

* feat: working demo with noise-adder

* feat: reduce memory leak intensity

* feat: final elixir conf code

* chore: lvn->embedded

* gitignore adjustments

Author: polvalente

.formatter.exs

@@ -1,4 +1,4 @@
 # Used by "mix format"
 [
-  inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"]
+  inputs: ["{mix,.formatter}.exs", "{liveview_native,config,lib,test}/**/*.{ex,exs}"]
 ]

.github/workflows/embedded_devices.yml

@@ -16,7 +16,7 @@ jobs:
       MIX_ENV: prod
       NX_IREE_PREFER_PRECOMPILED: false
       NX_IREE_SOURCE_DIR: ./build-cache/iree
-      IREE_GIT_REV: candidate-20240604.914
+      IREE_GIT_REV: candidate-20240822.993
     strategy:
       fail-fast: true
       matrix:

.github/workflows/precompiled_nif.yml

@@ -16,7 +16,7 @@ jobs:
       MIX_ENV: prod
       NX_IREE_PREFER_PRECOMPILED: false
       NX_IREE_SOURCE_DIR: ./build-cache/iree
-      IREE_GIT_REV: candidate-20240604.914
+      IREE_GIT_REV: candidate-20240822.993
       BUILD_IREE_RUNTIME: false
     strategy:
       fail-fast: false
@@ -83,7 +83,7 @@ jobs:
       MIX_ENV: prod
       NX_IREE_PREFER_PRECOMPILED: false
       NX_IREE_SOURCE_DIR: ./build-cache/iree
-      IREE_GIT_REV: candidate-20240604.914
+      IREE_GIT_REV: candidate-20240822.993
       ImageOS: ubuntu22
       LANG: en_US.UTF-8
       LANGUAGE: en_US:en

.gitignore

@@ -27,3 +27,7 @@ nx_iree-*.tar
 
 /iree-runtime/
 /cache/
+
+/priv/iree-compile
+/priv/iree-runtime
+/priv/lbnx_iree.so

Makefile

@@ -165,7 +165,7 @@ endif
 
 NX_IREE_LIB_DIR = $(MIX_APP_PATH)/priv/iree-runtime
 NX_IREE_LIB_LINK_PATH = $(CWD_RELATIVE_TO_PRIV_PATH)/$(NX_IREE_RUNTIME_LIB)
-NX_IREE_CACHE_SO_LINK_PATH = $(CWD_RELATIVE_TO_PRIV_PATH)/$(NX_IREE_CACHE_SO)
+NX_IREE_CACHE_SO_LINK_PATH = $(NX_IREE_CACHE_SO)
 
 SOURCES = $(wildcard c_src/*.cc)
 HEADERS = $(wildcard c_src/*.h)

cmake/src/runtime.cc

@@ -26,7 +26,7 @@ iree::runtime::Device::~Device() {
   }
 }
 
-iree::runtime::IREETensor::IREETensor(iree_hal_buffer_view_t *buffer_view, iree_hal_element_type_t type) : buffer_view(buffer_view), type(type) {
+iree::runtime::IREETensor::IREETensor(iree_hal_buffer_view_t *buffer_view, iree_hal_element_type_t type, iree_hal_device_t *device) : buffer_view(buffer_view), type(type), device(device) {
   size = iree_hal_buffer_view_byte_length(buffer_view);
   // TODO: fill in dim metadata
 }
@@ -81,6 +81,19 @@ std::vector<char> *iree::runtime::IREETensor::serialize() {
   size_t size_size = sizeof(size);
   buffer->insert(buffer->end(), reinterpret_cast<const char *>(&size), reinterpret_cast<const char *>(&size) + size_size);
 
+  if (data == nullptr) {
+    data = std::malloc(size);
+
+    if (data == nullptr) {
+      return nullptr;
+    }
+
+    auto status = read_buffer(device, buffer_view, data, size);
+
+    if (!iree_status_is_ok(status)) {
+      return nullptr;
+    }
+  }
   // Serialize 'data'
   buffer->insert(buffer->end(), reinterpret_cast<const char *>(data), reinterpret_cast<const char *>(data) + size);
 
@@ -341,7 +354,7 @@ call(iree_vm_instance_t *instance, iree_hal_device_t *device, std::string driver
     const iree_hal_dim_t *out_shape = iree_hal_buffer_view_shape_dims(output_buffer_view);
     iree_hal_element_type_t out_type = iree_hal_buffer_view_element_type(output_buffer_view);
 
-    auto tensor = new iree::runtime::IREETensor(output_buffer_view, out_type);
+    auto tensor = new iree::runtime::IREETensor(output_buffer_view, out_type, device);
     tensor->dims = std::vector<iree_hal_dim_t>();
     for (int j = 0; j < out_shape_rank; j++) {
       tensor->dims.push_back(out_shape[j]);

cmake/src/runtime.h

@@ -41,9 +41,10 @@ class IREETensor {
   std::vector<iree_hal_dim_t> dims;
   iree_hal_element_type_t type;
   iree_hal_buffer_view_t* buffer_view;
+  iree_hal_device_t* device;
 
   IREETensor(char* serialized_data);
-  IREETensor(iree_hal_buffer_view_t* buffer_view, iree_hal_element_type_t type);
+  IREETensor(iree_hal_buffer_view_t* buffer_view, iree_hal_element_type_t type, iree_hal_device_t* device);
   IREETensor(void* data, size_t size, std::vector<int64_t> in_dims, iree_hal_element_type_t type);
 
   // Destructor
@@ -74,7 +75,7 @@ class IREETensor {
 
 iree_vm_instance_t* create_instance();
 iree_hal_driver_registry_t* get_driver_registry();
-iree_hal_device_t* create_device(const std::string& device_uri);
+iree_hal_device_t* create_device(iree_hal_driver_registry_t* registry, const std::string& device_uri);
 
 std::pair<iree_status_t, std::optional<std::vector<iree::runtime::IREETensor*>>>
 call(iree_vm_instance_t* i, iree_hal_device_t*, std::string, unsigned char*, size_t, std::vector<iree::runtime::IREETensor*>);

compiler.exs

@@ -18,6 +18,7 @@ Nx.Defn.default_options(compiler: NxIREE.Compiler, iree_compiler_flags: flags, i
 
 f = Nx.Defn.compile(fun, args)
 
+Nx.default_backend(NxIREE.Tensor)
 arg0 = Nx.tensor([1.0, 2.0, 3.0, 4.0])
 arg1 = Nx.tensor([1, -1, 1, -1])
 f.(arg0, arg1) |> dbg()

embedded_devices/live_nx_iree/.formatter.exs

@@ -0,0 +1,6 @@
+[
+  import_deps: [:ecto, :ecto_sql, :phoenix],
+  subdirectories: ["priv/*/migrations"],
+  plugins: [Phoenix.LiveView.HTMLFormatter],
+  inputs: ["*.{heex,ex,exs}", "{config,lib,test}/**/*.{heex,ex,exs}", "priv/*/seeds.exs"]
+]

embedded_devices/live_nx_iree/.gitignore

@@ -0,0 +1,37 @@
+# The directory Mix will write compiled artifacts to.
+/_build/
+
+# If you run "mix test --cover", coverage assets end up here.
+/cover/
+
+# The directory Mix downloads your dependencies sources to.
+/deps/
+
+# Where 3rd-party dependencies like ExDoc output generated docs.
+/doc/
+
+# Ignore .fetch files in case you like to edit your project deps locally.
+/.fetch
+
+# If the VM crashes, it generates a dump, let's ignore it too.
+erl_crash.dump
+
+# Also ignore archive artifacts (built via "mix archive.build").
+*.ez
+
+# Temporary files, for example, from tests.
+/tmp/
+
+# Ignore package tarball (built via "mix hex.build").
+live_nx_iree-*.tar
+
+# Ignore assets that are produced by build tools.
+/priv/static/assets/
+
+# Ignore digested assets cache.
+/priv/static/cache_manifest.json
+
+# In case you use Node.js/npm, you want to ignore these.
+npm-debug.log
+/assets/node_modules/
+