From 7217e82e687d3fcf700e73798c764b6450ac8c90 Mon Sep 17 00:00:00 2001 From: Paulo Valente <16843419+polvalente@users.noreply.github.com> Date: Wed, 30 Oct 2024 02:46:02 -0300 Subject: [PATCH 1/7] fix: exla rpath --- exla/Makefile | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/exla/Makefile b/exla/Makefile index 695c7f9409..08674826f7 100644 --- a/exla/Makefile +++ b/exla/Makefile @@ -61,16 +61,24 @@ else NVCCFLAGS = $(CFLAGS) endif + +ifeq ($(MIX_BUILD_EMBEDDED), true) + XLA_EXTENSION_LIB_RPATH = "xla_extension/lib" +else + XLA_EXTENSION_LIB_RPATH = "../xla_extension/lib" +endif + ifeq ($(UNAME_S), Darwin) - LDFLAGS += -flat_namespace -undefined dynamic_lookup -rpath @loader_path/xla_extension/lib + LDFLAGS += -flat_namespace -undefined dynamic_lookup -rpath @loader_path/$(XLA_EXTENSION_LIB_RPATH) else # Use a relative RPATH, so at runtime libexla.so looks for libxla_extension.so # in ./lib regardless of the absolute location. This way priv can be safely # packed into an Elixir release. Also, we use $$ to escape Makefile variable # and single quotes to escape shell variable - LDFLAGS += -Wl,-rpath,'$$ORIGIN/xla_extension/lib' + LDFLAGS += -Wl,-rpath,'$$ORIGIN/$(XLA_EXTENSION_LIB_RPATH)' endif + $(EXLA_SO): $(EXLA_CACHE_SO) @ mkdir -p $(PRIV_DIR) @ mkdir -p $(PRIV_DIR)/xla_extension From 1b87d8deb7a34ca3e1d79904319ea3c995cf52b6 Mon Sep 17 00:00:00 2001 From: Paulo Valente <16843419+polvalente@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:18:29 -0300 Subject: [PATCH 2/7] refactor: copy versioned cache to cache --- exla/Makefile | 21 +++++++++------------ exla/mix.exs | 12 ++++++++++-- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/exla/Makefile b/exla/Makefile index 08674826f7..c11bd31d9b 100644 --- a/exla/Makefile +++ b/exla/Makefile @@ -8,7 +8,8 @@ XLA_EXTENSION_LIB = $(XLA_EXTENSION_DIR)/lib XLA_INCLUDE_PATH = $(XLA_EXTENSION_DIR)/include # Cache configuration -EXLA_CACHE_SO = cache/$(EXLA_VERSION)/libexla.so +EXLA_VERSIONED_CACHE_SO = cache/$(EXLA_VERSION)/libexla.so +EXLA_CACHE_SO = cache/libexla.so EXLA_CACHE_OBJ_DIR = cache/$(EXLA_VERSION)/objs # Private configuration @@ -61,21 +62,14 @@ else NVCCFLAGS = $(CFLAGS) endif - -ifeq ($(MIX_BUILD_EMBEDDED), true) - XLA_EXTENSION_LIB_RPATH = "xla_extension/lib" -else - XLA_EXTENSION_LIB_RPATH = "../xla_extension/lib" -endif - ifeq ($(UNAME_S), Darwin) - LDFLAGS += -flat_namespace -undefined dynamic_lookup -rpath @loader_path/$(XLA_EXTENSION_LIB_RPATH) + LDFLAGS += -flat_namespace -undefined dynamic_lookup -rpath @loader_path/xla_extension/lib else # Use a relative RPATH, so at runtime libexla.so looks for libxla_extension.so # in ./lib regardless of the absolute location. This way priv can be safely # packed into an Elixir release. Also, we use $$ to escape Makefile variable # and single quotes to escape shell variable - LDFLAGS += -Wl,-rpath,'$$ORIGIN/$(XLA_EXTENSION_LIB_RPATH)' + LDFLAGS += -Wl,-rpath,'$$ORIGIN/xla_extension/lib' endif @@ -105,8 +99,11 @@ $(EXLA_CACHE_OBJ_DIR)/%.o: $(EXLA_DIR)/%.cc $(HEADERS) @ mkdir -p $(EXLA_CACHE_OBJ_DIR)/custom_calls $(CXX) $(CFLAGS) -c $< -o $@ -$(EXLA_CACHE_SO): $(XLA_EXTENSION_DIR) $(OBJECTS) - $(CXX) $(OBJECTS) -o $(EXLA_CACHE_SO) $(LDFLAGS) +$(EXLA_VERSIONED_CACHE_SO): $(XLA_EXTENSION_DIR) $(OBJECTS) + $(CXX) $(OBJECTS) -o $(EXLA_VERSIONED_CACHE_SO) $(LDFLAGS) + +$(EXLA_CACHE_SO): $(EXLA_VERSIONED_CACHE_SO) + cp $(EXLA_VERSIONED_CACHE_SO) $(EXLA_CACHE_SO) clean: rm -rf cache diff --git a/exla/mix.exs b/exla/mix.exs index 8e9f66fd03..d511b17e7f 100644 --- a/exla/mix.exs +++ b/exla/mix.exs @@ -143,6 +143,9 @@ defmodule EXLA.MixProject do "0" -> :none + "false" -> + :none + "partial" -> :partial @@ -154,7 +157,7 @@ defmodule EXLA.MixProject do value -> Mix.raise( - "invalid value for EXLA_FORCE_REBUILD: '#{value}'. Expected one of: partial, true" + "invalid value for EXLA_FORCE_REBUILD: '#{value}'. Expected one of: partial, true, false" ) end @@ -183,8 +186,12 @@ defmodule EXLA.MixProject do cached? = File.exists?(cached_so) and force_rebuild_mode == :none if force_rebuild_mode in [:partial, :full] do - Mix.shell().info("Removing cached libexla.so file in cache/#{@version}/libexla.so") + Mix.shell().info( + "Removing cached libexla.so file in cache/#{@version}/libexla.so and cache/libexla.so" + ) + File.rm_rf!("cache/#{@version}/libexla.so") + File.rm_rf!("cache/libexla.so") Mix.shell().info("Removing libexla.so cache at #{cached_so}") File.rm_rf!(cached_so) @@ -193,6 +200,7 @@ defmodule EXLA.MixProject do if cached? do Mix.shell().info("Using libexla.so from #{cached_so}") File.cp!(cached_so, "cache/#{@version}/libexla.so") + File.cp!(cached_so, "cache/libexla.so") end result = Mix.Tasks.Compile.ElixirMake.run(args) From 90eea52b721231839dcde862282b1ab33a114980 Mon Sep 17 00:00:00 2001 From: Paulo Valente <16843419+polvalente@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:11:30 -0300 Subject: [PATCH 3/7] chore: changes due to code review --- exla/Makefile | 6 +----- exla/mix.exs | 21 +++++++-------------- 2 files changed, 8 insertions(+), 19 deletions(-) diff --git a/exla/Makefile b/exla/Makefile index c11bd31d9b..e150fd28b4 100644 --- a/exla/Makefile +++ b/exla/Makefile @@ -8,7 +8,6 @@ XLA_EXTENSION_LIB = $(XLA_EXTENSION_DIR)/lib XLA_INCLUDE_PATH = $(XLA_EXTENSION_DIR)/include # Cache configuration -EXLA_VERSIONED_CACHE_SO = cache/$(EXLA_VERSION)/libexla.so EXLA_CACHE_SO = cache/libexla.so EXLA_CACHE_OBJ_DIR = cache/$(EXLA_VERSION)/objs @@ -99,11 +98,8 @@ $(EXLA_CACHE_OBJ_DIR)/%.o: $(EXLA_DIR)/%.cc $(HEADERS) @ mkdir -p $(EXLA_CACHE_OBJ_DIR)/custom_calls $(CXX) $(CFLAGS) -c $< -o $@ -$(EXLA_VERSIONED_CACHE_SO): $(XLA_EXTENSION_DIR) $(OBJECTS) +$(EXLA_CACHE_SO): $(XLA_EXTENSION_DIR) $(OBJECTS) $(CXX) $(OBJECTS) -o $(EXLA_VERSIONED_CACHE_SO) $(LDFLAGS) -$(EXLA_CACHE_SO): $(EXLA_VERSIONED_CACHE_SO) - cp $(EXLA_VERSIONED_CACHE_SO) $(EXLA_CACHE_SO) - clean: rm -rf cache diff --git a/exla/mix.exs b/exla/mix.exs index d511b17e7f..fa1f63e485 100644 --- a/exla/mix.exs +++ b/exla/mix.exs @@ -135,29 +135,22 @@ defmodule EXLA.MixProject do end defp cached_make(args) do - force_rebuild_mode = - case System.get_env("EXLA_FORCE_REBUILD", "") do - "" -> - :none - - "0" -> - :none + force_rebuild_env_var = System.get_env("EXLA_FORCE_REBUILD", "") - "false" -> + force_rebuild_mode = + cond do + force_rebuild_env_var in ["", "false", "0"] -> :none - "partial" -> + force_rebuild_env_var == "partial" -> :partial - "true" -> - :full - - "1" -> + force_rebuild_env_var in ["true", "1"] -> :full value -> Mix.raise( - "invalid value for EXLA_FORCE_REBUILD: '#{value}'. Expected one of: partial, true, false" + "invalid value for EXLA_FORCE_REBUILD: '#{force_rebuild_env_var}'. Expected one of: partial, true, false" ) end From 582b45b4735f8b7ac6373c377fd5f55bc678e628 Mon Sep 17 00:00:00 2001 From: Paulo Valente <16843419+polvalente@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:12:23 -0300 Subject: [PATCH 4/7] fix: remove versioned libexla.so from mix.exs --- exla/mix.exs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/exla/mix.exs b/exla/mix.exs index fa1f63e485..0ef3f0492d 100644 --- a/exla/mix.exs +++ b/exla/mix.exs @@ -179,11 +179,8 @@ defmodule EXLA.MixProject do cached? = File.exists?(cached_so) and force_rebuild_mode == :none if force_rebuild_mode in [:partial, :full] do - Mix.shell().info( - "Removing cached libexla.so file in cache/#{@version}/libexla.so and cache/libexla.so" - ) + Mix.shell().info("Removing cached libexla.so file in cache/libexla.so") - File.rm_rf!("cache/#{@version}/libexla.so") File.rm_rf!("cache/libexla.so") Mix.shell().info("Removing libexla.so cache at #{cached_so}") @@ -192,7 +189,6 @@ defmodule EXLA.MixProject do if cached? do Mix.shell().info("Using libexla.so from #{cached_so}") - File.cp!(cached_so, "cache/#{@version}/libexla.so") File.cp!(cached_so, "cache/libexla.so") end @@ -201,7 +197,7 @@ defmodule EXLA.MixProject do if not cached? and match?({:ok, _}, result) do Mix.shell().info("Caching libexla.so at #{cached_so}") File.mkdir_p!(Path.dirname(cached_so)) - File.cp!("cache/#{@version}/libexla.so", cached_so) + File.cp!("cache/libexla.so", cached_so) end result From bc7feadbc117a498ec1bf35f1eb6916f67c7ca7c Mon Sep 17 00:00:00 2001 From: Paulo Valente <16843419+polvalente@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:13:49 -0300 Subject: [PATCH 5/7] fix: cond --- exla/mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exla/mix.exs b/exla/mix.exs index 0ef3f0492d..008e17ba05 100644 --- a/exla/mix.exs +++ b/exla/mix.exs @@ -148,7 +148,7 @@ defmodule EXLA.MixProject do force_rebuild_env_var in ["true", "1"] -> :full - value -> + true -> Mix.raise( "invalid value for EXLA_FORCE_REBUILD: '#{force_rebuild_env_var}'. Expected one of: partial, true, false" ) From 412de9aaca3540ec87f4df8728fa17a0bdd17718 Mon Sep 17 00:00:00 2001 From: Paulo Valente <16843419+polvalente@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:39:08 -0300 Subject: [PATCH 6/7] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Valim --- exla/Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/exla/Makefile b/exla/Makefile index e150fd28b4..ada1bb7001 100644 --- a/exla/Makefile +++ b/exla/Makefile @@ -71,7 +71,6 @@ else LDFLAGS += -Wl,-rpath,'$$ORIGIN/xla_extension/lib' endif - $(EXLA_SO): $(EXLA_CACHE_SO) @ mkdir -p $(PRIV_DIR) @ mkdir -p $(PRIV_DIR)/xla_extension @@ -99,7 +98,7 @@ $(EXLA_CACHE_OBJ_DIR)/%.o: $(EXLA_DIR)/%.cc $(HEADERS) $(CXX) $(CFLAGS) -c $< -o $@ $(EXLA_CACHE_SO): $(XLA_EXTENSION_DIR) $(OBJECTS) - $(CXX) $(OBJECTS) -o $(EXLA_VERSIONED_CACHE_SO) $(LDFLAGS) + $(CXX) $(OBJECTS) -o $(EXLA_CACHE_SO) $(LDFLAGS) clean: rm -rf cache From c3dd9ee151a8a4b5d2248bf5618ade9ee50d38af Mon Sep 17 00:00:00 2001 From: Paulo Valente <16843419+polvalente@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:39:34 -0300 Subject: [PATCH 7/7] Update exla/mix.exs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: José Valim --- exla/mix.exs | 1 - 1 file changed, 1 deletion(-) diff --git a/exla/mix.exs b/exla/mix.exs index 008e17ba05..7ce6727d2f 100644 --- a/exla/mix.exs +++ b/exla/mix.exs @@ -180,7 +180,6 @@ defmodule EXLA.MixProject do if force_rebuild_mode in [:partial, :full] do Mix.shell().info("Removing cached libexla.so file in cache/libexla.so") - File.rm_rf!("cache/libexla.so") Mix.shell().info("Removing libexla.so cache at #{cached_so}")