Improve

Author: iambriccardo

.github/workflows/audit.yml

@@ -14,7 +14,7 @@ permissions:
 
 jobs:
   security_audit:
-    runs-on: blacksmith-2vcpu-ubuntu-2404
+    runs-on: blacksmith-4vcpu-ubuntu-2404
     steps:
       - name: Checkout
         uses: actions/checkout@v4

.github/workflows/ci.yml

@@ -20,7 +20,7 @@ concurrency:
 jobs:
   check:
     name: Lint
-    runs-on: blacksmith-2vcpu-ubuntu-2404
+    runs-on: blacksmith-4vcpu-ubuntu-2404
     strategy:
       matrix:
         check: [fmt, clippy, sort]
@@ -56,7 +56,7 @@ jobs:
 
   test-partial:
     name: Tests (Partial)
-    runs-on: blacksmith-2vcpu-ubuntu-2404
+    runs-on: blacksmith-4vcpu-ubuntu-2404
     # Run on forks.
     if: github.event.pull_request.head.repo.fork == true
     permissions:
@@ -114,7 +114,7 @@ jobs:
 
   test-full:
     name: Tests (Full)
-    runs-on: blacksmith-2vcpu-ubuntu-2404
+    runs-on: blacksmith-4vcpu-ubuntu-2404
     # Run on non-forks.
     if: github.event.pull_request.head.repo.fork == false
     permissions:

.github/workflows/docker-build.yml

@@ -49,10 +49,10 @@ jobs:
       matrix:
         include:
           - platform: linux/amd64
-            runner: blacksmith-2vcpu-ubuntu-2404
+            runner: blacksmith-4vcpu-ubuntu-2404
             arch: amd64
           - platform: linux/arm64
-            runner: blacksmith-2vcpu-ubuntu-2404-arm
+            runner: blacksmith-4vcpu-ubuntu-2404-arm
             arch: arm64
     outputs:
       digest-amd64: ${{ steps.export-digest.outputs.digest-amd64 }}
@@ -119,7 +119,7 @@ jobs:
   create-manifest:
     name: Create Multi-Arch Manifest
     needs: build-platform
-    runs-on: blacksmith-2vcpu-ubuntu-2404
+    runs-on: blacksmith-4vcpu-ubuntu-2404
     if: inputs.push == true
     steps:
       - name: Extract Image Name

.github/workflows/docker-ci.yml

@@ -22,7 +22,7 @@ permissions:
 jobs:
   resolve-ref:
     name: Resolve Checkout Ref
-    runs-on: blacksmith-2vcpu-ubuntu-2404
+    runs-on: blacksmith-4vcpu-ubuntu-2404
     outputs:
       ref: ${{ steps.pick.outputs.ref }}
     steps:
@@ -49,7 +49,7 @@ jobs:
   validate-experimental:
     name: Validate Experimental Build
     needs: resolve-ref
-    runs-on: blacksmith-2vcpu-ubuntu-2404
+    runs-on: blacksmith-4vcpu-ubuntu-2404
     if: github.event_name == 'workflow_dispatch' && inputs.experimental == true
     steps:
       - name: Checkout

.github/workflows/docs.yml

@@ -15,7 +15,7 @@ permissions:
 
 jobs:
   deploy:
-    runs-on: blacksmith-2vcpu-ubuntu-2404
+    runs-on: blacksmith-4vcpu-ubuntu-2404
     if: github.event_name == 'push'
     steps:
       - name: Checkout
@@ -43,7 +43,7 @@ jobs:
         run: mkdocs gh-deploy --force
 
   build:
-    runs-on: blacksmith-2vcpu-ubuntu-2404
+    runs-on: blacksmith-4vcpu-ubuntu-2404
     if: github.event_name == 'pull_request'
     steps:
       - name: Checkout

.github/workflows/prepare-release.yml

@@ -28,7 +28,7 @@ permissions:
 jobs:
   prepare-release:
     name: Prepare Release
-    runs-on: blacksmith-2vcpu-ubuntu-2404
+    runs-on: blacksmith-4vcpu-ubuntu-2404
     env:
       CARGO_TERM_COLOR: always
     steps:

.github/workflows/release.yml

@@ -22,7 +22,7 @@ permissions:
 jobs:
   version:
     name: Determine Version
-    runs-on: blacksmith-2vcpu-ubuntu-2404
+    runs-on: blacksmith-4vcpu-ubuntu-2404
     # Gate: manual dispatch OR merged PRs labeled 'release'.
     if: |
       github.event_name == 'workflow_dispatch' || (
@@ -69,7 +69,7 @@ jobs:
     name: Create Tag
     needs: version
     if: needs.version.outputs.is_pr_merge == 'true' || github.event_name == 'workflow_dispatch'
-    runs-on: blacksmith-2vcpu-ubuntu-2404
+    runs-on: blacksmith-4vcpu-ubuntu-2404
     permissions:
       contents: write
     steps:
@@ -119,7 +119,7 @@ jobs:
   github-release:
     name: Create GitHub Release
     needs: [version, create-tag, build-and-push]
-    runs-on: blacksmith-2vcpu-ubuntu-2404
+    runs-on: blacksmith-4vcpu-ubuntu-2404
     steps:
       - name: Generate Release Notes
         id: notes

Cargo.toml

@@ -71,7 +71,9 @@ serde_json = { version = "1.0.141", default-features = false }
 serde_yaml = { version = "0.9.34", default-features = false }
 sqlx = { version = "0.8.6", default-features = false }
 tempfile = { version = "3.23.0", default-features = false }
-thiserror = "2.0.12"
+thiserror = "2.0.12"    
+tikv-jemalloc-ctl = { version = "0.6.0", default-features = false, features = ["stats"] }
+tikv-jemallocator = { version = "0.6.1", default-features = false, features = ["background_threads_runtime_support", "unprefixed_malloc_on_supported_platforms"] }
 tokio = { version = "1.47.0", default-features = false }
 tokio-postgres = { git = "https://github.com/iambriccardo/rust-postgres", default-features = false, rev = "31acf55c7e5c2244e5bb3a36e7afa2a01bf52c38" }
 tokio-rustls = { version = "0.26.2", default-features = false }

etl-api/Dockerfile

@@ -20,7 +20,8 @@ RUN apt-get update && \
     pkg-config \
     libssl-dev \
     clang \
-    lld && \
+    lld \
+    make && \
     rm -rf /var/lib/apt/lists/*
 
 # Copy cargo config for build optimizations (lld linker, etc.)

etl-api/src/configs/destination.rs

@@ -18,8 +18,10 @@ pub enum FullApiDestinationConfig {
     Memory,
     BigQuery {
         #[schema(example = "my-gcp-project")]
+        #[serde(deserialize_with = "crate::utils::trim_string")]
         project_id: String,
         #[schema(example = "my_dataset")]
+        #[serde(deserialize_with = "crate::utils::trim_string")]
         dataset_id: String,
         #[schema(example = "{\"type\": \"service_account\", \"project_id\": \"my-project\"}")]
         service_account_key: SerializableSecretString,
@@ -462,11 +464,17 @@ pub enum StoredIcebergConfig {
 pub enum FullApiIcebergConfig {
     Supabase {
         #[schema(example = "abcdefghijklmnopqrst")]
+        #[serde(deserialize_with = "crate::utils::trim_string")]
         project_ref: String,
         #[schema(example = "my-warehouse")]
+        #[serde(deserialize_with = "crate::utils::trim_string")]
         warehouse_name: String,
         #[schema(example = "my-namespace")]
-        #[serde(skip_serializing_if = "Option::is_none")]
+        #[serde(
+            default,
+            skip_serializing_if = "Option::is_none",
+            deserialize_with = "crate::utils::trim_option_string"
+        )]
         namespace: Option<String>,
         #[schema(
             example = "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IjFkNzFjMGEyNmIxMDFjODQ5ZTkxZmQ1NjdjYjA5NTJmIn0.eyJleHAiOjIwNzA3MTcxNjAsImlhdCI6MTc1NjE0NTE1MCwiaXNzIjoic3VwYWJhc2UiLCJyZWYiOiJhYmNkZWZnaGlqbGttbm9wcXJzdCIsInJvbGUiOiJzZXJ2aWNlX3JvbGUifQ.YdTWkkIvwjSkXot3NC07xyjPjGWQMNzLq5EPzumzrdLzuHrj-zuzI-nlyQtQ5V7gZauysm-wGwmpztRXfPc3AQ"
@@ -477,20 +485,25 @@ pub enum FullApiIcebergConfig {
         #[schema(example = "ca833e890916d848c69135924bcd75e5909184814a0ebc6c988937ee094120d4")]
         s3_secret_access_key: SerializableSecretString,
         #[schema(example = "ap-southeast-1")]
+        #[serde(deserialize_with = "crate::utils::trim_string")]
         s3_region: String,
     },
     Rest {
         #[schema(example = "https://abcdefghijklmnopqrst.storage.supabase.com/storage/v1/iceberg")]
+        #[serde(deserialize_with = "crate::utils::trim_string")]
         catalog_uri: String,
         #[schema(example = "my-warehouse")]
+        #[serde(deserialize_with = "crate::utils::trim_string")]
         warehouse_name: String,
         #[schema(example = "my-namespace")]
+        #[serde(default, deserialize_with = "crate::utils::trim_option_string")]
         namespace: Option<String>,
         #[schema(example = "9156667efc2c70d89af6588da86d2924")]
         s3_access_key_id: SerializableSecretString,
         #[schema(example = "ca833e890916d848c69135924bcd75e5909184814a0ebc6c988937ee094120d4")]
         s3_secret_access_key: SerializableSecretString,
         #[schema(example = "https://s3.endpoint")]
+        #[serde(deserialize_with = "crate::utils::trim_string")]
         s3_endpoint: String,
     },
 }