Merge remote-tracking branch 'origin/main' into fix-4.0.1-opa-authorizer

Author: maltesander

CHANGELOG.md

@@ -14,13 +14,21 @@ All notable changes to this project will be documented in this file.
 
 - airflow: Extend list of providers for 3.0.6 ([#1336])
 - airflow: Bump celery version to 5.5.3 for Airflow 3.x ([#1343]).
+- testing-tools: refactoring: Split image into multiple images, remove unnecessary components and switch to UBI as base image ([#1354]).
 - hive: fixed 4.0.1 shaded hive-metastore-opa-authorizer jar by relocating dependencies ([#1356]).
 
+### Removed
+
+- hive: Remove `4.0.0` ([#1340]).
+- opensearch: Remove the `performance-analyzer` plugin from the OpenSearch image ([#1357]).
+
 [#1336]: https://github.com/stackabletech/docker-images/pull/1336
 [#1337]: https://github.com/stackabletech/docker-images/pull/1337
-[#1343]: https://github.com/stackabletech/docker-images/pull/1343
 [#1340]: https://github.com/stackabletech/docker-images/pull/1340
+[#1343]: https://github.com/stackabletech/docker-images/pull/1343
+[#1354]: https://github.com/stackabletech/docker-images/pull/1354
 [#1356]: https://github.com/stackabletech/docker-images/pull/1356
+[#1357]: https://github.com/stackabletech/docker-images/pull/1357
 
 ## [25.11.0] - 2025-11-07
 

opensearch/Dockerfile

@@ -30,7 +30,6 @@ ARG OPENSEARCH_NEURAL_SEARCH_PLUGIN_VERSION
 ARG OPENSEARCH_NOTIFICATIONS_CORE_PLUGIN_VERSION
 ARG OPENSEARCH_NOTIFICATIONS_PLUGIN_VERSION
 ARG OPENSEARCH_OBSERVABILITY_PLUGIN_VERSION
-ARG OPENSEARCH_PERFORMANCE_ANALYZER_PLUGIN_VERSION
 ARG OPENSEARCH_REPORTS_SCHEDULER_PLUGIN_VERSION
 ARG OPENSEARCH_SEARCH_RELEVANCE_PLUGIN_VERSION
 ARG OPENSEARCH_SECURITY_ANALYTICS_PLUGIN_VERSION
@@ -128,7 +127,6 @@ rm -r jdk
     "org.opensearch.plugin:opensearch-notifications-core:${OPENSEARCH_NOTIFICATIONS_CORE_PLUGIN_VERSION}" \
     "org.opensearch.plugin:notifications:${OPENSEARCH_NOTIFICATIONS_PLUGIN_VERSION}" \
     "org.opensearch.plugin:opensearch-observability:${OPENSEARCH_OBSERVABILITY_PLUGIN_VERSION}" \
-    "org.opensearch.plugin:performance-analyzer:${OPENSEARCH_PERFORMANCE_ANALYZER_PLUGIN_VERSION}" \
     "org.opensearch.plugin:opensearch-reports-scheduler:${OPENSEARCH_REPORTS_SCHEDULER_PLUGIN_VERSION}" \
     "org.opensearch.plugin:opensearch-search-relevance:${OPENSEARCH_SEARCH_RELEVANCE_PLUGIN_VERSION}" \
     "org.opensearch.plugin:opensearch-security-analytics:${OPENSEARCH_SECURITY_ANALYTICS_PLUGIN_VERSION}" \
@@ -152,9 +150,7 @@ find /stackable/opensearch-${PRODUCT_VERSION}/config -type f -exec chmod 660 {}
 
 EOF
 
-# The OpenSearch Performance Analyzer needs a JDK, not just a JRE.
-# With a JRE, the following exception is thrown:
-# java.lang.ClassNotFoundException: com.sun.tools.attach.VirtualMachine
+# The upstream image is built with a bundled JDK and not just a JRE, so we also ship OpenSearch with a JDK.
 FROM local-image/jdk-base AS final
 
 ARG PRODUCT_VERSION

opensearch/boil-config.toml

@@ -20,7 +20,6 @@ jdk-base = "21"
 "opensearch-notifications-core-plugin-version" = "3.1.0.0"
 "opensearch-notifications-plugin-version" = "3.1.0.0"
 "opensearch-observability-plugin-version" = "3.1.0.0"
-"opensearch-performance-analyzer-plugin-version" = "3.1.0.0"
 "opensearch-reports-scheduler-plugin-version" = "3.1.0.0"
 "opensearch-search-relevance-plugin-version" = "3.1.0.0"
 "opensearch-security-analytics-plugin-version" = "3.1.0.0"

rust/boil/src/build/cli.rs

@@ -28,7 +28,7 @@ pub struct BuildArguments {
     /// The image version being built.
     #[arg(
         short, long,
-        value_parser = parse_image_version,
+        value_parser = BuildArguments::parse_image_version,
         default_value_t = Self::default_image_version(),
         help_heading = "Image Options"
     )]
@@ -112,15 +112,33 @@ pub struct BuildArguments {
     pub strip_architecture: bool,
 
     /// Loads the image into the local image store.
+    ///
+    /// DEPRECATED: Use -- --load instead.
     #[arg(long, help_heading = "Build Options")]
+    #[deprecated(since = "0.1.7", note = "Use -- --load instead")]
     pub load: bool,
 
     /// Dry run. This does not build the image(s) but instead prints out the bakefile.
     #[arg(short, long, alias = "dry")]
     pub dry_run: bool,
+
+    /// Arguments passed after '--' which are directly passed to the Docker command.
+    ///
+    /// Care needs to be taken, because these arguments can override/modify the behaviour defined
+    /// via the generated Bakefile. A few save arguments include but are not limited to: --load,
+    /// --no-cache, --progress.
+    #[arg(raw = true)]
+    pub rest: Vec<String>,
 }
 
 impl BuildArguments {
+    fn parse_image_version(input: &str) -> Result<Version, ParseImageVersionError> {
+        let version = Version::from_str(input).context(ParseVersionSnafu)?;
+        ensure!(version.build.is_empty(), ContainsBuildMetadataSnafu);
+
+        Ok(version)
+    }
+
     fn default_image_version() -> Version {
         "0.0.0-dev".parse().expect("must be a valid SemVer")
     }
@@ -151,13 +169,6 @@ pub enum ParseImageVersionError {
     ContainsBuildMetadata,
 }
 
-pub fn parse_image_version(input: &str) -> Result<Version, ParseImageVersionError> {
-    let version = Version::from_str(input).context(ParseVersionSnafu)?;
-    ensure!(version.build.is_empty(), ContainsBuildMetadataSnafu);
-
-    Ok(version)
-}
-
 #[derive(Debug, PartialEq, Snafu, EnumDiscriminants)]
 pub enum ParseHostPortError {
     #[snafu(display("unexpected empty input"))]

rust/boil/src/build/mod.rs

@@ -48,7 +48,7 @@ pub enum Error {
 }
 
 /// This is the `boil build` command handler function.
-pub fn run_command(args: BuildArguments, config: Config) -> Result<(), Error> {
+pub fn run_command(args: Box<BuildArguments>, config: Config) -> Result<(), Error> {
     // TODO (@Techassi): Parse Dockerfile instead to build the target graph
     // Validation
     ensure!(
@@ -77,10 +77,12 @@ pub fn run_command(args: BuildArguments, config: Config) -> Result<(), Error> {
     // or by building the image ourself.
 
     // Finally invoke the docker buildx bake command
+    #[allow(deprecated)]
     let mut child = Command::new("docker")
         .arg("buildx")
         .arg("bake")
         .arg_if(args.load, "--load")
+        .args(args.rest)
         .arg("--file")
         .arg("-")
         .stdin(Stdio::piped())

rust/boil/src/cli.rs

@@ -27,7 +27,7 @@ pub enum Command {
     ///
     /// Requires docker with the buildx extension.
     #[command(alias = "some-chicken")]
-    Build(BuildArguments),
+    Build(Box<BuildArguments>),
 
     /// Display various structured outputs in JSON format.
     Show(ShowArguments),

testing-tools/Dockerfile

@@ -1,15 +1,14 @@
 # syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7
 # check=error=true
 
-# Manifest list digest because of multi architecture builds ( https://www.redhat.com/architect/pull-container-image#:~:text=A%20manifest%20list%20exists%20to,system%20on%20a%20specific%20architecture )
-# https://hub.docker.com/_/python/tags
-# In Docker Hub, open up the tag and look for Index Digest. Otherwise do:
-# docker pull python:3.12-slim-bullseye and see the digest that appears in the output.
-FROM python:3.12-slim-bullseye@sha256:411fa4dcfdce7e7a3057c45662beba9dcd4fa36b2e50a2bfcd6c9333e59bf0db
+# Find the latest version at https://catalog.redhat.com/en/software/containers/ubi10/ubi-minimal/66f1504a379b9c2cf23e145c#get-this-image
+# IMPORTANT: Make sure to use the "Manifest List Digest" that references the images for multiple architectures
+# rather than just the "Image Digest" that references the image for the selected architecture.
+FROM registry.access.redhat.com/ubi10/ubi-minimal@sha256:28ec2f4662bdc4b0d4893ef0d8aebf36a5165dfb1d1dc9f46319bd8a03ed3365
 
 ARG PRODUCT_VERSION
+ARG PYTHON_VERSION
 ARG RELEASE_VERSION
-ARG KEYCLOAK_VERSION
 ARG STACKABLE_USER_UID
 ARG STACKABLE_USER_GID
 ARG STACKABLE_USER_NAME
@@ -25,50 +24,54 @@ LABEL name="Stackable Testing Tools" \
 # https://github.com/hadolint/hadolint/wiki/DL4006
 SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
 
-# This is needed so that krb5-user installs without prompting for a realm.
-ENV DEBIAN_FRONTEND=noninteractive
 
+# We configure microdnf to not install weak dependencies in this file
+# Not doing this caused the content of images to become unpredictable because
+# based on which packages get updated by `microdnf update` new weak dependencies
+# might be installed that were not present earlier (the ubi base image doesn't
+# seem to install weak dependencies)
+# This also affects the packages that are installed in our Dockerfiles (java as prime
+# example).
+# https://github.com/stackabletech/docker-images/pull/533
+COPY stackable-base/stackable/dnf.conf /etc/dnf/dnf.conf
 
-COPY testing-tools/python /stackable/python
-COPY testing-tools/licenses /licenses
+# Default curl configuration to avoid forgetting settings and to declutter the Dockerfiles
+COPY stackable-base/stackable/curlrc /root/.curlrc
 
+# Base requirements for all testing-tools images
+COPY testing-tools/requirements.txt /stackable/requirements.txt
+COPY testing-tools/licenses /licenses
 
 RUN <<EOF
-apt-get update
-apt-get install -y --no-install-recommends \
-  build-essential \
-  ca-certificates \
-  curl \
-  gzip \
-  jq \
-  `# krb5-user/libkrb5-dev are needed for Kerberos support. ` \
-  krb5-user \
-  libkrb5-dev \
-  kubernetes-client \
-  libssl-dev \
-  libxml2-dev \
-  libxslt1-dev \
+microdnf update
+
+microdnf install \
+  gcc \
+  make \
   pkg-config \
-  python3-certifi \
-  python3-idna \
-  python3-semver \
-  python3-thrift \
-  python3-toml \
-  python3-urllib3 \
+  openssl-devel \
+  libxml2-devel \
+  libxslt-devel \
+  python${PYTHON_VERSION} \
+  python${PYTHON_VERSION}-devel \
+  python${PYTHON_VERSION}-pip \
+  jq \
+  gzip \
+  curl \
   tar \
   zip \
-  unzip \
-  `# Java 11 seems like the best middle-ground for all tools` \
-  openjdk-11-jdk-headless
+  unzip
 
-apt-get clean
-rm -rf /var/lib/apt/lists/*
+microdnf clean all
+rm -rf /var/cache/yum
 
-curl --fail -L https://repo.stackable.tech/repository/packages/keycloak/keycloak-${KEYCLOAK_VERSION}.tar.gz | tar -xzC /stackable
-ln -s /stackable/keycloak-${KEYCLOAK_VERSION} /stackable/keycloak
+python3.12 -m pip install --no-cache-dir --upgrade pip
+python3.12 -m pip install --no-cache-dir -r /stackable/requirements.txt
 
-pip install --no-cache-dir --upgrade pip
-pip install --no-cache-dir -r /stackable/python/requirements.txt
+ln -s /usr/bin/python3.12 /usr/bin/python
+
+# Added only temporarily to create the user and group, removed again below
+microdnf install shadow-utils
 
 groupadd --gid ${STACKABLE_USER_GID} --system ${STACKABLE_USER_NAME}
 useradd \
@@ -80,11 +83,12 @@ useradd \
   --home-dir /stackable \
    ${STACKABLE_USER_NAME}
 
+microdnf remove shadow-utils
+microdnf clean all
+
 chown -R ${STACKABLE_USER_UID}:0 /stackable
 EOF
 
-ENV PATH=/stackable/keycloak/bin:$PATH
-
 USER ${STACKABLE_USER_UID}
 
 ENV STACKABLE_PRODUCT_VERSION=${PRODUCT_VERSION}

testing-tools/boil-config.toml

@@ -1,2 +1,2 @@
-[versions."0.2.0".build-arguments]
-keycloak-version = "26.3.5"
+[versions."0.3.0".build-arguments]
+python-version = "3.12"

testing-tools/hive/Dockerfile

@@ -0,0 +1,42 @@
+# syntax=docker/dockerfile:1.16.0@sha256:e2dd261f92e4b763d789984f6eab84be66ab4f5f08052316d8eb8f173593acf7
+# check=error=true
+
+FROM local-image/testing-tools
+
+ARG PRODUCT_VERSION
+ARG RELEASE_VERSION
+ARG STACKABLE_USER_UID
+
+LABEL name="Stackable Testing Tools - Hive" \
+      maintainer="info@stackable.tech" \
+      vendor="Stackable GmbH" \
+      version="${PRODUCT_VERSION}" \
+      release="${RELEASE_VERSION}" \
+      summary="Stackable tools for Hive integration tests." \
+      description="Hive-specific testing tools image based on testing-tools base image."
+
+# https://github.com/hadolint/hadolint/wiki/DL4006
+SHELL ["/bin/bash", "-euo", "pipefail", "-c"]
+
+USER root
+
+COPY testing-tools/hive/requirements.txt /stackable/hive/requirements.txt
+
+RUN <<EOF
+microdnf update
+
+microdnf install \
+  krb5-workstation \
+  krb5-devel
+
+microdnf clean all
+rm -rf /var/cache/yum
+
+pip install --no-cache-dir -r /stackable/hive/requirements.txt
+EOF
+
+USER ${STACKABLE_USER_UID}
+
+ENV STACKABLE_PRODUCT_VERSION=${PRODUCT_VERSION}
+
+WORKDIR /stackable

testing-tools/hive/boil-config.toml

@@ -0,0 +1,2 @@
+[versions."0.3.0".local-images]
+testing-tools = "0.3.0"