Look in more directories for upgrade binaries

This makes major upgrades compatible with images from other distros.

Issue: PGO-864

Author: cbandy

internal/controller/pgupgrade/jobs.go

@@ -21,6 +21,7 @@ import (
 	"github.com/crunchydata/postgres-operator/internal/feature"
 	"github.com/crunchydata/postgres-operator/internal/initialize"
 	"github.com/crunchydata/postgres-operator/internal/naming"
+	"github.com/crunchydata/postgres-operator/internal/postgres"
 	"github.com/crunchydata/postgres-operator/internal/shell"
 	"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
 )
@@ -87,13 +88,16 @@ func upgradeCommand(spec *v1beta1.PGUpgradeSettings, fetchKeyCommand string) []s
 
 		`section 'Step 2 of 7: Finding data and tools...'`,
 
-		// Expect Postgres executables at the Red Hat paths.
+		// Find directories that contain the desired Postgres executables.
+		`old_bin=$(` + postgres.ShellPath(oldVersion) + ` && command -v postgres)`,
+		`old_bin="${old_bin%/postgres}"`,
+		`new_bin=$(` + postgres.ShellPath(newVersion) + ` && command -v initdb)`,
+		`new_bin="${new_bin%/initdb}"`,
+
 		`old_data="${data_volume}/pg${old_version}"`,
 		`new_data="${data_volume}/pg${new_version}"`,
-		`old_bin="/usr/pgsql-${old_version}/bin"`,
-		`new_bin="/usr/pgsql-${new_version}/bin"`,
 
-		`[[ -x "${old_bin}/postgres" && -x "${new_bin}/initdb" && -d "${old_data}" ]]`,
+		`[[ -d "${old_bin}" && -d "${old_data}" && -d "${new_bin}" ]]`,
 
 		// pg_upgrade writes its files in "${new_data}/pg_upgrade_output.d" since PostgreSQL v15.
 		// Change to a writable working directory to be compatible with PostgreSQL v14 and earlier.
@@ -255,7 +259,7 @@ func removeDataCommand(upgrade *v1beta1.PGUpgrade) []string {
 		`delete() (set -x && rm -rf -- "$@")`,
 
 		`old_data="${data_volume}/pg${old_version}"`,
-		`control=$(LC_ALL=C /usr/pgsql-${old_version}/bin/pg_controldata "${old_data}")`,
+		`control=$(` + postgres.ShellPath(oldVersion) + ` && LC_ALL=C pg_controldata "${old_data}")`,
 		`read -r state <<< "${control##*cluster state:}"`,
 
 		// We expect exactly one state for a replica that has been stopped.

internal/controller/pgupgrade/jobs_test.go

@@ -213,11 +213,13 @@ spec:
           export LD_PRELOAD='libnss_wrapper.so' NSS_WRAPPER_GROUP NSS_WRAPPER_PASSWD
           id; [[ "$(id -nu)" == 'postgres' && "$(id -ng)" == 'postgres' ]]
           section 'Step 2 of 7: Finding data and tools...'
+          old_bin=$(PATH="/usr/lib/postgresql/19/bin:/usr/libexec/postgresql19:/usr/pgsql-19/bin${PATH+:${PATH}}" && command -v postgres)
+          old_bin="${old_bin%/postgres}"
+          new_bin=$(PATH="/usr/lib/postgresql/25/bin:/usr/libexec/postgresql25:/usr/pgsql-25/bin${PATH+:${PATH}}" && command -v initdb)
+          new_bin="${new_bin%/initdb}"
           old_data="${data_volume}/pg${old_version}"
           new_data="${data_volume}/pg${new_version}"
-          old_bin="/usr/pgsql-${old_version}/bin"
-          new_bin="/usr/pgsql-${new_version}/bin"
-          [[ -x "${old_bin}/postgres" && -x "${new_bin}/initdb" && -d "${old_data}" ]]
+          [[ -d "${old_bin}" && -d "${old_data}" && -d "${new_bin}" ]]
           cd "${data_volume}"
           section 'Step 3 of 7: Initializing new data directory...'
           PGDATA="${new_data}" "${new_bin}/initdb" --allow-group-access --data-checksums
@@ -353,7 +355,7 @@ spec:
           printf 'Removing PostgreSQL %s data...\n\n' "$@"
           delete() (set -x && rm -rf -- "$@")
           old_data="${data_volume}/pg${old_version}"
-          control=$(LC_ALL=C /usr/pgsql-${old_version}/bin/pg_controldata "${old_data}")
+          control=$(PATH="/usr/lib/postgresql/19/bin:/usr/libexec/postgresql19:/usr/pgsql-19/bin${PATH+:${PATH}}" && LC_ALL=C pg_controldata "${old_data}")
           read -r state <<< "${control##*cluster state:}"
           [[ "${state}" == 'shut down in recovery' ]] || { printf >&2 'Unexpected state! %q\n' "${state}"; exit 1; }
           delete "${old_data}/pg_wal/"

internal/postgres/config.go

@@ -264,6 +264,17 @@ func Environment(cluster *v1beta1.PostgresCluster) []corev1.EnvVar {
 	}
 }
 
+// ShellPath returns a POSIX shell command that prepends typical Postgres executable paths to the PATH variable.
+func ShellPath(postgresVersion int32) string {
+	return fmt.Sprintf(`PATH="`+
+		strings.Join([]string{
+			`/usr/lib/postgresql/%[1]d/bin`, // Debian
+			`/usr/libexec/postgresql%[1]d`,  // Alpine
+			`/usr/pgsql-%[1]d/bin`,          // Red Hat
+		}, ":")+
+		`${PATH+:${PATH}}"`, postgresVersion)
+}
+
 // reloadCommand returns an entrypoint that convinces PostgreSQL to reload
 // certificate files when they change. The process will appear as name in `ps`
 // and `top`.

internal/postgres/config_test.go

@@ -547,6 +547,37 @@ func TestBashSafeLink(t *testing.T) {
 	})
 }
 
+func TestShellPath(t *testing.T) {
+	t.Parallel()
+
+	script := ShellPath(11)
+
+	assert.Assert(t, cmp.Contains(script, `/usr/lib/postgresql/11/bin`))
+	assert.Assert(t, cmp.Contains(script, `/usr/libexec/postgresql11`))
+	assert.Assert(t, cmp.Contains(script, `/usr/pgsql-11/bin`))
+
+	t.Run("ShellCheckPOSIX", func(t *testing.T) {
+		shellcheck := require.ShellCheck(t)
+
+		dir := t.TempDir()
+		file := filepath.Join(dir, "script.sh")
+		assert.NilError(t, os.WriteFile(file, []byte(script), 0o600))
+
+		// Expect ShellCheck for "sh" to be happy.
+		// - https://www.shellcheck.net/wiki/SC2148
+		cmd := exec.CommandContext(t.Context(), shellcheck, "--enable=all", "--shell=sh", file)
+		output, err := cmd.CombinedOutput()
+		assert.NilError(t, err, "%q\n%s", cmd.Args, output)
+	})
+
+	t.Run("PrettyYAML", func(t *testing.T) {
+		b, err := yaml.Marshal(script)
+		assert.NilError(t, err)
+		assert.Assert(t, !strings.Contains(string(b), `\n`), "expected literal flow scalar, got:\n%s", b)
+		assert.Equal(t, 1, strings.Count(string(b), "\n"), "expected one trailing newline, got:\n%s", b)
+	})
+}
+
 func TestStartupCommand(t *testing.T) {
 	shellcheck := require.ShellCheck(t)
 	t.Parallel()