diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index fd2cb06ed..fcad3a0ae 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -118,3 +118,5 @@ steps: - docker-compose#v3.0.0: run: unit-test-docker-sticky-off config: docker/buildkite/docker-compose.yaml + +# TODO: delete this file after migration to github actions diff --git a/.buildkite/scripts/coverage_metadata.sh b/.buildkite/scripts/coverage_metadata.sh index 15ffb3a5d..4296e01d6 100755 --- a/.buildkite/scripts/coverage_metadata.sh +++ b/.buildkite/scripts/coverage_metadata.sh @@ -8,4 +8,6 @@ set -ex output_path="$1" echo "commit-sha: $(git rev-parse HEAD)" > "$output_path" echo "timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$output_path" -echo "Coverage metadata written to $output_path" \ No newline at end of file +echo "Coverage metadata written to $output_path" + +# TODO: delete this file after migration to github actions diff --git a/.buildkite/scripts/lint.sh b/.buildkite/scripts/lint.sh index cf3757453..1b7a19f44 100755 --- a/.buildkite/scripts/lint.sh +++ b/.buildkite/scripts/lint.sh @@ -9,4 +9,6 @@ if [ -n "$(git status --porcelain)" ]; then echo "Please rerun the command and commit the changes" git status --porcelain exit 1 -fi \ No newline at end of file +fi + +# TODO: delete this file after migration to github actions diff --git a/.github/scripts/coverage_metadata.sh b/.github/scripts/coverage_metadata.sh new file mode 100755 index 000000000..15ffb3a5d --- /dev/null +++ b/.github/scripts/coverage_metadata.sh @@ -0,0 +1,11 @@ +#!/bin/sh +set -ex +# This script generates coverage metadata for the coverage report. +# Output is used by SonarQube integration in Uber and not used by OS repo coverage tool itself. +# Example output: +# commit-sha: 6953daa563e8e44512bc349c9608484cfd4ec4ff +# timestamp: 2024-03-04T19:29:16Z +output_path="$1" +echo "commit-sha: $(git rev-parse HEAD)" > "$output_path" +echo "timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" >> "$output_path" +echo "Coverage metadata written to $output_path" \ No newline at end of file diff --git a/.github/scripts/lint.sh b/.github/scripts/lint.sh new file mode 100755 index 000000000..cf3757453 --- /dev/null +++ b/.github/scripts/lint.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +set -ex + +./gradlew goJF + +if [ -n "$(git status --porcelain)" ]; then + echo "There are changes after linting (used goJF) cmd: ./gradlew goJF" + echo "Please rerun the command and commit the changes" + git status --porcelain + exit 1 +fi \ No newline at end of file diff --git a/.github/workflows/ci-checks.yml b/.github/workflows/ci-checks.yml new file mode 100644 index 000000000..10c0f7caf --- /dev/null +++ b/.github/workflows/ci-checks.yml @@ -0,0 +1,102 @@ +name: CI Checks +on: + push: + pull_request: + +jobs: + lint-check: + name: Lint Check + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 # get full history for branch checking + + - name: Run lint check + run: docker compose -f docker/github_actions/docker-compose.yml run unit-test-test-service bash -c ".github/scripts/lint.sh" + + unit-test-with-test-service: + name: Unit Test with Test Service + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 # get full history for branch checking + + - name: Run unit test with test service + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 30 + command: | + docker compose -f docker/github_actions/docker-compose.yml run unit-test-test-service bash -c "./gradlew --no-daemon test jacocoTestReport && .github/scripts/coverage_metadata.sh build/reports/metadata.txt" + + - name: Upload coverage artifacts + uses: actions/upload-artifact@v4 + with: + name: unit-test-with-test-service-coverage + path: | + build/reports/tests/test/**/* + build/reports/jacoco/test/jacocoTestReport.xml + build/reports/metadata.txt + include-hidden-files: true + + unit-test-with-docker-service-sticky-on: + name: Unit Test with Docker Service (Sticky On) + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 # get full history for branch checking + + - name: Run unit test with docker service (sticky on) + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 50 + command: | + docker compose -f docker/github_actions/docker-compose.yml run unit-test-docker-sticky-on bash -c "./gradlew --no-daemon test" + + - name: Upload coverage artifacts + uses: actions/upload-artifact@v4 + with: + name: unit-test-with-docker-service-sticky-on-coverage + path: | + build/reports/tests/test/**/* + include-hidden-files: true + + unit-test-with-docker-service-sticky-off: + name: Unit Test with Docker Service (Sticky Off) + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 # get full history for branch checking + + - name: Run unit test with docker service (sticky off) + uses: nick-fields/retry@v3 + with: + max_attempts: 3 + timeout_minutes: 50 + command: | + docker compose -f docker/github_actions/docker-compose.yml run unit-test-docker-sticky-off bash -c "./gradlew --no-daemon test" + + - name: Upload coverage artifacts + uses: actions/upload-artifact@v4 + with: + name: unit-test-with-docker-service-sticky-off-coverage + path: | + build/reports/tests/test/**/* + include-hidden-files: true \ No newline at end of file diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml index 30bdef8eb..22f304655 100644 --- a/.github/workflows/codecov.yml +++ b/.github/workflows/codecov.yml @@ -9,7 +9,7 @@ jobs: with: submodules: 'true' - name: Install dependencies, run tests, and collect coverage - run: docker compose -f docker/buildkite/docker-compose.yaml run unit-test-test-service + run: docker compose -f docker/github_actions/docker-compose.yml run unit-test-test-service - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 env: diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..dba07a534 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,69 @@ +services: + cassandra: + image: cassandra:4.1.1 + ports: + - "9042:9042" + environment: + - "MAX_HEAP_SIZE=256M" + - "HEAP_NEWSIZE=128M" + healthcheck: + test: ["CMD", "cqlsh", "-u cassandra", "-p cassandra" ,"-e describe keyspaces"] + interval: 15s + timeout: 30s + retries: 10 + prometheus: + image: prom/prometheus:latest + volumes: + - ./prometheus:/etc/prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yml' + ports: + - '9090:9090' + node-exporter: + image: prom/node-exporter + ports: + - '9100:9100' + cadence: + image: ubercadence/server:master-auto-setup + ports: + - "8000:8000" + - "8001:8001" + - "8002:8002" + - "8003:8003" + - "7933:7933" + - "7934:7934" + - "7935:7935" + - "7939:7939" + - "7833:7833" + - "7936:7936" + environment: + - "CASSANDRA_SEEDS=cassandra" + - "PROMETHEUS_ENDPOINT_0=0.0.0.0:8000" + - "PROMETHEUS_ENDPOINT_1=0.0.0.0:8001" + - "PROMETHEUS_ENDPOINT_2=0.0.0.0:8002" + - "PROMETHEUS_ENDPOINT_3=0.0.0.0:8003" + - "DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development.yaml" + - "FRONTEND_PPROF_PORT=7936" + - "LOG_LEVEL=debug" + depends_on: + cassandra: + condition: service_healthy + prometheus: + condition: service_started + cadence-web: + image: ubercadence/web:latest + environment: + - "CADENCE_GRPC_PEERS=cadence:7833" + ports: + - "8088:8088" + depends_on: + - cadence + grafana: + image: grafana/grafana + volumes: + - ./grafana:/etc/grafana + user: "1000" + depends_on: + - prometheus + ports: + - '3000:3000' diff --git a/docker/buildkite/Dockerfile b/docker/buildkite/Dockerfile index 88e1957ec..b192fd72a 100644 --- a/docker/buildkite/Dockerfile +++ b/docker/buildkite/Dockerfile @@ -33,3 +33,5 @@ RUN apk del build-dependencies wget && rm -rf /var/cache/apk/* RUN mkdir /cadence-java-client WORKDIR /cadence-java-client + +# TODO: delete this file after migration to github actions diff --git a/docker/buildkite/README.md b/docker/buildkite/README.md index dc5c3d6d0..decf6bb94 100644 --- a/docker/buildkite/README.md +++ b/docker/buildkite/README.md @@ -19,3 +19,4 @@ And finally make sure to shutdown all docker resources: ```bash docker-compose down ``` +TODO: delete this file after migration to github actions diff --git a/docker/buildkite/docker-compose.yaml b/docker/buildkite/docker-compose.yaml index bf0d41381..44cab51e7 100644 --- a/docker/buildkite/docker-compose.yaml +++ b/docker/buildkite/docker-compose.yaml @@ -93,3 +93,5 @@ services: - "USE_DOCKER_SERVICE=false" volumes: - "../../:/cadence-java-client" + +# TODO: delete this file after migration to github actions diff --git a/docker/github_actions/Dockerfile b/docker/github_actions/Dockerfile new file mode 100644 index 000000000..80d770d4a --- /dev/null +++ b/docker/github_actions/Dockerfile @@ -0,0 +1,30 @@ +# Always use a release version or a specific commit hash below. Don't use +# a branch name since we'd keep getting updates as long as there are new +# commits to that branch and one of them can break the build +FROM adoptopenjdk/openjdk11:jdk-11.0.10_9-alpine + +# Apache Thrift version +ENV APACHE_THRIFT_VERSION=0.9.3 + +# Install dependencies using apk +RUN apk update && apk add --virtual wget ca-certificates wget && apk add --virtual build-dependencies build-base gcc +# Git is needed in order to update the dls submodule +RUN apk add git libstdc++ bash curl + +# Compile source +RUN set -ex ;\ + wget https://archive.apache.org/dist/thrift/${APACHE_THRIFT_VERSION}/thrift-${APACHE_THRIFT_VERSION}.tar.gz && \ + tar -xvf thrift-${APACHE_THRIFT_VERSION}.tar.gz && \ + rm thrift-${APACHE_THRIFT_VERSION}.tar.gz && \ + cd thrift-${APACHE_THRIFT_VERSION}/ && \ + ./configure --enable-libs=no --enable-tests=no --enable-tutorial=no --with-cpp=no --with-c_glib=no --with-java=yes --with-ruby=no --with-erlang=no --with-go=no --with-nodejs=no --with-python=no && \ + make && \ + make install && \ + cd .. && \ + rm -rf thrift-${APACHE_THRIFT_VERSION} + +# Cleanup packages and remove cache +RUN apk del build-dependencies wget && rm -rf /var/cache/apk/* + +RUN mkdir /cadence-java-client +WORKDIR /cadence-java-client diff --git a/docker/github_actions/README.md b/docker/github_actions/README.md new file mode 100644 index 000000000..e06e7fc6e --- /dev/null +++ b/docker/github_actions/README.md @@ -0,0 +1,21 @@ +# Debug Github Actions integration test +Sometimes the environment on IDE may be different as Github Actions. +You can run the following command to trigger the same test as running on Github Actions: + +```bash +docker-compose -f docker/github_actions/docker-compose.yml run unit-test-docker-sticky-on &> test.log +``` +Or +```bash +docker-compose -f docker/github_actions/docker-compose.yml run unit-test-docker-sticky-off &> test.log +``` + +Or +```bash +docker-compose -f docker/github_actions/docker-compose.yml run unit-test-test-service &> test.log +``` + +And finally make sure to shutdown all docker resources: +```bash +docker-compose down +``` diff --git a/docker/github_actions/docker-compose.yml b/docker/github_actions/docker-compose.yml new file mode 100644 index 000000000..af56802fb --- /dev/null +++ b/docker/github_actions/docker-compose.yml @@ -0,0 +1,95 @@ +version: '3.5' + +services: + cassandra: + image: cassandra:4.1.3 + deploy: + resources: + limits: + cpus: '4' + memory: 12000M + ports: + - "9042:9042" + + statsd: + image: hopsoft/graphite-statsd + deploy: + resources: + limits: + cpus: '0.50' + memory: 1024M + ports: + - "8080:80" + - "2003:2003" + - "8125:8125" + - "8126:8126" + + cadence: + image: ubercadence/server:master-auto-setup + deploy: + resources: + limits: + cpus: '2' + memory: 8192M + ports: + - "7933:7933" + - "7934:7934" + - "7935:7935" + - "7939:7939" + environment: + - "CASSANDRA_SEEDS=cassandra" + - "STATSD_ENDPOINT=statsd:8125" + depends_on: + - cassandra + - statsd + + unit-test-docker-sticky-off: + build: + context: ../../ + dockerfile: ./docker/github_actions/Dockerfile + command: "./gradlew --no-daemon test" + deploy: + resources: + limits: + cpus: '2' + memory: 8192M + environment: + - "USER=unittest" + - "CADENCE_SEEDS=cadence" + - "USE_DOCKER_SERVICE=true" + - "STICKY_OFF=true" + depends_on: + - cadence + volumes: + - "../../:/cadence-java-client" + + unit-test-docker-sticky-on: + build: + context: ../../ + dockerfile: ./docker/github_actions/Dockerfile + command: "./gradlew --no-daemon test" + deploy: + resources: + limits: + cpus: '2' + memory: 8192M + environment: + - "USER=unittest" + - "CADENCE_SEEDS=cadence" + - "USE_DOCKER_SERVICE=true" + - "STICKY_OFF=false" + depends_on: + - cadence + volumes: + - "../../:/cadence-java-client" + + unit-test-test-service: + build: + context: ../../ + dockerfile: ./docker/github_actions/Dockerfile + command: "./gradlew --no-daemon test" + environment: + - "USER=unittest" + - "USE_DOCKER_SERVICE=false" + volumes: + - "../../:/cadence-java-client"