Skip to content

Commit 89346f9

Browse files
ivan-khvostishkovivan-khvostishkov
committed
Fix code coverage and add performance profiling reports in pytest
1 parent a36475a commit 89346f9

File tree

12 files changed

+166
-38
lines changed

12 files changed

+166
-38
lines changed

‎.gitignore‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,3 +15,8 @@ __pycache__
1515
/tests/output/
1616
/tests/.pytest_cache/
1717
/cdk.out/
18+
/tests/.coverage
19+
/tests/coverage.xml
20+
/tests/htmlcov/
21+
/tests/prof/
22+
/djl-serving/

‎.gitlab-ci.yml‎

Lines changed: 70 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,10 @@ run_tests:
77
stage: test
88
resource_group: ssh-helper-tests
99
script:
10-
- export AWS_REGION=eu-west-1
11-
- export AWS_DEFAULT_REGION=eu-west-1
10+
- cat /etc/hosts
11+
# Creating venv
12+
- python -m venv ./venv
13+
- source ./venv/bin/activate
1214
# Install the package
1315
- pip freeze --all | tee pip_freeze_before.txt
1416
- pip install '.'
@@ -22,14 +24,22 @@ run_tests:
2224
# Scanning sources
2325
- bandit -r ./sagemaker_ssh_helper/ ./tests/ *.py --skip B603,B404,B101 2>&1 | tee bandit.txt
2426
- flake8 --extend-ignore E501,F401,F541,E402 ./sagemaker_ssh_helper/ ./tests/ *.py | tee flake8.txt
27+
# Configure local env
28+
- id
29+
- apt-get update
30+
- apt-get install -y sudo
31+
- sm-local-configure
32+
- source tests/generate_sagemaker_config.sh
2533
# Update IAM/SSM policies
2634
- curl -fsSL https://deb.nodesource.com/setup_lts.x | bash -
35+
- export DEBIAN_FRONTEND=noninteractive
2736
- apt-get install -y nodejs
2837
- npm install -g aws-cdk
2938
- cdk --version
30-
- REGION=$AWS_REGION
39+
- REGION=eu-west-1
3140
- USER_ROLE=$GITLAB_ROLE_ARN
3241
- APP="python -m sagemaker_ssh_helper.cdk.app"
42+
# Bootstrapping requires Administrator privileges:
3343
# cdk bootstrap aws://"$ACCOUNT_ID"/"$REGION" --require-approval never
3444
- AWS_REGION=$REGION cdk -a "$APP" deploy SSH-IAM-SSM-Stack-Tests
3545
--require-approval never -c sagemaker_role=$SAGEMAKER_ROLE -c user_role=$USER_ROLE
@@ -41,30 +51,38 @@ run_tests:
4151
# cdk bootstrap aws://"$ACCOUNT_ID"/"$REGION" --require-approval never
4252
- AWS_REGION=$REGION cdk -a "$APP" deploy SSM-Advanced-Tier-Stack
4353
--require-approval never -c sagemaker_role=$SAGEMAKER_ROLE -c user_role=$USER_ROLE
44-
# Configure local env
45-
- sm-local-configure
46-
- source tests/generate_sagemaker_config.sh
54+
- unset REGION
55+
- unset USER_ROLE
56+
- unset APP
57+
# Set bucket versioning to detect model repacking / dependencies overrides
58+
- aws s3api put-bucket-versioning
59+
--bucket $(AWS_DEFAULT_REGION=eu-west-1 bash tests/get_sagemaker_bucket.sh)
60+
--versioning-configuration Status=Enabled
61+
- aws s3api put-bucket-versioning
62+
--bucket $(AWS_DEFAULT_REGION=eu-west-2 bash tests/get_sagemaker_bucket.sh)
63+
--versioning-configuration Status=Enabled
64+
# Set default region for tests - need both to avoid confusion because one can override another
65+
- export AWS_REGION=eu-west-1
66+
- export AWS_DEFAULT_REGION=eu-west-1
67+
- aws configure list
4768
# Assume CI/CD role
48-
- sts=( $(
49-
aws sts assume-role
50-
--role-arn "$GITLAB_ROLE_ARN"
51-
--role-session-name "gitlab-ci"
52-
--duration-seconds 18000
53-
--query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]'
54-
--output text
55-
) )
69+
- sts=( $(source tests/assume-sagemaker-role.sh) )
5670
- export AWS_ACCESS_KEY_ID=${sts[0]}
5771
- export AWS_SECRET_ACCESS_KEY=${sts[1]}
5872
- export AWS_SESSION_TOKEN=${sts[2]}
59-
# Build BYOC docker image for tests
60-
- sm-docker build . --file tests/byoc/Dockerfile --repo byoc-ssh:latest
6173
# Run tests
6274
- echo "Extra args for pytest - $PYTEST_EXTRA_ARGS"
6375
- cd tests
64-
- coverage run -m pytest --html=pytest_report.html --self-contained-html --junitxml=pytest_report.xml -m 'not manual' -o sagemaker_role=$SAGEMAKER_ROLE -o kernel_gateway_name=$KERNEL_GATEWAY_NAME $PYTEST_EXTRA_ARGS
76+
- coverage run -m pytest
77+
--html=pytest_report.html --self-contained-html --junitxml=pytest_report.xml
78+
-m 'not manual'
79+
-o sagemaker_studio_domain=$SAGEMAKER_STUDIO_DOMAIN
80+
$PYTEST_EXTRA_ARGS || EXIT_CODE=$?
6581
- coverage report
6682
- coverage xml
83+
- coverage html --show-contexts
6784
- cd -
85+
- exit $EXIT_CODE
6886
coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
6987
artifacts:
7088
when: always
@@ -81,6 +99,40 @@ run_tests:
8199
- pip_freeze_diff_test.txt
82100
- tests/pytest_report.xml
83101
- tests/pytest_report.html
102+
- tests/htmlcov/
84103
- tests/coverage.xml
104+
- tests/.coverage
85105
- bandit.txt
86-
- flake8.txt
106+
- flake8.txt
107+
108+
profile_tests:
109+
stage: test
110+
resource_group: ssh-helper-tests
111+
script:
112+
- python -m venv ./venv
113+
- source ./venv/bin/activate
114+
- pip install '.[test]'
115+
- id
116+
- apt-get update
117+
- apt-get install -y sudo
118+
- sm-local-configure
119+
- source tests/generate_sagemaker_config.sh
120+
- export DEBIAN_FRONTEND=noninteractive
121+
- apt-get install -y graphviz
122+
- export AWS_REGION=eu-west-1
123+
- export AWS_DEFAULT_REGION=eu-west-1
124+
- sts=( $(source tests/assume-sagemaker-role.sh) )
125+
- export AWS_ACCESS_KEY_ID=${sts[0]}
126+
- export AWS_SECRET_ACCESS_KEY=${sts[1]}
127+
- export AWS_SESSION_TOKEN=${sts[2]}
128+
- cd tests
129+
- pytest
130+
-m 'not manual'
131+
-o sagemaker_studio_domain=$SAGEMAKER_STUDIO_DOMAIN
132+
--profile --profile-svg
133+
$PYTEST_EXTRA_ARGS
134+
- cd -
135+
artifacts:
136+
when: always
137+
paths:
138+
- tests/prof/

‎.idea/aws.xml‎

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎.idea/dataSources.xml‎

Lines changed: 12 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎.idea/deployment.xml‎

Lines changed: 19 additions & 12 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎.idea/sagemaker-ssh-helper.iml‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎setup.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
"pytest",
1919
"pytest-cov",
2020
"pytest-html",
21+
"pytest-profiling",
2122
"bandit",
2223
"aws-cdk-lib==2.64.0",
2324
"constructs>=10.0.0,<11.0.0",

‎tests/.coveragerc‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
[run]
2+
branch = True
3+
dynamic_context = test_function
4+
5+
source =
6+
./
7+
./source_dir/training_clean
8+
./source_dir/inference_tf_clean
9+
./source_dir/processing
10+
./source_dir/training_placeholder
11+
./source_dir/training
12+
./source_dir/inference_clean_model2
13+
./source_dir/inference_tf_clean_model2
14+
./source_dir/inference_tf_model2
15+
./source_dir/training_debug
16+
./source_dir/inference_clean
17+
./source_dir/inference
18+
./source_dir/inference_model2
19+
./source_dir/inference_tf
20+
../venv/lib/python3.8/site-packages/sagemaker_ssh_helper/
21+
../venv/lib/python3.8/site-packages/sagemaker/
22+
23+
omit =
24+
**/sagemaker_ssh_helper/cdk/**

‎tests/assume-sagemaker-role.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
aws sts assume-role \
2+
--role-arn "$GITLAB_ROLE_ARN" \
3+
--role-session-name "gitlab-ci" \
4+
--duration-seconds 28800 \
5+
--query 'Credentials.[AccessKeyId,SecretAccessKey,SessionToken]' \
6+
--output text

‎tests/byoc/Dockerfile‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,12 @@
11
# Based on https://github.com/aws/amazon-sagemaker-examples/blob/main/advanced_functionality/scikit_bring_your_own/container/Dockerfile
22

3-
# Run from SageMaker Studio at the project root (where setup.py is located):
43
# pip install sagemaker-studio-image-build
5-
# sm-docker build . --file tests/byoc/Dockerfile --repo byoc-ssh:latest
64

7-
# See https://aws.amazon.com/blogs/machine-learning/using-the-amazon-sagemaker-studio-image-build-cli-to-build-container-images-from-your-studio-notebooks/ .
5+
# Run inside the tests/ directory:
6+
# sm-docker build . --file byoc/Dockerfile --repo byoc-ssh:latest
87

8+
# See https://aws.amazon.com/blogs/machine-learning/using-the-amazon-sagemaker-studio-image-build-cli-to-build-container-images-from-your-studio-notebooks/ .
9+
# Also see https://medium.com/@pandey.vikesh/why-bring-your-own-container-to-amazon-sagemaker-and-how-to-do-it-right-bc158fe41ed1 .
910
FROM public.ecr.aws/docker/library/ubuntu:20.04
1011

1112
RUN apt-get -y update && apt-get install -y --no-install-recommends \
@@ -24,12 +25,12 @@ RUN pip --no-cache-dir install numpy scipy scikit-learn pandas flask gunicorn
2425
# RUN pip --no-cache-dir install sagemaker-ssh-helper # <--- NEW
2526

2627
# Test current release (for developers):
27-
COPY . /tmp/sagemaker-ssh-helper/
28+
COPY ../ /tmp/sagemaker-ssh-helper/
2829
RUN pip --no-cache-dir install /tmp/sagemaker-ssh-helper/
2930

3031
ENV PYTHONUNBUFFERED=TRUE
3132
ENV PYTHONDONTWRITEBYTECODE=TRUE
3233
ENV PATH="/opt/program:${PATH}"
3334

34-
COPY tests/byoc/decision_trees /opt/program
35+
COPY byoc/decision_trees /opt/program
3536
WORKDIR /opt/program

0 commit comments

Comments
 (0)