Skip to content

Commit 32120ba

Browse files
woopwoop
authored
Add load test GitHub Action (#897)
* Fix bug in pre-submit docker build script which trims SHA * Fix typo in docker build * Add load test to GitHub Actions * Fix dependencies * Add wait step for docker images * Use latest version until merged * Update environmental variable docker compose * Add setuptools installation * Fix python comand * Add update command * Change from Python 3 to Python 2 for hdr-plot * Add self-hosted option * Automatically install apt-get * Update dependencies * Add hdr-plot to repository * Allow tests per commit sha * Tighted load test range * Add description * Add label to GitHub Action * Add comments to test-load.sh * Clean up docker compose if failure * Add GitHub Event based SHA * Increase wait time for docker image build * Undo PULL_BASE_SHA commit
1 parent 75f8272 commit 32120ba

File tree

6 files changed

+390
-0
lines changed

6 files changed

+390
-0
lines changed

‎.github/workflows/load_test.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
name: load test
2+
3+
on: [push, pull_request]
4+
5+
jobs:
6+
load-test:
7+
runs-on: [self-hosted, load]
8+
name: load-test
9+
steps:
10+
- uses: actions/checkout@v2
11+
- name: Run load test
12+
run: make test-load
13+
- uses: actions/upload-artifact@v2
14+
with:
15+
name: load-test-results
16+
path: load-test-output/

‎Makefile‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,5 +177,11 @@ build-html: clean-html
177177
cp -r $(ROOT_DIR)/sdk/python/docs/html/* $(ROOT_DIR)/dist/python
178178

179179
# Versions
180+
180181
lint-versions:
181182
./infra/scripts/validate-version-consistency.sh
183+
184+
# Performance
185+
186+
test-load:
187+
./infra/scripts/test-load.sh

‎infra/scripts/test-load.sh‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
#!/usr/bin/env bash
2+
3+
set -e
4+
5+
echo "
6+
============================================================
7+
Running Load Tests
8+
============================================================
9+
"
10+
11+
clean_up() {
12+
ARG=$?
13+
14+
# Shut down docker-compose images
15+
cd "${PROJECT_ROOT_DIR}"/infra/docker-compose
16+
17+
docker-compose \
18+
-f docker-compose.yml \
19+
-f docker-compose.online.yml down
20+
21+
# Remove configuration file
22+
rm .env
23+
24+
exit $ARG
25+
}
26+
27+
CURRENT_SHA=$(git rev-parse HEAD)
28+
29+
if [ "$GITHUB_EVENT_NAME" == "pull_request" ]; then
30+
export CURRENT_SHA=$(cat "$GITHUB_EVENT_PATH" | jq -r .pull_request.head.sha)
31+
fi
32+
33+
export PROJECT_ROOT_DIR=$(git rev-parse --show-toplevel)
34+
export COMPOSE_INTERACTIVE_NO_CLI=1
35+
36+
# Wait for docker images to be available
37+
"${PROJECT_ROOT_DIR}"/infra/scripts/wait-for-docker-images.sh "${CURRENT_SHA}"
38+
39+
# Clean up Docker Compose if failure
40+
trap clean_up EXIT
41+
42+
# Create Docker Compose configuration file
43+
cd "${PROJECT_ROOT_DIR}"/infra/docker-compose/
44+
cp .env.sample .env
45+
46+
# Start Docker Compose containers
47+
FEAST_VERSION=${CURRENT_SHA} docker-compose -f docker-compose.yml -f docker-compose.online.yml up -d
48+
49+
# Get Jupyter container IP address
50+
export JUPYTER_DOCKER_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' feast_jupyter_1)
51+
52+
# Print Jupyter container information
53+
docker inspect feast_jupyter_1
54+
docker logs feast_jupyter_1
55+
56+
# Wait for Jupyter Notebook Container to come online
57+
"${PROJECT_ROOT_DIR}"/infra/scripts/wait-for-it.sh ${JUPYTER_DOCKER_CONTAINER_IP_ADDRESS}:8888 --timeout=60
58+
59+
# Get Feast Core container IP address
60+
export FEAST_CORE_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' feast_core_1)
61+
62+
# Wait for Feast Core to be ready
63+
"${PROJECT_ROOT_DIR}"/infra/scripts/wait-for-it.sh ${FEAST_CORE_CONTAINER_IP_ADDRESS}:6565 --timeout=120
64+
65+
# Get Feast Online Serving container IP address
66+
export FEAST_ONLINE_SERVING_CONTAINER_IP_ADDRESS=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' feast_online-serving_1)
67+
68+
# Wait for Feast Online Serving to be ready
69+
"${PROJECT_ROOT_DIR}"/infra/scripts/wait-for-it.sh ${FEAST_ONLINE_SERVING_CONTAINER_IP_ADDRESS}:6566 --timeout=120
70+
71+
# Ingest data into Feast
72+
pip install --user matplotlib feast pytz matplotlib --upgrade
73+
python "${PROJECT_ROOT_DIR}"/tests/load/ingest.py "${FEAST_CORE_CONTAINER_IP_ADDRESS}":6565 "${FEAST_ONLINE_SERVING_CONTAINER_IP_ADDRESS}":6566
74+
75+
# Download load test tool and proxy
76+
cd $(mktemp -d)
77+
wget -c https://github.com/feast-dev/feast-load-test-proxy/releases/download/v0.1.1/feast-load-test-proxy_0.1.1_Linux_x86_64.tar.gz -O - | tar -xz
78+
git clone https://github.com/giltene/wrk2.git
79+
cd wrk2
80+
make
81+
cd ..
82+
cp wrk2/wrk .
83+
84+
# Start load test server
85+
LOAD_FEAST_SERVING_HOST=${FEAST_ONLINE_SERVING_CONTAINER_IP_ADDRESS} LOAD_FEAST_SERVING_PORT=6566 ./feast-load-test-proxy &
86+
sleep 5
87+
88+
# Run load tests
89+
./wrk -t2 -c10 -d30s -R20 --latency http://localhost:8080/echo
90+
./wrk -t2 -c10 -d30s -R20 --latency http://localhost:8080/send?entity_count=10 > load_test_results_1fs_13f_10e_20rps
91+
./wrk -t2 -c10 -d30s -R50 --latency http://localhost:8080/send?entity_count=10 > load_test_results_1fs_13f_10e_50rps
92+
./wrk -t2 -c10 -d30s -R250 --latency http://localhost:8080/send?entity_count=10 > load_test_results_1fs_13f_10e_250rps
93+
./wrk -t2 -c10 -d30s -R20 --latency http://localhost:8080/send?entity_count=50 > load_test_results_1fs_13f_50e_20rps
94+
./wrk -t2 -c10 -d30s -R50 --latency http://localhost:8080/send?entity_count=50 > load_test_results_1fs_13f_50e_50rps
95+
./wrk -t2 -c10 -d30s -R250 --latency http://localhost:8080/send?entity_count=50 > load_test_results_1fs_13f_50e_250rps
96+
97+
# Print load test results
98+
cat $(ls -lah | grep load_test_results | awk '{print $9}' | tr '\n' ' ')
99+
100+
# Create hdr-plot of load tests
101+
export PLOT_FILE_NAME="load_test_graph_${CURRENT_SHA}"_$(date "+%Y%m%d-%H%M%S").png
102+
python $PROJECT_ROOT_DIR/tests/load/hdr_plot.py --output "$PLOT_FILE_NAME" --title "Load test: ${CURRENT_SHA}" $(ls -lah | grep load_test_results | awk '{print $9}' | tr '\n' ' ')
103+
104+
# Persist artifact
105+
mkdir -p "${PROJECT_ROOT_DIR}"/load-test-output/
106+
cp "${PLOT_FILE_NAME}" "${PROJECT_ROOT_DIR}"/load-test-output/

‎infra/scripts/wait-for-docker-images.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/usr/bin/env bash
2+
#
3+
# This script will block until both the Feast Serving and Feast Core docker images are available for use for a specific tag.
4+
#
5+
6+
[[ -z "$1" ]] && { echo "Please pass the Git SHA as the first parameter" ; exit 1; }
7+
8+
GIT_SHA=$1
9+
10+
# Set allowed failure count
11+
poll_count=0
12+
maximum_poll_count=150
13+
14+
# Wait for Feast Core to be available on GCR
15+
until docker pull gcr.io/kf-feast/feast-core:"${GIT_SHA}"
16+
do
17+
# Exit when we have tried enough times
18+
if [[ "$poll_count" -gt "$maximum_poll_count" ]]; then
19+
exit 1
20+
fi
21+
22+
# Sleep and increment counter on failure
23+
echo "gcr.io/kf-feast/feast-core:${GIT_SHA} could not be found";
24+
sleep 5;
25+
((poll_count++))
26+
done
27+
28+
# Wait for Feast Serving to be available on GCR
29+
until docker pull gcr.io/kf-feast/feast-serving:"${GIT_SHA}"
30+
do
31+
# Exit when we have tried enough times
32+
if [[ "$poll_count" -gt "$maximum_poll_count" ]]; then
33+
exit 1
34+
fi
35+
36+
# Sleep and increment counter on failure
37+
echo "gcr.io/kf-feast/feast-serving:${GIT_SHA} could not be found";
38+
sleep 5;
39+
((poll_count++))
40+
done

‎tests/load/hdr_plot.py‎

Lines changed: 118 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
#
2+
# hdr-plot.py v0.2.0 - A simple HdrHistogram plotting script.
3+
# Copyright © 2018 Bruno Bonacci - Distributed under the Apache License v 2.0
4+
#
5+
# usage: hdr-plot.py [-h] [--output OUTPUT] [--title TITLE] [--nobox] files [files ...]
6+
#
7+
# A standalone plotting script for https://github.com/giltene/wrk2 and
8+
# https://github.com/HdrHistogram/HdrHistogram.
9+
#
10+
# This is just a quick and unsophisticated script to quickly plot the
11+
# HdrHistograms directly from the output of `wkr2` benchmarks.
12+
13+
import argparse
14+
import re
15+
import pandas as pd
16+
import matplotlib.pyplot as plt
17+
import matplotlib.ticker as ticker
18+
19+
regex = re.compile(r'\s+([0-9.]+)\s+([0-9.]+)\s+([0-9.]+)\s+([0-9.]+)')
20+
filename = re.compile(r'(.*/)?([^.]*)(\.\w+\d+)?')
21+
22+
23+
def parse_percentiles(file):
24+
lines = [line for line in open(file) if re.match(regex, line)]
25+
values = [re.findall(regex, line)[0] for line in lines]
26+
pctles = [(float(v[0]), float(v[1]), int(v[2]), float(v[3])) for v in values]
27+
percentiles = pd.DataFrame(pctles, columns=['Latency', 'Percentile', 'TotalCount', 'inv-pct'])
28+
return percentiles
29+
30+
31+
def parse_files(files):
32+
return [parse_percentiles(file) for file in files]
33+
34+
35+
def info_text(name, data):
36+
textstr = '%-18s\n------------------\n%-6s = %6.2f ms\n%-6s = %6.2f ms\n%-6s = %6.2f ms\n' % (
37+
name,
38+
"min", data['Latency'].min(),
39+
"median", data[data["Percentile"] == 0.5]["Latency"],
40+
"max", data['Latency'].max())
41+
return textstr
42+
43+
44+
def info_box(ax, text):
45+
props = dict(boxstyle='round', facecolor='lightcyan', alpha=0.5)
46+
47+
# place a text box in upper left in axes coords
48+
ax.text(0.05, 0.95, text, transform=ax.transAxes,
49+
verticalalignment='top', bbox=props, fontname='monospace')
50+
51+
52+
def plot_summarybox(ax, percentiles, labels):
53+
# add info box to the side
54+
textstr = '\n'.join([info_text(labels[i], percentiles[i]) for i in range(len(labels))])
55+
info_box(ax, textstr)
56+
57+
58+
def plot_percentiles(percentiles, labels):
59+
y_range = [0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50]
60+
x_range = [0.25, 0.5, 0.9, 0.99, 0.999, 0.9999, 0.99999, 0.999999]
61+
62+
fig, ax = plt.subplots(figsize=(24, 16))
63+
plt.ylim(0, 50)
64+
# plot values
65+
for data in percentiles:
66+
ax.plot(data['Percentile'], data['Latency'])
67+
68+
# set axis and legend
69+
ax.grid()
70+
ax.set(xlabel='Percentile',
71+
ylabel='Latency (milliseconds)',
72+
title='Latency Percentiles (lower is better)')
73+
ax.set_xscale('logit')
74+
plt.yticks(y_range)
75+
plt.xticks(x_range)
76+
majors = ["25%", "50%", "90%", "99%", "99.9%", "99.99%", "99.999%", "99.9999%"]
77+
ax.xaxis.set_major_formatter(ticker.FixedFormatter(majors))
78+
ax.xaxis.set_minor_formatter(ticker.NullFormatter())
79+
plt.legend(bbox_to_anchor=(0., 1.02, 1., .102),
80+
loc=3, ncol=2, borderaxespad=0.,
81+
labels=labels)
82+
83+
return fig, ax
84+
85+
86+
def arg_parse():
87+
parser = argparse.ArgumentParser(description='Plot HDRHistogram latencies.')
88+
parser.add_argument('files', nargs='+', help='list HDR files to plot')
89+
parser.add_argument('--output', default='latency.png',
90+
help='Output file name (default: latency.png)')
91+
parser.add_argument('--title', default='', help='The plot title.')
92+
parser.add_argument("--nobox", help="Do not plot summary box",
93+
action="store_true")
94+
args = parser.parse_args()
95+
return args
96+
97+
98+
def main():
99+
# print command line arguments
100+
args = arg_parse()
101+
102+
# load the data and create the plot
103+
pct_data = parse_files(args.files)
104+
labels = [re.findall(filename, file)[0][1] for file in args.files]
105+
# plotting data
106+
fig, ax = plot_percentiles(pct_data, labels)
107+
# plotting summary box
108+
if not args.nobox:
109+
plot_summarybox(ax, pct_data, labels)
110+
# add title
111+
plt.suptitle(args.title)
112+
# save image
113+
plt.savefig(args.output)
114+
print("Wrote: " + args.output)
115+
116+
117+
if __name__ == "__main__":
118+
main()

0 commit comments

Comments
 (0)