diff --git a/bench-scripts/README.md b/bench-scripts/README.md index 66aa983..e91f190 100644 --- a/bench-scripts/README.md +++ b/bench-scripts/README.md @@ -57,6 +57,33 @@ Apart from adjusting paths in nginx.conf the script also sets option `work_process` to auto. Nginx server configuration is saved along the results for each test. +## HA-proxy + +The HA-proxy configuration for tests is based on State of SSL stacks [8] +paper published by HA-proxy project. The test uses chain of 10 hA-proxy +instances by default. You may use `BENCH_PROXY_CHAIN` env. variable to +change that. The test collects results for 1, 2, 4, ..., 32, 64 threads. +It uses h1load [9] and siege [6]. Both HA-proxy and h1load client are +linked with target SSL library for testing. The h1load client runs with +options as follows: + + # h1load \ + -c 1280 \ # 1280 concurrent connections + -n 40000 \ # 40000 requests per connection + -r 1 \ # 1 request per connection + -P \ # gather percentile data (currently not processsed) + -t 64 \ # number of threads + https://127.0.0.1:xxxx + +The h1load client fetches a static content provided by HA-proxy. The benchmark +uses the h1load test duration to compare SSL libraries. The longer duration the +worse result. + +The siege client does not build with aws-lc library. To workaround that, +all siege tests connect to HA-proxy via, which then establishes SSL connection +towards httpterm [10] server. To collect performance data The siege client +executes requests which fetch 1k of data from httpterm server. + ## Build requirements Requirements for ubuntu are the following: @@ -145,3 +172,9 @@ should be named as `bench_run_xtool.sh` [6]: https://www.joedog.org/siege-home/ [7]: https://www.joedog.org/siege-manual/ + +[8]: https://www.haproxy.com/blog/state-of-ssl-stacks + +[9]: https://github.com/wtarreau/h1load + +[10]: https://github.com/wtarreau/httpterm diff --git a/bench-scripts/bench_config_haproxy.sh b/bench-scripts/bench_config_haproxy.sh index 5911c7d..18d5d5b 100755 --- a/bench-scripts/bench_config_haproxy.sh +++ b/bench-scripts/bench_config_haproxy.sh @@ -144,15 +144,14 @@ global ssl-server-verify none EOF - } function emit_frontend { - typeset HAPROXY_CONF=$1 - typeset REUSE_LABEL=$2 - typeset BASEPORT=$3 - typeset PROXYCERT=$4 - typeset SSL_REUSE=$5 + typeset HAPROXY_CONF=$1 + typeset REUSE_LABEL=$2 + typeset BASEPORT=$3 + typeset PROXYCERT=$4 + typeset SSL_REUSE=$5 cat <> ${HAPROXY_CONF} @@ -172,12 +171,16 @@ frontend port${BASEPORT} EOF } +# +# http connection to httpterm server. The server +# does not support https +# function emit_httpterm { - typeset HAPROXY_CONF=$1 - typeset REUSE_LABEL=$2 - typeset BASEPORT=$3 - typeset PROXYCERT=$4 - typeset SSL_REUSE=$5 + typeset HAPROXY_CONF=$1 + typeset REUSE_LABEL=$2 + typeset BASEPORT=$3 + typeset PROXYCERT=$4 + typeset SSL_REUSE=$5 cat <> ${HAPROXY_CONF} defaults ${REUSE_LABEL} @@ -199,14 +202,22 @@ backend httpterm${BASEPORT} EOF } +# +# make runtime stats available on URL below. +# https://127.0.0.1:{$PORT_RSA_REUSE, $PORT_RSA}, $PORT_EC_REUSE, $PORT_EC}/stats +# see link here to find more information on this: +# https://www.haproxy.com/documentation/haproxy-configuration-tutorials/alerts-and-monitoring/statistics/ +# function emit_stats { - typeset HAPROXY_CONF=$1 - typeset BASEPORT=$2 - typeset PROXYCERT=$3 + typeset HAPROXY_CONF=$1 + typeset BASEPORT=$2 + typeset PROXYCERT=$3 cat <> ${HAPROXY_CONF} listen port${BASEPORT} bind ${HOST}:${BASEPORT} ssl crt ${PROXYCERT} + stats enable + stats refresh 5s stats uri /stats server next ${HOST}:$(( ${BASEPORT} - 1)) @@ -214,9 +225,9 @@ EOF } function emit_https_port { - typeset HAPROXY_CONF=$1 - typeset PORT=$2 - typeset PROXYCERT=$3 + typeset HAPROXY_CONF=$1 + typeset PORT=$2 + typeset PROXYCERT=$3 cat <> ${HAPROXY_CONF} listen port${PORT} bind ${HOST}:${PORT} ssl crt ${PROXYCERT} @@ -225,10 +236,15 @@ listen port${PORT} EOF } +# +# the http port is used by siege client. +# proxy accepts http connections and then +# connects with https towards proxy-chain +# function emit_http_port { - typeset HAPROXY_CONF=$1 - typeset HTTP_PORT=$2 - typeset PORT=$3 + typeset HAPROXY_CONF=$1 + typeset HTTP_PORT=$2 + typeset PORT=$3 cat <> ${HAPROXY_CONF} listen port${HTTP_PORT} @@ -273,6 +289,7 @@ function config_haproxy { typeset BASEPORT_RSA='' typeset BASEPORT_EC_REUSE='' typeset BASEPORT_EC='' + typeset EMIT_STATS=0 if [[ -z "${SSL_LIB}" ]] ; then SSL_LIB='openssl-=master' @@ -321,23 +338,24 @@ function config_haproxy { emit_httpterm ${HAPROXY_CONF} ${REUSE_LABEL} ${BASEPORT} ${PROXYCERT} ${SSL_REUSE} fi + TOPPORT=$(( ${BASEPORT} + ${PROXY_CHAIN} )) BASEPORT=$(( ${BASEPORT} + 1)) - TOPPORT=$(( ${BASEPORT} + ${PROXY_CHAIN} - 1 )) - emit_stats ${HAPROXY_CONF} ${BASEPORT} ${PROXYCERT} - - BASEPORT=$(( ${BASEPORT} + 1)) + EMIT_STATS=1 for PORT in $(seq ${BASEPORT} ${TOPPORT}) ; do - emit_https_port ${HAPROXY_CONF} ${PORT} ${PROXYCERT} + if [[ ${EMIT_STATS} -eq 1 ]] ; then + emit_stats ${HAPROXY_CONF} ${BASEPORT} ${PROXYCERT} + EMIT_STATS=0 + else + emit_https_port ${HAPROXY_CONF} ${PORT} ${PROXYCERT} + fi done - PORT=$(( ${TOPPORT} + 1 )) # # tests use siege client without https support. # so here we create http to https proxy. The proxy # is created for no-reuse tests only. # if [[ ${REUSE_LABEL} = 'no-ssl-reuse' ]] ; then - PORT=$(( ${TOPPORT} + 1 )) HTTP_PORT=$(( ${PORT} + 1)) emit_http_port ${HAPROXY_CONF} ${HTTP_PORT} ${PORT} fi @@ -380,21 +398,21 @@ function setup_tests { clean_build cd "${WORKSPACE_ROOT}" - install_wolfssl 5.8.2 '--enable-haproxy --enable-quic' - install_haproxy wolfssl-5.8.2 - install_httpterm wolfssl-5.8.2 - install_h1load wolfssl-5.8.2 - install_siege wolfssl-5.8.2 - config_haproxy wolfssl-5.8.2 + install_wolfssl ${HAPROXY_WOLF_VERSION} '--enable-haproxy --enable-quic' + install_haproxy wolfssl-${HAPROXY_WOLF_VERSION} + install_httpterm wolfssl-${HAPROXY_WOLF_VERSION} + install_h1load wolfssl-${HAPROXY_WOLF_VERSION} + install_siege wolfssl-${HAPROXY_WOLF_VERSION} + config_haproxy wolfssl-${HAPROXY_WOLF_VERSION} clean_build cd "${WORKSPACE_ROOT}" - install_libressl 4.1.0 - install_haproxy libressl-4.1.0 - install_httpterm libressl-4.1.0 - install_h1load libressl-4.1.0 - install_siege libressl-4.1.0 - config_haproxy libressl-4.1.0 + install_libressl ${HAPROXY_LIBRE_VERSION} + install_haproxy libressl-${HAPROXY_LIBRE_VERSION} + install_httpterm libressl-${HAPROXY_LIBRE_VERSION} + install_h1load libressl-${HAPROXY_LIBRE_VERSION} + install_siege libressl-${HAPROXY_LIBRE_VERSION} + config_haproxy libressl-${HAPROXY_LIBRE_VERSION} clean_build # diff --git a/bench-scripts/bench_run_haproxy.sh b/bench-scripts/bench_run_haproxy.sh index c400d77..4122fdc 100755 --- a/bench-scripts/bench_run_haproxy.sh +++ b/bench-scripts/bench_run_haproxy.sh @@ -23,7 +23,7 @@ CERT_SUBJ=${BENCH_CERT_SUBJ:-'/CN=localhost'} CERT_ALT_SUBJ=${BENCH_CERT_ALT_SUBJ:-'subjectAltName=DNS:localhost,IP:127.0.0.1'} HTTPTERM_HOST=${BENCH_HTTPTERM_HOST:-${HOST}} HTTPTERM_PORT=${BENCH_HTTPTERM_PORT:-9999} -PROXY_CHAIN=${BENCH_PROXY_CHAIN:-21} +PROXY_CHAIN=${BENCH_PROXY_CHAIN:-10} HAPROXY_VERSION='v3.2.0' TEST_TIME=${BENCH_TEST_TIME:-'10'} @@ -42,7 +42,7 @@ function run_haproxy { LD_LIBRARY_PATH="${OPENSSL_DIR}/lib" "${HAPROXY}" \ -f "${OPENSSL_DIR}/etc/haproxy.conf" \ - -p ${HAPPIDFILE} \ + -p ${HAPPIDFILE} \ -D if [[ $? -ne 0 ]] ; then echo "could not start haproxy" @@ -60,7 +60,7 @@ function run_httpterm { typeset HTTPTERM="${OPENSSL_DIR}"/bin/httpterm LD_LIBRARY_PATH="${OPENSSL_DIR}/lib" "${HTTPTERM}" \ - -p ${HTTPTERMPIDFILE} \ + -p ${HTTPTERMPIDFILE} \ -L ${HTTPTERM_HOST}:${HTTPTERM_PORT} \ -D if [[ $? -ne 0 ]] ; then @@ -99,7 +99,7 @@ function run_test { run_haproxy ${SSL_LIB} ${HAPPIDFILE} run_httpterm ${SSL_LIB} ${HTTPTERMPIDFILE} - echo "proxy running for ${SSL_LIB} ${THREAD_COUNT}" + echo "HA-proxy running with ${SSL_LIB}, h1load uses ${THREAD_COUNT} threads" for TEST_NAME in dh-rsa-reuse dh-rsa-noreuse ec-dsa-reuse ec-dsa-noreuse ; do case ${TEST_NAME} in @@ -121,11 +121,18 @@ function run_test { ;; esac RESULT=${RESULT_DIR}/h1load-${TEST_NAME}-${THREAD_COUNT}-${SSL_LIB}.out + # + # -c number of concurrent connections, must be divisible by thread count + # -n number of requests, should X * number of connections, here X == 5 + # -r number of requests per connection + # -P add percentile data to result (those are not plotted yet) + # -t number of threads + # LD_LIBRARY_PATH=${OPENSSL_DIR}/lib ${H1LOAD} \ - -l \ - -P \ - -d ${TEST_TIME} \ - -c 500 \ + -c 1280 \ + -n 40000 \ + -r 1 \ + -P \ -t ${THREAD_COUNT} \ ${BASE_URL}${PORT} > ${RESULT} || exit 1 done @@ -168,8 +175,8 @@ function run_tests { run_test openssl-${i} ${t} done run_test OpenSSL_1_1_1-stable ${t} - run_test libressl-4.1.0 ${t} - run_test wolfssl-5.8.2 ${t} + run_test libressl-${HAPROXY_LIBRE_VERSION} ${t} + run_test wolfssl-${HAPROXY_WOLF_VERSION} ${t} run_test aws-lc ${t} # # could not get haproxy working with boringssl @@ -177,27 +184,114 @@ function run_tests { done } +# +# function uses gnuplot(1) to generate .png +# with plot of performance data we got from +# siege. It currently plots data for: +# openssl-master, ..., openssl-3.0, openssl-1.1.1, +# libressl, wolfssl, aws-lc +# +function plot_siege { + typeset DATA_FILE=${1} + typeset OUT_FILE=${2} + typeset TITLE=${3} + typeset YLABEL=${4} + + gnuplot << EOF +set title "${TITLE}" +set grid lt 0 lw 1 ls 1 lc rgb "#d7d7d7" +set xlabel "Number of threads" +set ylabel "${YLABEL}" +set terminal pngcairo size 800,400 background rgb "#f8f8f8" +set output "${OUT_FILE}" +set key autotitle columnhead outside +set auto x +set style data histogram +set style histogram cluster gap 1 +set style fill solid border -1 +set boxwidth 0.9 +plot \ + "${DATA_FILE}" using 3:xticlabels(2) ti col, \ + "${DATA_FILE}" using 4 ti col, \ + "${DATA_FILE}" using 5 ti col, \ + "${DATA_FILE}" using 6 ti col, \ + "${DATA_FILE}" using 7 ti col, \ + "${DATA_FILE}" using 8 ti col, \ + "${DATA_FILE}" using 9 ti col, \ + "${DATA_FILE}" using 10 ti col, \ + "${DATA_FILE}" using 11 ti col, \ + "${DATA_FILE}" using 12 ti col, \ + "${DATA_FILE}" using 13 ti col, \ + "${DATA_FILE}" using 14 ti col, \ + "${DATA_FILE}" using 15 ti col +EOF +} + +# +# function uses gnuplot(1) to generate .png +# with plot of performance data we got from +# h1load. It currently plots data for: +# openssl-master, ..., openssl-3.0, openssl-1.1.1, +# libressl, wolfssl, aws-lc +# +function plot_h1load { + typeset DATA_FILE=${1} + typeset OUT_FILE=${2} + typeset TITLE=${3} + typeset YLABEL='secs to complete h1load' + + gnuplot << EOF +set title "${TITLE}" +set grid lt 0 lw 1 ls 1 lc rgb "#d7d7d7" +set xlabel "Number of threads" +set ylabel "${YLABEL}" +set terminal pngcairo size 800,400 background rgb "#f8f8f8" +set output "${OUT_FILE}" +set key autotitle columnhead outside +set auto x +set style data histogram +set style histogram cluster gap 1 +set style fill solid border -1 +set boxwidth 0.9 +plot \ + "${DATA_FILE}" using 2:xticlabels(1) ti col, \ + "${DATA_FILE}" using 3 ti col, \ + "${DATA_FILE}" using 4 ti col, \ + "${DATA_FILE}" using 5 ti col, \ + "${DATA_FILE}" using 6 ti col, \ + "${DATA_FILE}" using 7 ti col, \ + "${DATA_FILE}" using 8 ti col, \ + "${DATA_FILE}" using 9 ti col, \ + "${DATA_FILE}" using 10 ti col, \ + "${DATA_FILE}" using 11 ti col, \ + "${DATA_FILE}" using 12 ti col, \ + "${DATA_FILE}" using 13 ti col, \ + "${DATA_FILE}" using 14 ti col +EOF +} + # # function merges siege tests to tables so results -# can be compared plotted. The tests collect data -# to files. Each file contains a combination of: +# can be plotted. The tests collect data to files. +# Each file contains a combination of: # - ha-proxy configuration # - ssl library # - number of processes used # the list of files looks then as follows: -# h1load-dh-rsa-noreuse-1-openssl-3.4.out +# siege-dh-rsa-noreuse-1-openssl-3.4.out # ... -# h1load-dh-rsa-noreuse-2-wolfssl-5.8.2.out +# siege-dh-rsa-noreuse-2-wolfssl-5.8.2.out # ... -# h1load-dh-rsa-noreuse-64-openssl-master.out +# siege-dh-rsa-noreuse-64-openssl-master.out # -# the h1load-dh-rsa-noreuse- identifies ha-proxy configuration -# used for testing. +# the siege-dh-rsa-noreuse- identifies ha-proxy configuration +# and test tool used to obtain data. In this case data +# come from siege using RSA key exchange during SSL handshake. # # the next -1, -3, ..., -64 infix represents number of threads/cpus # used for test. # -# openss-3.4, weolfssl-5.8.2, ... is the ssl library used +# openssl-3.4, weolfssl-5.8.2, ... is the ssl library used # for testing # # this function merges collected data to tables so we can @@ -218,6 +312,7 @@ function merge_siege { typeset INPUT_FILE='' typeset OUTPUT_FILE='' typeset SAVE_IFS='' + typeset LINE=1 for HANDSHAKE in siege-dh-rsa-noreuse siege-ec-dsa-noreuse ; do SAVE_IFS=${IFS} @@ -233,15 +328,18 @@ function merge_siege { # # print header with column labels # + printf "line-no.\tThreads" >> ${OUTPUT_FILE} for SSL_LIB in `ssl_libs_haproxy` ; do printf "\t${SSL_LIB}" >> ${OUTPUT_FILE} done printf '\n' >> ${OUTPUT_FILE} + LINE=1 for PROCS in `procs` ; do # # row header with number CPUs used for test # - printf "${PROCS}" >> ${OUTPUT_FILE} + printf "${LINE}\t${PROCS}" >> ${OUTPUT_FILE} + LINE=$(( ${LINE} + 1)) for SSL_LIB in `ssl_libs_haproxy` ; do INPUT_FILE=${HANDSHAKE}-${PROCS}-${SSL_LIB}.out INPUT_FILE=${RESULT_DIR}/${INPUT_FILE} @@ -272,5 +370,149 @@ function merge_siege { done } +# +# function merges h1load tests to tables so results +# can be plotted. The tests collect data +# to files. Each file contains a combination of: +# - ha-proxy configuration +# - ssl library +# - number of processes used +# the list of files looks then as follows: +# h1load-dh-rsa-noreuse-1-openssl-3.4.out +# ... +# h1load-dh-rsa-noreuse-2-wolfssl-5.8.2.out +# ... +# h1load-dh-rsa-noreuse-64-openssl-master.out +# +# the h1load-dh-rsa-noreuse- identifies ha-proxy configuration +# and tool used for testing. +# +# the next -1, -3, ..., -64 infix represents number of threads/cpus +# used for test. +# +# openss-3.4, weolfssl-5.8.2, ... is the ssl library used +# for testing +# +# the merge process uses a test duration to compose the results +# into single table to plot. The idea is the longer the test run +# takes the worse performance. +# +function merge_h1load { + typeset RESULT_DIR=${1:-'.'} + typeset HANDSHAKE='' + typeset PROCS='' + typeset SSL_LIB='' + typeset DURATION='' + typeset INPUT_FILE='' + typeset OUTPUT_FILE='' + + for HANDSHAKE in h1load-dh-rsa-noreuse h1load-dh-rsa-reuse h1load-ec-dsa-noreuse h1load-ec-dsa-reuse ; do + OUTPUT_FILE=${RESULT_DIR}/${HANDSHAKE}.merged + printf "Threads" > ${OUTPUT_FILE} + for SSL_LIB in `ssl_libs_haproxy` ; do + printf "\t${SSL_LIB}" >> ${OUTPUT_FILE} + done + printf "\n" >> ${OUTPUT_FILE} + for PROCS in `procs` ; do + printf "${PROCS}" >> ${OUTPUT_FILE} + for SSL_LIB in `ssl_libs_haproxy` ; do + INPUT_FILE=${RESULT_DIR}/${HANDSHAKE}-${PROCS}-${SSL_LIB}.out + # + # h1load outputs performance data combined with percentile table. Those + # parts are delimited by ^#= delimiter. The sed expression chops off + # the first part (performance table). + # tail -1 then grabs the last line where we find the test duration + # in secs. The test duration is found in the first column we read + # using awk + # + DURATION=$(sed -ne '/^#=/q;p' "${INPUT_FILE}" |tail -1 |awk '{ printf($1); }') + printf "\t${DURATION}" >> ${OUTPUT_FILE} + one + # + # new line + # + printf "\n" >> ${OUTPUT_FILE} + done + done +} + +# +# siege charts to plot: +# Trnsaction Rate (in trans/sec) +# +function create_siege_plots { + typeset RESULTS=${1} + typeset HANDSHAKE='' + typeset DATA_FILE='' + typeset OUT_FILE='' + + for HANDSHAKE in siege-dh-rsa-noreuse siege-ec-dsa-noreuse ; do + DATA_FILE=${RESULTS}/${HANDSHAKE}-Transactions.merged + OUT_FILE=${RESULTS}/${HANDSHAKE}-Transactions.png + plot_siege ${DATA_FILE} ${OUT_FILE} \ + "Number of transactions in ${TEST_TIME} secs (${HANDSHAKE})" \ + "Transactions" + + DATA_FILE=${RESULTS}/${HANDSHAKE}-Data_transferred.merged + OUT_FILE=${RESULTS}/${HANDSHAKE}-Data_transferred.png + plot_siege ${DATA_FILE} ${OUT_FILE} \ + "Bytes transferred in ${TEST_TIME} secs (${HANDSHAKE})" \ + "Data Transfer [MB]" + + DATA_FILE=${RESULTS}/${HANDSHAKE}-Longest_transaction.merged + OUT_FILE=${RESULTS}/${HANDSHAKE}-Longest_transaction.png + plot_siege ${DATA_FILE} ${OUT_FILE} \ + "Longest transaction (${HANDSHAKE})" \ + "Duration [ms]" + + DATA_FILE=${RESULTS}/${HANDSHAKE}-Shortest_transaction.merged + OUT_FILE=${RESULTS}/${HANDSHAKE}-Shortest_trnsaction.png + plot_siege ${DATA_FILE} ${OUT_FILE} \ + "Shortest transaction (${HANDSHAKE})" \ + "Duration [ms]" + + DATA_FILE=${RESULTS}/${HANDSHAKE}-Response_time.merged + OUT_FILE=${RESULTS}/${HANDSHAKE}-Response_time.png + plot_siege ${DATA_FILE} ${OUT_FILE} \ + "Average response time (${HANDSHAKE})" \ + "time [ms]" + + DATA_FILE=${RESULTS}/${HANDSHAKE}-Throughput.merged + OUT_FILE=${RESULTS}/${HANDSHAKE}-Throughput.png + plot_siege ${DATA_FILE} ${OUT_FILE} \ + "Throughput (${HANDSHAKE})" \ + "MB/sec" + + DATA_FILE=${RESULTS}/${HANDSHAKE}-Throughput.merged + OUT_FILE=${RESULTS}/${HANDSHAKE}-Throughput.png + plot_siege ${DATA_FILE} ${OUT_FILE} \ + "Throughput (${HANDSHAKE})" \ + "MB/sec" + + DATA_FILE=${RESULTS}/${HANDSHAKE}-Transaction_rate.merged + OUT_FILE=${RESULTS}/${HANDSHAKE}-Transaction_rate.png + plot_siege ${DATA_FILE} ${OUT_FILE} \ + "Transaction rate (${HANDSHAKE})" \ + "trans/sec" + done +} + +function create_h1load_plots { + typeset RESULTS=${1} + typeset HANDSHAKE='' + typeset DATA_FILE='' + typeset OUT_FILE='' + + for HANDSHAKE in dh-rsa-noreuse dh-rsa-reuse ec-dsa-noreuse ec-dsa-reuse ; do + DATA_FILE=${RESULTS}/h1load-${HANDSHAKE}.merged + OUT_FILE=${RESULTS}/h1load-${HANDSHAKE}.png + plot_h1load ${DATA_FILE} ${OUT_FILE} \ + "Time in seconds to complete the\n${HANDSHAKE} test" + done +} + run_tests merge_siege ${RESULT_DIR} +merge_h1load ${RESULT_DIR} +create_siege_plots ${RESULT_DIR} +create_h1load_plots ${RESULT_DIR} diff --git a/bench-scripts/common_util.sh b/bench-scripts/common_util.sh index 45d899c..ab79d82 100644 --- a/bench-scripts/common_util.sh +++ b/bench-scripts/common_util.sh @@ -399,13 +399,16 @@ function gen_certkey_ec { # # openssl-master openssl-3.0 .... libressl-4.1.0 wolfssl-5.8.2 aws-lc # +HAPROXY_LIBRE_VERSION='4.2.1' +HAPROXY_WOLF_VERSION='5.8.4' function ssl_libs_haproxy { echo -n 'openssl-master' for i in 3.0 3.1 3.2 3.3 3.4 3.5 3.6 ; do echo -n " openssl-$i" done - echo -n ' libressl-4.1.0' - echo -n ' wolfssl-5.8.2' + echo -n ' OpenSSL_1_1_1-stable' + echo -n " libressl-${HAPROXY_LIBRE_VERSION}" + echo -n " wolfssl-${HAPROXY_WOLF_VERSION}" echo ' aws-lc' }