From 0063d98516e6f950f63e65a84bfd7321ca72fb09 Mon Sep 17 00:00:00 2001 From: Norbert Pocs Date: Mon, 29 Sep 2025 12:09:46 +0200 Subject: [PATCH 1/3] Add haproxy to benchmarking tool Adding as a proxy - a middle actor. Currently adding only for apache. Support all the encryption modes for haproxy: - client to proxy - proxy to server - client to server - pure http without encryption Signed-off-by: Norbert Pocs --- bench-scripts/apache_bench.sh | 98 ++++++++++++++++-- bench-scripts/common_util.sh | 1 + bench-scripts/haproxy_bench.sh | 177 +++++++++++++++++++++++++++++++++ 3 files changed, 268 insertions(+), 8 deletions(-) create mode 100755 bench-scripts/haproxy_bench.sh diff --git a/bench-scripts/apache_bench.sh b/bench-scripts/apache_bench.sh index c98d1a2a..f910904f 100755 --- a/bench-scripts/apache_bench.sh +++ b/bench-scripts/apache_bench.sh @@ -63,8 +63,10 @@ CERT_ALT_SUBJ=${BENCH_CERT_ALT_SUBJ:-'subjectAltName=DNS:localhost,IP:127.0.0.1' TEST_TIME=${BENCH_TEST_TIME:-'5M'} HOST=${BENCH_HOST:-'127.0.0.1'} APACHE_VERSION='2.4.65' +HAPROXY='no' . ./common_util.sh +. ./haproxy_bench.sh function install_wolfssl_for_apache { typeset VERSION=$1 @@ -810,18 +812,25 @@ function enable_mpm_prefork { function run_test { typeset SSL_LIB=$1 - typeset HTTP='https' + typeset HAPROXY=$2 typeset i=0 typeset PORT=${HTTPS_PORT} + typeset PROTOCOL="https" if [[ -z "${SSL_LIB}" ]] ; then SSL_LIB='openssl-master' fi + if [[ -z "${HAPROXY}" ]] ; then + HAPROXY='no' + fi typeset RESULTS="${SSL_LIB}".txt if [[ "${SSL_LIB}" = 'nossl' ]] ; then - HTTP='http' SSL_LIB='openssl-master' RESULTS='nossl.txt' PORT=${HTTP_PORT} + PROTOCOL="http" + fi + if [[ "${HAPROXY}" != 'no' ]] ; then + RESULTS="haproxy-${SSL_LIB}-${HAPROXY}.txt" fi typeset HTDOCS="${INSTALL_ROOT}/${SSL_LIB}"/htdocs typeset SIEGE="${INSTALL_ROOT}"/openssl-master/bin/siege @@ -839,11 +848,35 @@ function run_test { # # generate URLs for sewage # + # The different modes for haproxy are: + # no: client - server + # no-ssl: client -http- haproxy -http- server + # server: client -https- haproxy -http- server + # client: client -http- haproxy -https- server + # both: client -https- haproxy -https- server + # + # Otherwise said, haproxy is a client when it encrypts the outgoing encryption; + # or it's client side. + # rm -f siege_urls.txt for i in `ls -1 ${HTDOCS}/*.txt` ; do - echo "${HTTP}://${HOST}:${PORT}/`basename $i`" >> siege_urls.txt + if [[ "${HAPROXY}" = "no" ]] ; then + echo "${PROTOCOL}://${HOST}:${PORT}/`basename $i`" >> siege_urls.txt + elif [[ "${HAPROXY}" = "no-ssl" ]] ; then + echo "http://${HOST}:${HAPROXY_NOSSL_PORT}/`basename $i`" >> siege_urls.txt + elif [[ "${HAPROXY}" = "server" ]] ; then + echo "https://${HOST}:${HAPROXY_C2P_PORT}/`basename $i`" >> siege_urls.txt + elif [[ "${HAPROXY}" = "client" ]] ; then + echo "http://${HOST}:${HAPROXY_P2S_PORT}/`basename $i`" >> siege_urls.txt + elif [[ "${HAPROXY}" = "both" ]] ; then + echo "https://${HOST}:${HAPROXY_C2S_PORT}/`basename $i`" >> siege_urls.txt + fi done + if [[ "${HAPROXY}" = "server" ]] || [[ "${HAPROXY}" = "both" ]] ; then + conf_siege_haproxy_cert $SSL_LIB + fi + # # start apache httpd server # @@ -856,13 +889,14 @@ function run_test { LD_LIBRARY_PATH=${INSTALL_ROOT}/openssl-master/lib "${SIEGE}" -t ${TEST_TIME} -b \ -f siege_urls.txt 2> "${RESULT_DIR}/${RESULTS}" if [[ $? -ne 0 ]] ; then - echo "${INSTALL_ROOT}/${SSL_LIB} can not run siege" + echo "${INSTALL_ROOT}/${SSL_LIB} can not run siege" cat "${RESULT_DIR}/${RESULTS}" exit 1 fi LD_LIBRARY_PATH=${INSTALL_ROOT}/${SSL_LIB}/lib \ ${INSTALL_ROOT}/${SSL_LIB}/bin/httpd -k stop || exit 1 + sleep 1 pgrep httpd while [[ $? -eq 0 ]] ; do sleep 1 @@ -881,6 +915,10 @@ function run_test { ${RESULT_DIR}/httpd-${SSL_LIB}.conf cp ${INSTALL_ROOT}/${SSL_LIB}/conf/extra/httpd-ssl.conf \ ${RESULT_DIR}/httpd-ssl-${SSL_LIB}.conf + + if [[ "${HAPROXY}" = "server" ]] || [[ "${HAPROXY}" = "both" ]] ; then + unconf_siege_haproxy_cert + fi } function setup_tests { @@ -888,6 +926,7 @@ function setup_tests { install_siege openssl-master install_apache openssl-master config_apache openssl-master + install_haproxy openssl-master cd "${WORKSPACE_ROOT}" clean_build @@ -895,6 +934,7 @@ function setup_tests { install_openssl openssl-$i ; install_apache openssl-$i config_apache openssl-$i + install_haproxy openssl-$i cd "${WORKSPACE_ROOT}" clean_build done @@ -967,6 +1007,7 @@ function setup_tests { function run_tests { typeset SAVE_RESULT_DIR="${RESULT_DIR}" + typeset HAPROXY_OPTIONS=('no' 'client' 'server' 'both') for i in event worker pre-fork ; do mkdir -p ${RESULT_DIR}/$i || exit 1 @@ -975,12 +1016,25 @@ function run_tests { enable_mpm_event RESULT_DIR="${SAVE_RESULT_DIR}/event" run_test nossl + run_haproxy + run_test nossl 'no-ssl' + kill_haproxy for i in 3.0 3.1 3.2 3.3 3.4 3.5 3.6 ; do enable_mpm_event openssl-${i} - run_test openssl-${i} + run_haproxy openssl-${i} + for OPTION in ${HAPROXY_OPTIONS[@]} + do + run_test openssl-${i} ${OPTION} + done + kill_haproxy done enable_mpm_event openssl-master - run_test openssl-master + run_haproxy openssl-master + for OPTION in ${HAPROXY_OPTIONS[@]} + do + run_test openssl-master ${OPTION} + done + kill_haproxy enable_mpm_event OpenSSL_1_1_1-stable run_test OpenSSL_1_1_1-stable enable_mpm_event libressl-4.1.0 @@ -995,12 +1049,26 @@ function run_tests { enable_mpm_worker RESULT_DIR="${SAVE_RESULT_DIR}/worker" run_test nossl + run_haproxy + run_test nossl 'no-ssl' + kill_haproxy for i in 3.0 3.1 3.2 3.3 3.4 3.5 3.6 ; do enable_mpm_worker openssl-${i} run_test openssl-${i} + run_haproxy openssl-${i} + for OPTION in ${HAPROXY_OPTIONS[@]} + do + run_test openssl-${i} ${OPTION} + done + kill_haproxy done enable_mpm_worker openssl-master - run_test openssl-master + run_haproxy openssl-master + for OPTION in ${HAPROXY_OPTIONS[@]} + do + run_test openssl-master ${OPTION} + done + kill_haproxy enable_mpm_worker OpenSSL_1_1_1-stable run_test OpenSSL_1_1_1-stable enable_mpm_worker libressl-4.1.0 @@ -1015,12 +1083,26 @@ function run_tests { enable_mpm_prefork RESULT_DIR="${SAVE_RESULT_DIR}/pre-fork" run_test nossl + run_haproxy + run_test nossl 'no-ssl' + kill_haproxy for i in 3.0 3.1 3.2 3.3 3.4 3.5 3.6 ; do enable_mpm_prefork openssl-${i} run_test openssl-${i} + run_haproxy openssl-${i} + for OPTION in ${HAPROXY_OPTIONS[@]} + do + run_test openssl-${i} ${OPTION} + done + kill_haproxy done enable_mpm_prefork openssl-master - run_test openssl-master + run_haproxy openssl-master + for OPTION in ${HAPROXY_OPTIONS[@]} + do + run_test openssl-master ${OPTION} + done + kill_haproxy enable_mpm_prefork OpenSSL_1_1_1-stable run_test OpenSSL_1_1_1-stable enable_mpm_prefork libressl-4.1.0 diff --git a/bench-scripts/common_util.sh b/bench-scripts/common_util.sh index 21f2230d..f06a4e17 100644 --- a/bench-scripts/common_util.sh +++ b/bench-scripts/common_util.sh @@ -292,6 +292,7 @@ set term png size 1600, 600 set boxwidth 0.4 set autoscale set output "${PNG_FILE}" +set xtics rotate by 90 right plot "${DATA_FILE}" using 1:3:xtic(2) with boxes EOF gnuplot ${PLOT_FILE} diff --git a/bench-scripts/haproxy_bench.sh b/bench-scripts/haproxy_bench.sh new file mode 100755 index 00000000..8e965b82 --- /dev/null +++ b/bench-scripts/haproxy_bench.sh @@ -0,0 +1,177 @@ +#!/usr/bin/env ksh +# +# Copyright 2025 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# + +set -x + +INSTALL_ROOT=${BENCH_INSTALL_ROOT:-"/tmp/bench.binaries"} +RESULT_DIR=${BENCH_RESULTS:-"${INSTALL_ROOT}/results"} +WORKSPACE_ROOT=${BENCH_WORKSPACE_ROOT:-"/tmp/bench.workspace"} +MAKE_OPTS=${BENCH_MAKE_OPTS} +HAPROXY_NOSSL_PORT='42128' +HAPROXY_C2P_PORT='42132' +HAPROXY_P2S_PORT='42134' +HAPROXY_C2S_PORT='42136' +CERT_SUBJ=${BENCH_CERT_SUBJ:-'/CN=localhost'} +CERT_ALT_SUBJ=${BENCH_CERT_ALT_SUBJ:-'subjectAltName=DNS:localhost,IP:127.0.0.1'} +HOST=${BENCH_HOST:-'127.0.0.1'} +HAPROXY_VERSION='v3.2.0' + +function install_haproxy { + typeset SSL_LIB=$1 + typeset VERSION=${HAPROXY_VERSION:-v3.2.0} + typeset HAPROXY_REPO="https://github.com/haproxy/haproxy.git" + typeset BASENAME='haproxy' + typeset DIRNAME="${BASENAME}-${VERSION}" + typeset CERTDIR="${INSTALL_ROOT}/${SSL_LIB}/conf/certs" + + if [[ -z "${SSL_LIB}" ]] ; then + SSL_LIB="openssl-master" + fi + + if [[ -f "${INSTALL_ROOT}/${SSL_LIB}/sbin/haproxy" ]] ; then + echo "haproxy already installed; skipping.." + else + cd "${WORKSPACE_ROOT}" + mkdir -p "${DIRNAME}" || exit 1 + cd "${DIRNAME}" + git clone "${HAPROXY_REPO}" -b ${VERSION} --depth 1 . || exit 1 + + # haproxy does not have a configure script; only a big makefile + make ${MAKE_OPTS} \ + TARGET=generic \ + USE_OPENSSL=1 \ + SSL_INC="${INSTALL_ROOT}/${SSL_LIB}/include" \ + SSL_LIB="${INSTALL_ROOT}/${SSL_LIB}/lib" || exit 1 + + make install ${MAKE_OPTS} \ + PREFIX="${INSTALL_ROOT}/${SSL_LIB}" || exit 1 + fi + + mkdir -p ${CERTDIR} + + # now generate the certificates + echo "generating new certificates for haproxy" + OPENSSL_BIN="env LD_LIBRARY_PATH=${INSTALL_ROOT}/${SSL_LIB}/lib ${INSTALL_ROOT}/${SSL_LIB}/bin/openssl" + + # generating the key, cert of ca + $OPENSSL_BIN genpkey -algorithm rsa -pkeyopt rsa_keygen_bits:2048 -out "${CERTDIR}/ca_key.pem" || exit 1 + $OPENSSL_BIN req -new -x509 -days 1 -key "${CERTDIR}/ca_key.pem" -out "${CERTDIR}/ca_cert.pem" -subj "/CN=Root CA" \ + -addext "basicConstraints=critical,CA:true" \ + -addext "keyUsage=critical,keyCertSign,cRLSign" || exit 1 + + # generating the client side + $OPENSSL_BIN genpkey -algorithm rsa -pkeyopt rsa_keygen_bits:2048 -out "${CERTDIR}/client_key.pem" || exit 1 + $OPENSSL_BIN pkey -in "${CERTDIR}/client_key.pem" -pubout -out "${CERTDIR}/client_key_pub.pem" || exit 1 + $OPENSSL_BIN req -new -out "${CERTDIR}/client_csr.pem" -subj "/CN=${HOST}" -key "${CERTDIR}/client_key.pem" \ + -addext "${CERT_ALT_SUBJ}" \ + -addext "keyUsage=critical,digitalSignature" || exit 1 + $OPENSSL_BIN x509 -req -out "${CERTDIR}/client_cert.pem" -CAkey "${CERTDIR}/ca_key.pem" -CA "${CERTDIR}/ca_cert.pem" \ + -days 1 -in "${CERTDIR}/client_csr.pem" -copy_extensions copy -ext "subjectAltName,keyUsage" \ + -extfile <(printf "basicConstraints=critical,CA:false\nsubjectKeyIdentifier=none\n") || exit 1 + + # generating the server side + $OPENSSL_BIN genpkey -algorithm rsa -pkeyopt rsa_keygen_bits:2048 -out "${CERTDIR}/server_key.pem" || exit 1 + $OPENSSL_BIN pkey -in "${CERTDIR}/server_key.pem" -pubout -out "${CERTDIR}/server_key_pub.pem" || exit 1 + $OPENSSL_BIN req -new -out "${CERTDIR}/server_csr.pem" -subj "/CN=${HOST}" -key "${CERTDIR}/server_key.pem" \ + -addext "${CERT_ALT_SUBJ}" \ + -addext "keyUsage=critical,digitalSignature" || exit 1 + $OPENSSL_BIN x509 -req -out "${CERTDIR}/server_cert.pem" -CAkey "${CERTDIR}/ca_key.pem" -CA "${CERTDIR}/ca_cert.pem" \ + -days 1 -in "${CERTDIR}/server_csr.pem" -copy_extensions copy -ext "subjectAltName,keyUsage" \ + -extfile <(printf "subjectKeyIdentifier=none\n" + printf "${CERT_ALT_SUBJ}\n" + printf "basicConstraints=critical,CA:false\n" + printf "keyUsage=critical,keyEncipherment\n") || exit 1 + + # HAProxy PEM must be: server cert + server key (+ chain) + cat "${CERTDIR}/server_cert.pem" "${CERTDIR}/server_key.pem" "${CERTDIR}/ca_cert.pem" > "${CERTDIR}/haproxy_server.pem" + + # setting up SSL Termination mode for now + # haproxy modes: encoding from client to haproxy, to server from haproxy, both + # the first needs a non TLS connection to the server - use the HTTP_PORT, otherwise use the HTTPS_PORT + cat < "${INSTALL_ROOT}/${SSL_LIB}/conf/haproxy.cfg" +defaults + timeout server 10s + timeout client 10s + timeout connect 10s + +frontend test_no_ssl + mode http + bind :${HAPROXY_NOSSL_PORT} + default_backend http_test + +frontend test_client2proxy + mode http + bind :${HAPROXY_C2P_PORT} ssl crt ${CERTDIR}/haproxy_server.pem ca-file ${CERTDIR}/ca_cert.pem verify required + default_backend http_test + +frontend test_proxy2server + mode http + bind :${HAPROXY_P2S_PORT} + default_backend https_test + +frontend test_client2server + mode http + bind :${HAPROXY_C2S_PORT} ssl crt ${CERTDIR}/haproxy_server.pem ca-file ${CERTDIR}/ca_cert.pem verify required + default_backend https_test + +backend http_test + mode http + balance random + server s1 ${HOST}:${HTTP_PORT} + +backend https_test + mode http + balance random + server s2 ${HOST}:${HTTPS_PORT} ssl verify required ca-file ${INSTALL_ROOT}/${SSL_LIB}/conf/server.crt +EOF +} + +function run_haproxy { + typeset SSL_LIB=$1 + if [[ -z "${SSL_LIB}" ]] ; then + SSL_LIB="openssl-master" + fi + typeset OPENSSL_DIR="${INSTALL_ROOT}/${SSL_LIB}" + + LD_LIBRARY_PATH="${OPENSSL_DIR}/lib:${LD_LIBRARY_PATH}" "${OPENSSL_DIR}/sbin/haproxy" -f "${OPENSSL_DIR}/conf/haproxy.cfg" -D + if [[ $? -ne 0 ]] ; then + echo "could not start haproxy" + exit 1 + fi +} + +function conf_siege_haproxy_cert { + typeset SSL_LIB=$1 + if [[ -z "${SSL_LIB}" ]] ; then + SSL_LIB="openssl-master" + fi + typeset OPENSSL_DIR="${INSTALL_ROOT}/${SSL_LIB}" + # siege is currently installed only with openssl-master + typeset SIEGE_CONF="${INSTALL_ROOT}/openssl-master/etc/siegerc" + # configure siege to use haproxy + if [[ ! -f "${SIEGE_CONF}" ]] ; then + echo "Did not found siegerc. Siege should be installed first." + exit 1 + fi + echo "#haproxy" >> "${SIEGE_CONF}" + echo "ssl-cert = ${OPENSSL_DIR}/conf/certs/client_cert.pem" >> "${SIEGE_CONF}" + echo "ssl-key = ${OPENSSL_DIR}/conf/certs/client_key.pem" >> "${SIEGE_CONF}" +} + +function unconf_siege_haproxy_cert { + typeset SIEGE_CONF="${INSTALL_ROOT}/openssl-master/etc/siegerc" + + # clear the siege config + sed -i '/#haproxy/{N;d;}' "${SIEGE_CONF}" || exit 1 +} + +function kill_haproxy { + pkill -TERM -f haproxy +} From 7f2f043953612ec583c7f75b644e1b61561e67c5 Mon Sep 17 00:00:00 2001 From: sashan Date: Tue, 21 Oct 2025 01:26:12 +0200 Subject: [PATCH 2/3] - need to add typeset for i (forloop counter), there is a risk the global state gets polluted. missing typeset in generate_download_files() prevents haproxy to build --- bench-scripts/apache_bench.sh | 2 ++ bench-scripts/common_util.sh | 2 ++ bench-scripts/haproxy_bench.sh | 9 +++++---- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/bench-scripts/apache_bench.sh b/bench-scripts/apache_bench.sh index f910904f..655f0c25 100755 --- a/bench-scripts/apache_bench.sh +++ b/bench-scripts/apache_bench.sh @@ -922,6 +922,7 @@ function run_test { } function setup_tests { + typeset i=0 install_openssl master install_siege openssl-master install_apache openssl-master @@ -1008,6 +1009,7 @@ function setup_tests { function run_tests { typeset SAVE_RESULT_DIR="${RESULT_DIR}" typeset HAPROXY_OPTIONS=('no' 'client' 'server' 'both') + typeset i="" for i in event worker pre-fork ; do mkdir -p ${RESULT_DIR}/$i || exit 1 diff --git a/bench-scripts/common_util.sh b/bench-scripts/common_util.sh index f06a4e17..039e7c4c 100644 --- a/bench-scripts/common_util.sh +++ b/bench-scripts/common_util.sh @@ -92,6 +92,7 @@ function cleanup { function clean_build { typeset SAVE_DIR=`pwd` + typeset i="" cd "${WORKSPACE_ROOT}" for i in * ; do if [[ -d $i ]] ; then @@ -309,6 +310,7 @@ function plot_results { function generate_download_files { typeset HTDOCS=$1 + typeset i="" mkdir -p ${HTDOCS} || exit 1 diff --git a/bench-scripts/haproxy_bench.sh b/bench-scripts/haproxy_bench.sh index 8e965b82..a5cbb708 100755 --- a/bench-scripts/haproxy_bench.sh +++ b/bench-scripts/haproxy_bench.sh @@ -24,16 +24,16 @@ HOST=${BENCH_HOST:-'127.0.0.1'} HAPROXY_VERSION='v3.2.0' function install_haproxy { - typeset SSL_LIB=$1 - typeset VERSION=${HAPROXY_VERSION:-v3.2.0} + typeset SSL_LIB=$1 + typeset VERSION=${HAPROXY_VERSION:-v3.2.0} typeset HAPROXY_REPO="https://github.com/haproxy/haproxy.git" typeset BASENAME='haproxy' typeset DIRNAME="${BASENAME}-${VERSION}" typeset CERTDIR="${INSTALL_ROOT}/${SSL_LIB}/conf/certs" - if [[ -z "${SSL_LIB}" ]] ; then + if [[ -z "${SSL_LIB}" ]] ; then SSL_LIB="openssl-master" - fi + fi if [[ -f "${INSTALL_ROOT}/${SSL_LIB}/sbin/haproxy" ]] ; then echo "haproxy already installed; skipping.." @@ -44,6 +44,7 @@ function install_haproxy { git clone "${HAPROXY_REPO}" -b ${VERSION} --depth 1 . || exit 1 # haproxy does not have a configure script; only a big makefile + make clean make ${MAKE_OPTS} \ TARGET=generic \ USE_OPENSSL=1 \ From a5771183101e2d7b3ccfe7697a2d8e0fb1356b94 Mon Sep 17 00:00:00 2001 From: sashan Date: Tue, 21 Oct 2025 01:30:42 +0200 Subject: [PATCH 3/3] - trailing whitespace --- bench-scripts/haproxy_bench.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bench-scripts/haproxy_bench.sh b/bench-scripts/haproxy_bench.sh index a5cbb708..96741bc2 100755 --- a/bench-scripts/haproxy_bench.sh +++ b/bench-scripts/haproxy_bench.sh @@ -172,7 +172,7 @@ function unconf_siege_haproxy_cert { # clear the siege config sed -i '/#haproxy/{N;d;}' "${SIEGE_CONF}" || exit 1 } - + function kill_haproxy { pkill -TERM -f haproxy }