diff --git a/bench-scripts/README.md b/bench-scripts/README.md index d05ea754..66aa9831 100644 --- a/bench-scripts/README.md +++ b/bench-scripts/README.md @@ -75,6 +75,63 @@ Requirements for ubuntu are the following: - libtool - g++ +## Running scripts + +Each benchmark here is implemented by its configuration script which sets up +complete environment (for example `bench_config_apache.sh`) and execution +script which runs the benchmark (for example `bench_run_apache.sh`). All script +which are meant to be executed follow that naming convention: + - `bench_config_*.sh` script builds and configures all tools required to run + the desired benchmark + - `bench_run_*.sh` script executes the benchmarks +There are no command line options all scripts are controlled via environment +variables. The common environment variables which are used by both scripts +are as follows: + - `BENCH_INSTALL_ROOT` - installation prefix for tools and libraries, + `/tmp/bench.binaries` is default + - `BENCH_WORKSPACE_ROOT` - directory used to download and build tools and + libraries, `/tmp/bench.workspace` is default + - `BENCH_MAKE_OPTS` - command line options for `make`, none by default. You may + want to add `-j` option here. + - `BENCH_HTTPS_PORT` - port to use by https server, default is 4430 + - `BENCH_HTTP_PORT` - port to use by http server, default is 8080 + +The `bench_config_*.sh` script further use those two variables here to +control generation of self-signed certificates: + - `BENCH_CERT_SUBJ` - used to generate self-signed certificate, it's + set to `/CN=localhost` by default + - `BENCH_ALT_SUBJ` - it's set to `'subjectAltName=DNS:localhost,IP:127.0.0.1` + by default +Default values are good enough when testing is conveyed over loopback network +interface. You may need to change those values when testing with different +IP addresses/hostnames. + +The environment variables which are specific to `bench_run_*.sh` scripts are +as follows: + - `BENCH_RESULT_DIR` directory where to save collected results, it's + `$BENCH_INSTALL_ROOT/results` by default + - `BENCH_HOST` remote server used by test client tools, `localhost` by + default. + - `BENCH_TEST_TIME` duration of each test run, it's 5 minutes by default + `5M` (tools are using siege [6] client tool) + +## Contributing + +The scripts are written in ksh shell which is widely available on many POSIX +platforms. Those scripts are meant to be portable as much as possible, please +do not introduce platform specific dependencies there. + +When it comes time to implement a script which collects data using kernel +counters (or other mean which is specific to platform), then new script should +be implemented (or derived) from existing scripts. The new script should follow +the naming convention introduced here. All new variables which control script +behavior should be prefixed with `BENCH_`. The file should be named as +`bench_run_apache_xyz.sh` where `xyz` should be replaced by platform name (for +example linux). + +Similarly when adding a script which will use a xtool to collect performance data +should be named as `bench_run_xtool.sh` + [1]: https://www.openssl.org/ [2]: https://www.wolfssl.com/ diff --git a/bench-scripts/apache_bench.sh b/bench-scripts/bench_config_apache.sh similarity index 77% rename from bench-scripts/apache_bench.sh rename to bench-scripts/bench_config_apache.sh index 655f0c25..60d74174 100755 --- a/bench-scripts/apache_bench.sh +++ b/bench-scripts/bench_config_apache.sh @@ -53,20 +53,17 @@ set -x # INSTALL_ROOT=${BENCH_INSTALL_ROOT:-"/tmp/bench.binaries"} -RESULT_DIR=${BENCH_RESULTS:-"${INSTALL_ROOT}/results"} WORKSPACE_ROOT=${BENCH_WORKSPACE_ROOT:-"/tmp/bench.workspace"} MAKE_OPTS=${BENCH_MAKE_OPTS} HTTPS_PORT=${BENCH_HTTPS_PORT:-'4430'} HTTP_PORT=${BENCH_HTTP_PORT:-'8080'} CERT_SUBJ=${BENCH_CERT_SUBJ:-'/CN=localhost'} CERT_ALT_SUBJ=${BENCH_CERT_ALT_SUBJ:-'subjectAltName=DNS:localhost,IP:127.0.0.1'} -TEST_TIME=${BENCH_TEST_TIME:-'5M'} HOST=${BENCH_HOST:-'127.0.0.1'} APACHE_VERSION='2.4.65' -HAPROXY='no' . ./common_util.sh -. ./haproxy_bench.sh +. ./bench_config_haproxy.sh function install_wolfssl_for_apache { typeset VERSION=$1 @@ -744,183 +741,6 @@ function config_apache { generate_download_files "${INSTALL_ROOT}/${SSL_LIB}/htdocs" } -function enable_mpm_event { - typeset SSL_LIB=$1 - if [[ -z "${SSL_LIB}" ]] ; then - SSL_LIB='openssl-master' - fi - typeset CONF_FILE="${INSTALL_ROOT}/${SSL_LIB}/conf/httpd.conf" - - # - # comment out currently loaded mpm module - # - cp "${CONF_FILE}" "${CONF_FILE}".wrk - sed -e 's/\(^LoadModule mpm_.*$\)/#\1/g' \ - "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 - - # - # enable event mpm module - # - cp "${CONF_FILE}" "${CONF_FILE}".wrk - sed -e 's/\(^#\)\(LoadModule mpm_event_module .*$\)/\2/g' \ - "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 -} - -function enable_mpm_worker { - typeset SSL_LIB=$1 - if [[ -z "${SSL_LIB}" ]] ; then - SSL_LIB='openssl-master' - fi - typeset CONF_FILE="${INSTALL_ROOT}/${SSL_LIB}/conf/httpd.conf" - - # - # comment out currently loaded mpm module - # - cp "${CONF_FILE}" "${CONF_FILE}".wrk - sed -e 's/\(^LoadModule mpm_.*$\)/#\1/g' \ - "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 - - # - # enable worker mpm module - # - cp "${CONF_FILE}" "${CONF_FILE}".wrk - sed -e 's/\(^#\)\(LoadModule mpm_worker_module .*$\)/\2/g' \ - "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 -} - -function enable_mpm_prefork { - typeset SSL_LIB=$1 - if [[ -z "${SSL_LIB}" ]] ; then - SSL_LIB='openssl-master' - fi - typeset CONF_FILE="${INSTALL_ROOT}/${SSL_LIB}/conf/httpd.conf" - - # - # comment out currently loaded mpm module - # - cp "${CONF_FILE}" "${CONF_FILE}".wrk - sed -e 's/\(^LoadModule mpm_.*$\)/#\1/g' \ - "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 - - # - # enable pre-fork mpm module - # - cp "${CONF_FILE}" "${CONF_FILE}".wrk - sed -e 's/\(^#\)\(LoadModule mpm_prefork_module .*$\)/\2/g' \ - "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 -} - -function run_test { - typeset SSL_LIB=$1 - typeset HAPROXY=$2 - typeset i=0 - typeset PORT=${HTTPS_PORT} - typeset PROTOCOL="https" - if [[ -z "${SSL_LIB}" ]] ; then - SSL_LIB='openssl-master' - fi - if [[ -z "${HAPROXY}" ]] ; then - HAPROXY='no' - fi - typeset RESULTS="${SSL_LIB}".txt - if [[ "${SSL_LIB}" = 'nossl' ]] ; then - SSL_LIB='openssl-master' - RESULTS='nossl.txt' - PORT=${HTTP_PORT} - PROTOCOL="http" - fi - if [[ "${HAPROXY}" != 'no' ]] ; then - RESULTS="haproxy-${SSL_LIB}-${HAPROXY}.txt" - fi - typeset HTDOCS="${INSTALL_ROOT}/${SSL_LIB}"/htdocs - typeset SIEGE="${INSTALL_ROOT}"/openssl-master/bin/siege - - # - # we always try to use siege from openssl master by default, - # if not found then we try the one which is installed for - # openssl version we'd like to test. - # - if [[ ! -x "${SIEGE}" ]] ; then - echo "no ${SIEGE}" - exit 1 - fi - - # - # generate URLs for sewage - # - # The different modes for haproxy are: - # no: client - server - # no-ssl: client -http- haproxy -http- server - # server: client -https- haproxy -http- server - # client: client -http- haproxy -https- server - # both: client -https- haproxy -https- server - # - # Otherwise said, haproxy is a client when it encrypts the outgoing encryption; - # or it's client side. - # - rm -f siege_urls.txt - for i in `ls -1 ${HTDOCS}/*.txt` ; do - if [[ "${HAPROXY}" = "no" ]] ; then - echo "${PROTOCOL}://${HOST}:${PORT}/`basename $i`" >> siege_urls.txt - elif [[ "${HAPROXY}" = "no-ssl" ]] ; then - echo "http://${HOST}:${HAPROXY_NOSSL_PORT}/`basename $i`" >> siege_urls.txt - elif [[ "${HAPROXY}" = "server" ]] ; then - echo "https://${HOST}:${HAPROXY_C2P_PORT}/`basename $i`" >> siege_urls.txt - elif [[ "${HAPROXY}" = "client" ]] ; then - echo "http://${HOST}:${HAPROXY_P2S_PORT}/`basename $i`" >> siege_urls.txt - elif [[ "${HAPROXY}" = "both" ]] ; then - echo "https://${HOST}:${HAPROXY_C2S_PORT}/`basename $i`" >> siege_urls.txt - fi - done - - if [[ "${HAPROXY}" = "server" ]] || [[ "${HAPROXY}" = "both" ]] ; then - conf_siege_haproxy_cert $SSL_LIB - fi - - # - # start apache httpd server - # - LD_LIBRARY_PATH=${INSTALL_ROOT}/${SSL_LIB}/lib \ - ${INSTALL_ROOT}/${SSL_LIB}/bin/httpd -k start || exit 1 - if [[ $? -ne 0 ]] ; then - echo "could not start ${INSTALL_ROOT}/${SSL_LIB}/bin/httpd" - exit 1 - fi - LD_LIBRARY_PATH=${INSTALL_ROOT}/openssl-master/lib "${SIEGE}" -t ${TEST_TIME} -b \ - -f siege_urls.txt 2> "${RESULT_DIR}/${RESULTS}" - if [[ $? -ne 0 ]] ; then - echo "${INSTALL_ROOT}/${SSL_LIB} can not run siege" - cat "${RESULT_DIR}/${RESULTS}" - exit 1 - fi - - LD_LIBRARY_PATH=${INSTALL_ROOT}/${SSL_LIB}/lib \ - ${INSTALL_ROOT}/${SSL_LIB}/bin/httpd -k stop || exit 1 - sleep 1 - pgrep httpd - while [[ $? -eq 0 ]] ; do - sleep 1 - LD_LIBRARY_PATH=${INSTALL_ROOT}/${SSL_LIB}/lib \ - ${INSTALL_ROOT}/${SSL_LIB}/bin/httpd -k stop || exit 1 - echo "stopping ${INSTALL_ROOT}/${SSL_LIB}/bin/httpd" - pgrep httpd - done - - # - # save apache configuration used for testing along the results. - # we do care about httpd.conf and httpd-ssl.conf only as only - # those two were modified. - # - cp ${INSTALL_ROOT}/${SSL_LIB}/conf/httpd.conf \ - ${RESULT_DIR}/httpd-${SSL_LIB}.conf - cp ${INSTALL_ROOT}/${SSL_LIB}/conf/extra/httpd-ssl.conf \ - ${RESULT_DIR}/httpd-ssl-${SSL_LIB}.conf - - if [[ "${HAPROXY}" = "server" ]] || [[ "${HAPROXY}" = "both" ]] ; then - unconf_siege_haproxy_cert - fi -} - function setup_tests { typeset i=0 install_openssl master @@ -1006,127 +826,7 @@ function setup_tests { clean_build } -function run_tests { - typeset SAVE_RESULT_DIR="${RESULT_DIR}" - typeset HAPROXY_OPTIONS=('no' 'client' 'server' 'both') - typeset i="" - - for i in event worker pre-fork ; do - mkdir -p ${RESULT_DIR}/$i || exit 1 - done - - enable_mpm_event - RESULT_DIR="${SAVE_RESULT_DIR}/event" - run_test nossl - run_haproxy - run_test nossl 'no-ssl' - kill_haproxy - for i in 3.0 3.1 3.2 3.3 3.4 3.5 3.6 ; do - enable_mpm_event openssl-${i} - run_haproxy openssl-${i} - for OPTION in ${HAPROXY_OPTIONS[@]} - do - run_test openssl-${i} ${OPTION} - done - kill_haproxy - done - enable_mpm_event openssl-master - run_haproxy openssl-master - for OPTION in ${HAPROXY_OPTIONS[@]} - do - run_test openssl-master ${OPTION} - done - kill_haproxy - enable_mpm_event OpenSSL_1_1_1-stable - run_test OpenSSL_1_1_1-stable - enable_mpm_event libressl-4.1.0 - run_test libressl-4.1.0 - #enable_mpm_event wolfssl-5.8.2 - #run_test wolfssl-5.8.2 - enable_mpm_event boringssl - run_test boringssl - enable_mpm_event aws-lc - run_test aws-lc - - enable_mpm_worker - RESULT_DIR="${SAVE_RESULT_DIR}/worker" - run_test nossl - run_haproxy - run_test nossl 'no-ssl' - kill_haproxy - for i in 3.0 3.1 3.2 3.3 3.4 3.5 3.6 ; do - enable_mpm_worker openssl-${i} - run_test openssl-${i} - run_haproxy openssl-${i} - for OPTION in ${HAPROXY_OPTIONS[@]} - do - run_test openssl-${i} ${OPTION} - done - kill_haproxy - done - enable_mpm_worker openssl-master - run_haproxy openssl-master - for OPTION in ${HAPROXY_OPTIONS[@]} - do - run_test openssl-master ${OPTION} - done - kill_haproxy - enable_mpm_worker OpenSSL_1_1_1-stable - run_test OpenSSL_1_1_1-stable - enable_mpm_worker libressl-4.1.0 - run_test libressl-4.1.0 - #enable_mpm_worker wolfssl-5.8.2 - #run_test wolfssl-5.8.2 - enable_mpm_worker boringssl - run_test boringssl - enable_mpm_worker aws-lc - run_test aws-lc - - enable_mpm_prefork - RESULT_DIR="${SAVE_RESULT_DIR}/pre-fork" - run_test nossl - run_haproxy - run_test nossl 'no-ssl' - kill_haproxy - for i in 3.0 3.1 3.2 3.3 3.4 3.5 3.6 ; do - enable_mpm_prefork openssl-${i} - run_test openssl-${i} - run_haproxy openssl-${i} - for OPTION in ${HAPROXY_OPTIONS[@]} - do - run_test openssl-${i} ${OPTION} - done - kill_haproxy - done - enable_mpm_prefork openssl-master - run_haproxy openssl-master - for OPTION in ${HAPROXY_OPTIONS[@]} - do - run_test openssl-master ${OPTION} - done - kill_haproxy - enable_mpm_prefork OpenSSL_1_1_1-stable - run_test OpenSSL_1_1_1-stable - enable_mpm_prefork libressl-4.1.0 - run_test libressl-4.1.0 - #enable_mpm_prefork wolfssl-5.8.2 - #run_test wolfssl-5.8.2 - enable_mpm_prefork boringssl - run_test boringssl - enable_mpm_prefork aws-lc - run_test aws-lc - - RESULT_DIR=${SAVE_RESULT_DIR} -} - check_env setup_tests -run_tests -SAVE_RESULT_DIR=${RESULT_DIR} -for i in event worker pre-fork ; do - RESULT_DIR=${SAVE_RESULT_DIR}/$i - plot_results -done -RESULT_DIR=${SAVE_RESULT_DIR} -echo "testing using siege is complete, results can be found ${RESULT_DIR}:" +echo "apache was successfully configured." diff --git a/bench-scripts/haproxy_bench.sh b/bench-scripts/bench_config_haproxy.sh similarity index 81% rename from bench-scripts/haproxy_bench.sh rename to bench-scripts/bench_config_haproxy.sh index 96741bc2..9a7c70b0 100755 --- a/bench-scripts/haproxy_bench.sh +++ b/bench-scripts/bench_config_haproxy.sh @@ -133,46 +133,3 @@ backend https_test server s2 ${HOST}:${HTTPS_PORT} ssl verify required ca-file ${INSTALL_ROOT}/${SSL_LIB}/conf/server.crt EOF } - -function run_haproxy { - typeset SSL_LIB=$1 - if [[ -z "${SSL_LIB}" ]] ; then - SSL_LIB="openssl-master" - fi - typeset OPENSSL_DIR="${INSTALL_ROOT}/${SSL_LIB}" - - LD_LIBRARY_PATH="${OPENSSL_DIR}/lib:${LD_LIBRARY_PATH}" "${OPENSSL_DIR}/sbin/haproxy" -f "${OPENSSL_DIR}/conf/haproxy.cfg" -D - if [[ $? -ne 0 ]] ; then - echo "could not start haproxy" - exit 1 - fi -} - -function conf_siege_haproxy_cert { - typeset SSL_LIB=$1 - if [[ -z "${SSL_LIB}" ]] ; then - SSL_LIB="openssl-master" - fi - typeset OPENSSL_DIR="${INSTALL_ROOT}/${SSL_LIB}" - # siege is currently installed only with openssl-master - typeset SIEGE_CONF="${INSTALL_ROOT}/openssl-master/etc/siegerc" - # configure siege to use haproxy - if [[ ! -f "${SIEGE_CONF}" ]] ; then - echo "Did not found siegerc. Siege should be installed first." - exit 1 - fi - echo "#haproxy" >> "${SIEGE_CONF}" - echo "ssl-cert = ${OPENSSL_DIR}/conf/certs/client_cert.pem" >> "${SIEGE_CONF}" - echo "ssl-key = ${OPENSSL_DIR}/conf/certs/client_key.pem" >> "${SIEGE_CONF}" -} - -function unconf_siege_haproxy_cert { - typeset SIEGE_CONF="${INSTALL_ROOT}/openssl-master/etc/siegerc" - - # clear the siege config - sed -i '/#haproxy/{N;d;}' "${SIEGE_CONF}" || exit 1 -} - -function kill_haproxy { - pkill -TERM -f haproxy -} diff --git a/bench-scripts/nginx_bench.sh b/bench-scripts/bench_config_nginx.sh similarity index 84% rename from bench-scripts/nginx_bench.sh rename to bench-scripts/bench_config_nginx.sh index 394e0772..d58092ff 100755 --- a/bench-scripts/nginx_bench.sh +++ b/bench-scripts/bench_config_nginx.sh @@ -370,82 +370,7 @@ function setup_tests { # clean_build } -function run_test { - typeset SSL_LIB=$1 - typeset HTTP='https' - typeset i=0 - typeset PORT=${HTTPS_PORT} - if [[ -z "${SSL_LIB}" ]] ; then - SSL_LIB='openssl-master' - fi - typeset RESULTS="${SSL_LIB}".txt - if [[ "${SSL_LIB}" = 'nossl' ]] ; then - HTTP='http' - SSL_LIB='openssl-master' - RESULTS='nossl.txt' - PORT=${HTTP_PORT} - fi - typeset HTDOCS="${INSTALL_ROOT}/${SSL_LIB}"/html - typeset SIEGE="${INSTALL_ROOT}"/openssl-master/bin/siege - - # - # we always try to use siege from openssl master by default, - # if not found then we try the one which is installed for - # openssl version we'd like to test. - # - if [[ ! -x "${SIEGE}" ]] ; then - echo "no ${SIEGE}" - exit 1 - fi - - rm -f siege_urls.txt - for i in `ls -1 ${HTDOCS}/*.txt` ; do - echo "${HTTP}://${HOST}:${PORT}/`basename $i`" >> siege_urls.txt - done - - # - # start nginx server - # - echo LD_LIBRARY_PATH=${INSTALL_ROOT}/${SSL_LIB}/lib ${INSTALL_ROOT}/${SSL_LIB}/sbin/nginx - LD_LIBRARY_PATH=${INSTALL_ROOT}/${SSL_LIB}/lib ${INSTALL_ROOT}/${SSL_LIB}/sbin/nginx - if [[ $? -ne 0 ]] ; then - echo "could not start ${INSTALL_ROOT}/${SSL_LIB}/sbin/nginx" - exit 1 - fi - - LD_LIBRARY_PATH=${INSTALL_ROOT}/openssl-master/lib "${SIEGE}" -t ${TEST_TIME} -b \ - -f siege_urls.txt 2> "${RESULT_DIR}/${RESULTS}" - - # - # stop nginx server - # - LD_LIBRARY_PATH=${INSTALL_ROOT}/${SSL_LIB}/lib ${INSTALL_ROOT}/${SSL_LIB}/sbin/nginx -s quit - - # - # save nginx.conf used for testing along the results - # - cp ${INSTALL_ROOT}/${SSL_LIB}/conf/nginx.conf ${RESULT_DIR}/nginx-${SSL_LIB}.conf -} - -function run_tests { - run_test nossl - for i in 3.0 3.1 3.2 3.3 3.4 3.5 3.6 ; do - run_test openssl-${i} - done - run_test openssl-master - run_test OpenSSL_1_1_1-stable - run_test libressl-4.1.0 - # - # could not get apache with wolfssl working - # - run_test wolfssl-5.8.2 - run_test boringssl - #run_test aws-lc -} - check_env setup_tests -run_tests -plot_results -echo "testing using siege is complete, results can be found ${RESULT_DIR}:" +echo "nginx was successfully configured." diff --git a/bench-scripts/bench_run_apache.sh b/bench-scripts/bench_run_apache.sh new file mode 100755 index 00000000..cbc4641c --- /dev/null +++ b/bench-scripts/bench_run_apache.sh @@ -0,0 +1,368 @@ +#!/usr/bin/env ksh +# +# Copyright 2025 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# + +set -x + +# +# +# make sure to disable firewall +# ufw disable +# it feels like ipv6 loopback traffic is disabled on ubuntu +# + +# +# This is the output of apachectl -V we use to test +# libraries: +# +# Server version: Apache/2.4.65 (Unix) +# Server built: Sep 12 2025 14:49:08 +# Server's Module Magic Number: 20120211:141 +# Server loaded: APR 1.7.6, APR-UTIL 1.6.3, PCRE 10.42 2022-12-11 +# Compiled using: APR 1.7.6, APR-UTIL 1.6.3, PCRE 10.42 2022-12-11 +# Architecture: 64-bit +# Server MPM: event +# threaded: yes (fixed thread count) +# forked: yes (variable process count) +# +# the siege client downloads static files which look as follows +# for all tests: +# 64 Sep 12 14:49 test.txt +# 128 Sep 12 14:49 test_1.txt +# 256 Sep 12 14:49 test_2.txt +# 512 Sep 12 14:49 test_3.txt +# 1.0K Sep 12 14:49 test_4.txt +# 2.0K Sep 12 14:49 test_5.txt +# 4.0K Sep 12 14:49 test_6.txt +# 8.0K Sep 12 14:49 test_7.txt +# 16K Sep 12 14:49 test_8.txt +# 32K Sep 12 14:49 test_9.txt +# 64K Sep 12 14:49 test_10.txt +# 128K Sep 12 14:49 test_11.txt +# 256K Sep 12 14:49 test_12.txt +# 512K Sep 12 14:49 test_13.txt +# 1.0M Sep 12 14:49 test_14.txt +# 2.0M Sep 12 14:49 test_15.txt +# 4.0M Sep 12 14:49 test_16.txt +# + +INSTALL_ROOT=${BENCH_INSTALL_ROOT:-"/tmp/bench.binaries"} +RESULT_DIR=${BENCH_RESULTS:-"${INSTALL_ROOT}/results"} +HTTPS_PORT=${BENCH_HTTPS_PORT:-'4430'} +HTTP_PORT=${BENCH_HTTP_PORT:-'8080'} +CERT_SUBJ=${BENCH_CERT_SUBJ:-'/CN=localhost'} +CERT_ALT_SUBJ=${BENCH_CERT_ALT_SUBJ:-'subjectAltName=DNS:localhost,IP:127.0.0.1'} +TEST_TIME=${BENCH_TEST_TIME:-'5M'} +HOST=${BENCH_HOST:-'127.0.0.1'} +APACHE_VERSION='2.4.65' +HAPROXY='no' + +. ./common_util.sh +. ./bench_run_haproxy.sh + +function enable_mpm_event { + typeset SSL_LIB=$1 + if [[ -z "${SSL_LIB}" ]] ; then + SSL_LIB='openssl-master' + fi + typeset CONF_FILE="${INSTALL_ROOT}/${SSL_LIB}/conf/httpd.conf" + + # + # comment out currently loaded mpm module + # + cp "${CONF_FILE}" "${CONF_FILE}".wrk + sed -e 's/\(^LoadModule mpm_.*$\)/#\1/g' \ + "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 + + # + # enable event mpm module + # + cp "${CONF_FILE}" "${CONF_FILE}".wrk + sed -e 's/\(^#\)\(LoadModule mpm_event_module .*$\)/\2/g' \ + "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 +} + +function enable_mpm_worker { + typeset SSL_LIB=$1 + if [[ -z "${SSL_LIB}" ]] ; then + SSL_LIB='openssl-master' + fi + typeset CONF_FILE="${INSTALL_ROOT}/${SSL_LIB}/conf/httpd.conf" + + # + # comment out currently loaded mpm module + # + cp "${CONF_FILE}" "${CONF_FILE}".wrk + sed -e 's/\(^LoadModule mpm_.*$\)/#\1/g' \ + "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 + + # + # enable worker mpm module + # + cp "${CONF_FILE}" "${CONF_FILE}".wrk + sed -e 's/\(^#\)\(LoadModule mpm_worker_module .*$\)/\2/g' \ + "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 +} + +function enable_mpm_prefork { + typeset SSL_LIB=$1 + if [[ -z "${SSL_LIB}" ]] ; then + SSL_LIB='openssl-master' + fi + typeset CONF_FILE="${INSTALL_ROOT}/${SSL_LIB}/conf/httpd.conf" + + # + # comment out currently loaded mpm module + # + cp "${CONF_FILE}" "${CONF_FILE}".wrk + sed -e 's/\(^LoadModule mpm_.*$\)/#\1/g' \ + "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 + + # + # enable pre-fork mpm module + # + cp "${CONF_FILE}" "${CONF_FILE}".wrk + sed -e 's/\(^#\)\(LoadModule mpm_prefork_module .*$\)/\2/g' \ + "${CONF_FILE}".wrk > "${CONF_FILE}" || exit 1 +} + +function run_test { + typeset SSL_LIB=$1 + typeset HAPROXY=$2 + typeset i=0 + typeset PORT=${HTTPS_PORT} + typeset PROTOCOL="https" + if [[ -z "${SSL_LIB}" ]] ; then + SSL_LIB='openssl-master' + fi + if [[ -z "${HAPROXY}" ]] ; then + HAPROXY='no' + fi + typeset RESULTS="${SSL_LIB}".txt + if [[ "${SSL_LIB}" = 'nossl' ]] ; then + SSL_LIB='openssl-master' + RESULTS='nossl.txt' + PORT=${HTTP_PORT} + PROTOCOL="http" + fi + if [[ "${HAPROXY}" != 'no' ]] ; then + RESULTS="haproxy-${SSL_LIB}-${HAPROXY}.txt" + fi + typeset HTDOCS="${INSTALL_ROOT}/${SSL_LIB}"/htdocs + typeset SIEGE="${INSTALL_ROOT}"/openssl-master/bin/siege + + # + # we always try to use siege from openssl master by default, + # if not found then we try the one which is installed for + # openssl version we'd like to test. + # + if [[ ! -x "${SIEGE}" ]] ; then + echo "no ${SIEGE}" + exit 1 + fi + + # + # generate URLs for sewage + # + # The different modes for haproxy are: + # no: client - server + # no-ssl: client -http- haproxy -http- server + # server: client -https- haproxy -http- server + # client: client -http- haproxy -https- server + # both: client -https- haproxy -https- server + # + # Otherwise said, haproxy is a client when it encrypts the outgoing encryption; + # or it's client side. + # + rm -f siege_urls.txt + for i in `ls -1 ${HTDOCS}/*.txt` ; do + if [[ "${HAPROXY}" = "no" ]] ; then + echo "${PROTOCOL}://${HOST}:${PORT}/`basename $i`" >> siege_urls.txt + elif [[ "${HAPROXY}" = "no-ssl" ]] ; then + echo "http://${HOST}:${HAPROXY_NOSSL_PORT}/`basename $i`" >> siege_urls.txt + elif [[ "${HAPROXY}" = "server" ]] ; then + echo "https://${HOST}:${HAPROXY_C2P_PORT}/`basename $i`" >> siege_urls.txt + elif [[ "${HAPROXY}" = "client" ]] ; then + echo "http://${HOST}:${HAPROXY_P2S_PORT}/`basename $i`" >> siege_urls.txt + elif [[ "${HAPROXY}" = "both" ]] ; then + echo "https://${HOST}:${HAPROXY_C2S_PORT}/`basename $i`" >> siege_urls.txt + fi + done + + if [[ "${HAPROXY}" = "server" ]] || [[ "${HAPROXY}" = "both" ]] ; then + conf_siege_haproxy_cert $SSL_LIB + fi + + # + # start apache httpd server + # + LD_LIBRARY_PATH=${INSTALL_ROOT}/${SSL_LIB}/lib \ + ${INSTALL_ROOT}/${SSL_LIB}/bin/httpd -k start || exit 1 + if [[ $? -ne 0 ]] ; then + echo "could not start ${INSTALL_ROOT}/${SSL_LIB}/bin/httpd" + exit 1 + fi + LD_LIBRARY_PATH=${INSTALL_ROOT}/openssl-master/lib "${SIEGE}" -t ${TEST_TIME} -b \ + -f siege_urls.txt 2> "${RESULT_DIR}/${RESULTS}" + if [[ $? -ne 0 ]] ; then + echo "${INSTALL_ROOT}/${SSL_LIB} can not run siege" + cat "${RESULT_DIR}/${RESULTS}" + exit 1 + fi + + LD_LIBRARY_PATH=${INSTALL_ROOT}/${SSL_LIB}/lib \ + ${INSTALL_ROOT}/${SSL_LIB}/bin/httpd -k stop || exit 1 + sleep 1 + pgrep httpd + while [[ $? -eq 0 ]] ; do + sleep 1 + LD_LIBRARY_PATH=${INSTALL_ROOT}/${SSL_LIB}/lib \ + ${INSTALL_ROOT}/${SSL_LIB}/bin/httpd -k stop || exit 1 + echo "stopping ${INSTALL_ROOT}/${SSL_LIB}/bin/httpd" + pgrep httpd + done + + # + # save apache configuration used for testing along the results. + # we do care about httpd.conf and httpd-ssl.conf only as only + # those two were modified. + # + cp ${INSTALL_ROOT}/${SSL_LIB}/conf/httpd.conf \ + ${RESULT_DIR}/httpd-${SSL_LIB}.conf + cp ${INSTALL_ROOT}/${SSL_LIB}/conf/extra/httpd-ssl.conf \ + ${RESULT_DIR}/httpd-ssl-${SSL_LIB}.conf + + if [[ "${HAPROXY}" = "server" ]] || [[ "${HAPROXY}" = "both" ]] ; then + unconf_siege_haproxy_cert + fi +} + +function run_tests { + typeset SAVE_RESULT_DIR="${RESULT_DIR}" + typeset HAPROXY_OPTIONS=('no' 'client' 'server' 'both') + typeset i="" + + for i in event worker pre-fork ; do + mkdir -p ${RESULT_DIR}/$i || exit 1 + done + + enable_mpm_event + RESULT_DIR="${SAVE_RESULT_DIR}/event" + run_test nossl + run_haproxy + run_test nossl 'no-ssl' + kill_haproxy + for i in 3.0 3.1 3.2 3.3 3.4 3.5 3.6 ; do + enable_mpm_event openssl-${i} + run_haproxy openssl-${i} + for OPTION in ${HAPROXY_OPTIONS[@]} + do + run_test openssl-${i} ${OPTION} + done + kill_haproxy + done + enable_mpm_event openssl-master + run_haproxy openssl-master + for OPTION in ${HAPROXY_OPTIONS[@]} + do + run_test openssl-master ${OPTION} + done + kill_haproxy + enable_mpm_event OpenSSL_1_1_1-stable + run_test OpenSSL_1_1_1-stable + enable_mpm_event libressl-4.1.0 + run_test libressl-4.1.0 + #enable_mpm_event wolfssl-5.8.2 + #run_test wolfssl-5.8.2 + enable_mpm_event boringssl + run_test boringssl + enable_mpm_event aws-lc + run_test aws-lc + + enable_mpm_worker + RESULT_DIR="${SAVE_RESULT_DIR}/worker" + run_test nossl + run_haproxy + run_test nossl 'no-ssl' + kill_haproxy + for i in 3.0 3.1 3.2 3.3 3.4 3.5 3.6 ; do + enable_mpm_worker openssl-${i} + run_test openssl-${i} + run_haproxy openssl-${i} + for OPTION in ${HAPROXY_OPTIONS[@]} + do + run_test openssl-${i} ${OPTION} + done + kill_haproxy + done + enable_mpm_worker openssl-master + run_haproxy openssl-master + for OPTION in ${HAPROXY_OPTIONS[@]} + do + run_test openssl-master ${OPTION} + done + kill_haproxy + enable_mpm_worker OpenSSL_1_1_1-stable + run_test OpenSSL_1_1_1-stable + enable_mpm_worker libressl-4.1.0 + run_test libressl-4.1.0 + #enable_mpm_worker wolfssl-5.8.2 + #run_test wolfssl-5.8.2 + enable_mpm_worker boringssl + run_test boringssl + enable_mpm_worker aws-lc + run_test aws-lc + + enable_mpm_prefork + RESULT_DIR="${SAVE_RESULT_DIR}/pre-fork" + run_test nossl + run_haproxy + run_test nossl 'no-ssl' + kill_haproxy + for i in 3.0 3.1 3.2 3.3 3.4 3.5 3.6 ; do + enable_mpm_prefork openssl-${i} + run_test openssl-${i} + run_haproxy openssl-${i} + for OPTION in ${HAPROXY_OPTIONS[@]} + do + run_test openssl-${i} ${OPTION} + done + kill_haproxy + done + enable_mpm_prefork openssl-master + run_haproxy openssl-master + for OPTION in ${HAPROXY_OPTIONS[@]} + do + run_test openssl-master ${OPTION} + done + kill_haproxy + enable_mpm_prefork OpenSSL_1_1_1-stable + run_test OpenSSL_1_1_1-stable + enable_mpm_prefork libressl-4.1.0 + run_test libressl-4.1.0 + #enable_mpm_prefork wolfssl-5.8.2 + #run_test wolfssl-5.8.2 + enable_mpm_prefork boringssl + run_test boringssl + enable_mpm_prefork aws-lc + run_test aws-lc + + RESULT_DIR=${SAVE_RESULT_DIR} +} + +check_env +run_tests +SAVE_RESULT_DIR=${RESULT_DIR} +for i in event worker pre-fork ; do + RESULT_DIR=${SAVE_RESULT_DIR}/$i + plot_results +done +RESULT_DIR=${SAVE_RESULT_DIR} + +echo "testing using siege is complete, results can be found ${RESULT_DIR}:" diff --git a/bench-scripts/bench_run_haproxy.sh b/bench-scripts/bench_run_haproxy.sh new file mode 100755 index 00000000..940aa58d --- /dev/null +++ b/bench-scripts/bench_run_haproxy.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env ksh +# +# Copyright 2025 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# + +set -x + +INSTALL_ROOT=${BENCH_INSTALL_ROOT:-"/tmp/bench.binaries"} +WORKSPACE_ROOT=${BENCH_WORKSPACE_ROOT:-"/tmp/bench.workspace"} +HAPROXY_NOSSL_PORT='42128' +HAPROXY_C2P_PORT='42132' +HAPROXY_P2S_PORT='42134' +HAPROXY_C2S_PORT='42136' +CERT_SUBJ=${BENCH_CERT_SUBJ:-'/CN=localhost'} +CERT_ALT_SUBJ=${BENCH_CERT_ALT_SUBJ:-'subjectAltName=DNS:localhost,IP:127.0.0.1'} +HOST=${BENCH_HOST:-'127.0.0.1'} +HAPROXY_VERSION='v3.2.0' + +# +# Starts haproxy based on the configuration that was done beforehand calling +# install_haproxy. +# +function run_haproxy { + typeset SSL_LIB=$1 + if [[ -z "${SSL_LIB}" ]] ; then + SSL_LIB="openssl-master" + fi + typeset OPENSSL_DIR="${INSTALL_ROOT}/${SSL_LIB}" + + LD_LIBRARY_PATH="${OPENSSL_DIR}/lib:${LD_LIBRARY_PATH}" "${OPENSSL_DIR}/sbin/haproxy" -f "${OPENSSL_DIR}/conf/haproxy.cfg" -D + if [[ $? -ne 0 ]] ; then + echo "could not start haproxy" + exit 1 + fi +} + +# +# Configures the client (siege) to run with haproxy modes server and both. +# Those modes require the client to have the haproxy certificates. +# +function conf_siege_haproxy_cert { + typeset SSL_LIB=$1 + if [[ -z "${SSL_LIB}" ]] ; then + SSL_LIB="openssl-master" + fi + typeset OPENSSL_DIR="${INSTALL_ROOT}/${SSL_LIB}" + # siege is currently installed only with openssl-master + typeset SIEGE_CONF="${INSTALL_ROOT}/openssl-master/etc/siegerc" + # configure siege to use haproxy + if [[ ! -f "${SIEGE_CONF}" ]] ; then + echo "Did not found siegerc. Siege should be installed first." + exit 1 + fi + echo "#haproxy" >> "${SIEGE_CONF}" + echo "ssl-cert = ${OPENSSL_DIR}/conf/certs/client_cert.pem" >> "${SIEGE_CONF}" + echo "ssl-key = ${OPENSSL_DIR}/conf/certs/client_key.pem" >> "${SIEGE_CONF}" +} + +# +# Clears the haproxy certificates from the siege client config. +# +function unconf_siege_haproxy_cert { + typeset SIEGE_CONF="${INSTALL_ROOT}/openssl-master/etc/siegerc" + + # clear the siege config + sed -i '/#haproxy/{N;d;}' "${SIEGE_CONF}" || exit 1 +} + +function kill_haproxy { + pkill -TERM -f haproxy +} diff --git a/bench-scripts/bench_run_nginx.sh b/bench-scripts/bench_run_nginx.sh new file mode 100755 index 00000000..e12f7c17 --- /dev/null +++ b/bench-scripts/bench_run_nginx.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env ksh +# +# Copyright 2025 The OpenSSL Project Authors. All Rights Reserved. +# +# Licensed under the Apache License 2.0 (the "License"). You may not use +# this file except in compliance with the License. You can obtain a copy +# in the file LICENSE in the source distribution or at +# https://www.openssl.org/source/license.html +# + +set -x + +# +# +# make sure to disable firewall +# ufw disable +# it feels like ipv6 loopback traffic is disabled on ubuntu +# +INSTALL_ROOT=${BENCH_INSTALL_ROOT:-"/tmp/bench.binaries"} +RESULT_DIR=${BENCH_RESULTS:-"${INSTALL_ROOT}/results"} +WORKSPACE_ROOT=${BENCH_WORKSPACE_ROOT:-"/tmp/bench.workspace"} +MAKE_OPTS=${BENCH_MAKE_OPTS} +HTTPS_PORT=${BENCH_HTTPS_PORT:-'4430'} +HTTP_PORT=${BENCH_HTTP_PORT:-'8080'} +CERT_SUBJ=${BENCH_CERT_SUBJ:-'/CN=localhost'} +CERT_ALT_SUBJ=${BENCH_CERT_ALT_SUBJ:-'subjectAltName=DNS:localhost,IP:127.0.0.1'} +TEST_TIME=${BENCH_TEST_TIME:-'5M'} +HOST=${BENCH_HOST:-'127.0.0.1'} + +. ./common_util.sh + +# +# the script builds various libssl libraries: +# openssl, wolfssl, boringss, libressl +# each library is installed to its own install root under INSTALL_ROOT +# directory. The script also builds nginx server version 1.28 and installs +# it alongside each openssl library. +# +# for openssl the build process is straightforward: +# clone desired version from github +# +# build it with prefix set to INSTALL_ROOT/openssl-version +# and install it +# +# then clone the nginx server version 1.28, build process +# for nginx is straightforward. the build options are as follows: +# --with-http_ssl_module \ +# --with-threads \ +# --with-cc-opt="-fPIC" \ +# --with-ld-opt="-Wl,-rpath,${INSTALL_ROOT}/${SSL_LIB}/lib -L ${INSTALL_ROOT}/${SSL_LIB}/lib -lcrypto -L ${INSTALL_ROOT}/${SSL_LIB}/lib -lssl" \ +# --with-openssl="${WORKSPACE_ROOT}/${SSL_LIB}" || exit 1 +# with-openssl flag points to openssl sources +# we deliberately pass lcrypto/lssl as we want to link them dynamically +# without this hack the build process picks static version +# found in WORKSPACE_ROOT/SSL_LIB +# +# for libressl the build process is similar. however we download +# ,tar.gz package instead doing git clone +# +# for boringssl the build process slightly differs as boringssl uses cmake. +# the build flags for boringssl are as follows: +# -DCMAKE_INSTALL_PREFIX="${INSTALL_ROOT}/${BORING_NAME}" \ +# -DBUILD_SHARED_LIBS=1 \ +# -DCMAKE_BUILD_TYPE=Release +# nginx build process needs to be adjusted too. It happens in separate +# function setup_sslib_for_nginx() here in shell. Basically we create +# .openssl directory under $WORKSPACE_ROOT/boringssl sourcetree and populate +# it with boringssl headers files and create ,openssl/lib/libcrypto.a +# and .openssl/lib/libssl.a empty files using touch(1) this hack +# is sufficient to get nginx build process going. The nginx expects +# static libraries but we are forcing it to use dynamic versions +# by tweaking --with-ld-opt flags. +# +# there is a separate function install_wolf_nginx() which builds nginx +# with wolfssl it follows guide found here: +# https://github.com/wolfssl/wolfssl-nginx +# +# all nginx servers use the same configuration: +# +# worker_processes auto; +# events { +# worker_connections 1024; +#} +# +# http { +# include mime.types; +# default_type application/octet-stream; +# #access_log logs/access.log main; +# sendfile on; +# #tcp_nopush on; +# #keepalive_timeout 0; +# keepalive_timeout 65; +# +# #gzip on; +# +# server { +# listen ${HTTP_PORT}; +# server_name ${SERVER_NAME}; +# +# location / { +# root html; +# index index.html index.htm; +# } +# +# #error_page 404 /404.html; +# +# # redirect server error pages to the static page /50x.html +# # +# error_page 500 502 503 504 /50x.html; +# location = /50x.html { +# root html; +# } +# +# } +# +# # HTTPS server +# # +# server { +# listen ${HTTPS_PORT} ssl; +# server_name ${SERVER_NAME}; +# +# ssl_certificate ${SERVERCERT}; +# ssl_certificate_key ${SERVERKEY}; +# +# ssl_ciphers HIGH:!aNULL:!MD5; +# ssl_prefer_server_ciphers on; +# +# location / { +# root html; +# index index.html index.htm; +# } +# } +# } +# +# the serverkey and servercert are self-signed certificate for +# localhost/127.0.0.1 +# +# The performance is tested using siege (https://github.com/JoeDog/siege +# scripts build it and installs it along openssl-master +# the client by default fetches set of 17 urls for 5 minutes to +# gather performance data for each nginx/ssl combination. +# the sizes of files which are downloaded are {64, 128, 256, ... 4MB) +# + + +function run_test { + typeset SSL_LIB=$1 + typeset HTTP='https' + typeset i=0 + typeset PORT=${HTTPS_PORT} + if [[ -z "${SSL_LIB}" ]] ; then + SSL_LIB='openssl-master' + fi + typeset RESULTS="${SSL_LIB}".txt + if [[ "${SSL_LIB}" = 'nossl' ]] ; then + HTTP='http' + SSL_LIB='openssl-master' + RESULTS='nossl.txt' + PORT=${HTTP_PORT} + fi + typeset HTDOCS="${INSTALL_ROOT}/${SSL_LIB}"/html + typeset SIEGE="${INSTALL_ROOT}"/openssl-master/bin/siege + + # + # we always try to use siege from openssl master by default, + # if not found then we try the one which is installed for + # openssl version we'd like to test. + # + if [[ ! -x "${SIEGE}" ]] ; then + echo "no ${SIEGE}" + exit 1 + fi + + rm -f siege_urls.txt + for i in `ls -1 ${HTDOCS}/*.txt` ; do + echo "${HTTP}://${HOST}:${PORT}/`basename $i`" >> siege_urls.txt + done + + # + # start nginx server + # + echo LD_LIBRARY_PATH=${INSTALL_ROOT}/${SSL_LIB}/lib ${INSTALL_ROOT}/${SSL_LIB}/sbin/nginx + LD_LIBRARY_PATH=${INSTALL_ROOT}/${SSL_LIB}/lib ${INSTALL_ROOT}/${SSL_LIB}/sbin/nginx + if [[ $? -ne 0 ]] ; then + echo "could not start ${INSTALL_ROOT}/${SSL_LIB}/sbin/nginx" + exit 1 + fi + + LD_LIBRARY_PATH=${INSTALL_ROOT}/openssl-master/lib "${SIEGE}" -t ${TEST_TIME} -b \ + -f siege_urls.txt 2> "${RESULT_DIR}/${RESULTS}" + + # + # stop nginx server + # + LD_LIBRARY_PATH=${INSTALL_ROOT}/${SSL_LIB}/lib ${INSTALL_ROOT}/${SSL_LIB}/sbin/nginx -s quit + + # + # save nginx.conf used for testing along the results + # + cp ${INSTALL_ROOT}/${SSL_LIB}/conf/nginx.conf ${RESULT_DIR}/nginx-${SSL_LIB}.conf +} + +function run_tests { + run_test nossl + for i in 3.0 3.1 3.2 3.3 3.4 3.5 3.6 ; do + run_test openssl-${i} + done + run_test openssl-master + run_test OpenSSL_1_1_1-stable + run_test libressl-4.1.0 + # + # could not get apache with wolfssl working + # + run_test wolfssl-5.8.2 + run_test boringssl + #run_test aws-lc +} + +check_env +run_tests +plot_results + +echo "testing using siege is complete, results can be found ${RESULT_DIR}:"