diff --git a/Makefile b/Makefile index 3b834618..d7f9f812 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,8 @@ NAME = mailserver2/mailserver:testing +# Maximum time in seconds to wait for a service to become ready +WAIT_TIMEOUT = 300 + all: build-no-cache default reverse ldap ldap2 sieve ecdsa traefik_acmev1 traefik_acmev2 clean no-build: default reverse ldap ldap2 sieve ecdsa traefik_acmev1 traefik_acmev2 clean default: init_default fixtures_default run_default stop_default @@ -29,6 +32,9 @@ init_openldap: -e LDAP_TLS=false \ -v "`pwd`/test/config/ldap/struct.ldif":/container/service/slapd/assets/config/bootstrap/ldif/custom/struct.ldif \ -t osixia/openldap:1.4.0 --copy-service + @echo "Waiting for openldap to become ready ..." + @timeout $(WAIT_TIMEOUT) sh -c 'until docker exec openldap ldapsearch -x -H ldap://localhost -D "cn=admin,dc=domain,dc=tld" -w testpasswd -b "o=mx,dc=domain,dc=tld" -s base >/dev/null 2>&1; do sleep 2; done' \ + || { echo "TIMEOUT: openldap did not become ready"; docker logs --tail 50 openldap; exit 1; } init_redis: -docker rm -f \ @@ -37,7 +43,9 @@ init_redis: -d \ --name redis \ -t redis:7.0-alpine - sleep 10 + @echo "Waiting for redis to become ready ..." + @timeout $(WAIT_TIMEOUT) sh -c 'until docker exec redis redis-cli ping 2>/dev/null | grep -q PONG; do sleep 2; done' \ + || { echo "TIMEOUT: redis did not become ready"; docker logs --tail 50 redis; exit 1; } init_mariadb: -docker rm -f \ @@ -52,6 +60,9 @@ init_mariadb: -v "`pwd`/test/config/mariadb/struct.sql":/docker-entrypoint-initdb.d/struct.sql \ -v "`pwd`/test/config/mariadb/bind.cnf":/etc/mysql/conf.d/bind.cnf \ -t mysql:8 + @echo "Waiting for mariadb to become ready ..." + @timeout $(WAIT_TIMEOUT) sh -c 'until docker exec mariadb mysql -upostfix -ptestpasswd -e "SELECT 1 FROM mailbox LIMIT 1" postfix >/dev/null 2>&1; do sleep 2; done' \ + || { echo "TIMEOUT: mariadb did not become ready"; docker logs --tail 50 mariadb; exit 1; } init_postgres: -docker rm -f \ @@ -64,6 +75,9 @@ init_postgres: -e POSTGRES_PASSWORD=testpasswd \ -v "`pwd`/test/config/postgres":/docker-entrypoint-initdb.d \ -t postgres:16-alpine + @echo "Waiting for postgres to become ready ..." + @timeout $(WAIT_TIMEOUT) sh -c 'until docker exec postgres psql -U postfix -d postfix -c "SELECT 1 FROM mailbox LIMIT 1" >/dev/null 2>&1; do sleep 2; done' \ + || { echo "TIMEOUT: postgres did not become ready"; docker logs --tail 50 postgres; exit 1; } init_ldap: init_openldap init_redis -docker rm -f \ @@ -125,18 +139,21 @@ fixtures_ldap: # Wait for rspamd to start (ldap) docker exec mailserver_ldap /bin/sh -c "while ! echo PING | nc -z 0.0.0.0 11332 ; do sleep 1 ; done" - docker exec mailserver_ldap /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-existing-user.txt" - docker exec mailserver_ldap /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-existing-user-spam-learning.txt" - docker exec mailserver_ldap /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-valid-user-subaddress.txt" - docker exec mailserver_ldap /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-non-existing-user.txt" - docker exec mailserver_ldap /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-existing-alias.txt" - docker exec mailserver_ldap /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-existing-alias-forward.txt" - docker exec mailserver_ldap /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-existing-alias-group.txt" - docker exec mailserver_ldap /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-spam-to-existing-user.txt" - docker exec mailserver_ldap /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-virus-to-existing-user.txt" - docker exec mailserver_ldap /bin/sh -c "openssl s_client -ign_eof -connect 0.0.0.0:587 -starttls smtp < /tmp/tests/email-templates/internal-user-to-existing-user.txt" - docker exec mailserver_ldap /bin/sh -c "openssl s_client -ign_eof -connect 0.0.0.0:587 -starttls smtp < /tmp/tests/email-templates/internal-rejected-user-to-existing-user.txt" - sleep 2 + docker exec mailserver_ldap /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-existing-user.txt" + docker exec mailserver_ldap /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-existing-user-spam-learning.txt" + docker exec mailserver_ldap /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-valid-user-subaddress.txt" + docker exec mailserver_ldap /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-non-existing-user.txt" + docker exec mailserver_ldap /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-existing-alias.txt" + docker exec mailserver_ldap /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-existing-alias-forward.txt" + docker exec mailserver_ldap /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-existing-alias-group.txt" + docker exec mailserver_ldap /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-spam-to-existing-user.txt" + docker exec mailserver_ldap /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-virus-to-existing-user.txt" + docker exec mailserver_ldap /bin/sh -c "python3 /tmp/tests/smtp-send.py --starttls 0.0.0.0 587 /tmp/tests/email-templates/internal-user-to-existing-user.txt" + docker exec mailserver_ldap /bin/sh -c "python3 /tmp/tests/smtp-send.py --starttls 0.0.0.0 587 /tmp/tests/email-templates/internal-rejected-user-to-existing-user.txt" + # Wait until postfix has delivered everything it accepted + @echo "Waiting for the postfix queue to drain (mailserver_ldap) ..." + @timeout $(WAIT_TIMEOUT) sh -c 'until [ -z "$$(docker exec mailserver_ldap find /var/spool/postfix/incoming /var/spool/postfix/active /var/spool/postfix/maildrop -type f 2>/dev/null)" ]; do sleep 2; done' \ + || { echo "TIMEOUT: postfix queue did not drain"; docker exec mailserver_ldap postqueue -p; exit 1; } docker exec mailserver_ldap /bin/sh -c "openssl s_client -ign_eof -connect 0.0.0.0:993 < /tmp/tests/sieve/trigger-spam-ham-learning.txt" run_ldap: ./test/bats/bin/bats test/ldap.bats @@ -199,15 +216,21 @@ init_ldap2: init_openldap init_redis -t $(NAME) fixtures_ldap2: docker exec mailserver_ldap2 /bin/sh -c "while ! echo PING | nc -z 0.0.0.0 25 ; do sleep 1 ; done" - sleep 30 - docker exec mailserver_ldap2 /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-existing-user.txt" - docker exec mailserver_ldap2 /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-valid-user-subaddress.txt" - docker exec mailserver_ldap2 /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-non-existing-user.txt" - docker exec mailserver_ldap2 /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-existing-alias.txt" - docker exec mailserver_ldap2 /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-existing-alias-forward.txt" - docker exec mailserver_ldap2 /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-existing-alias-group.txt" - docker exec mailserver_ldap2 /bin/sh -c "openssl s_client -ign_eof -connect 0.0.0.0:587 -starttls smtp < /tmp/tests/email-templates/internal-user-to-existing-user.txt" - sleep 10 + # Wait for the rest of the stack to start listening + docker exec mailserver_ldap2 /bin/sh -c "while ! echo PING | nc -z 0.0.0.0 11332 ; do sleep 1 ; done" # rspamd + docker exec mailserver_ldap2 /bin/sh -c "while ! echo PING | nc -z 0.0.0.0 993 ; do sleep 1 ; done" # dovecot imaps + docker exec mailserver_ldap2 /bin/sh -c "while ! echo PING | nc -z 0.0.0.0 587 ; do sleep 1 ; done" # submission + docker exec mailserver_ldap2 /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-existing-user.txt" + docker exec mailserver_ldap2 /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-valid-user-subaddress.txt" + docker exec mailserver_ldap2 /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-non-existing-user.txt" + docker exec mailserver_ldap2 /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-existing-alias.txt" + docker exec mailserver_ldap2 /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-existing-alias-forward.txt" + docker exec mailserver_ldap2 /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-existing-alias-group.txt" + docker exec mailserver_ldap2 /bin/sh -c "python3 /tmp/tests/smtp-send.py --starttls 0.0.0.0 587 /tmp/tests/email-templates/internal-user-to-existing-user.txt" + # Wait until postfix has delivered everything it accepted + @echo "Waiting for the postfix queue to drain (mailserver_ldap2) ..." + @timeout $(WAIT_TIMEOUT) sh -c 'until [ -z "$$(docker exec mailserver_ldap2 find /var/spool/postfix/incoming /var/spool/postfix/active /var/spool/postfix/maildrop -type f 2>/dev/null)" ]; do sleep 2; done' \ + || { echo "TIMEOUT: postfix queue did not drain"; docker exec mailserver_ldap2 postqueue -p; exit 1; } run_ldap2: ./test/bats/bin/bats test/ldap2.bats stop_ldap2: @@ -218,8 +241,6 @@ init_default: init_redis init_mariadb -docker rm -f \ mailserver_default || true - sleep 60 - docker run \ -d \ --name mailserver_default \ @@ -244,7 +265,6 @@ init_default: init_redis init_mariadb init_reverse: init_redis init_postgres -docker rm -f \ mailserver_reverse || true - sleep 10 docker run \ -d \ --name mailserver_reverse \ @@ -284,15 +304,20 @@ init_reverse: init_redis init_postgres -t $(NAME) fixtures_reverse: docker exec mailserver_reverse /bin/sh -c "while ! echo PING | nc -z 0.0.0.0 25 ; do sleep 1 ; done" - sleep 30 - docker exec mailserver_reverse /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-existing-user.txt" - docker exec mailserver_reverse /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-valid-user-subaddress-with-default-separator.txt" - docker exec mailserver_reverse /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-non-existing-user.txt" - docker exec mailserver_reverse /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-existing-alias.txt" - docker exec mailserver_reverse /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-spam-to-existing-user.txt" - docker exec mailserver_reverse /bin/sh -c "openssl s_client -ign_eof -connect 0.0.0.0:587 -starttls smtp < /tmp/tests/email-templates/internal-user-to-existing-user.txt" - # Wait until all mails have been processed - sleep 10 + # Wait for the rest of the stack to start listening + docker exec mailserver_reverse /bin/sh -c "while ! echo PING | nc -z 0.0.0.0 11332 ; do sleep 1 ; done" # rspamd + docker exec mailserver_reverse /bin/sh -c "while ! echo PING | nc -z 0.0.0.0 993 ; do sleep 1 ; done" # dovecot imaps + docker exec mailserver_reverse /bin/sh -c "while ! echo PING | nc -z 0.0.0.0 587 ; do sleep 1 ; done" # submission + docker exec mailserver_reverse /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-existing-user.txt" + docker exec mailserver_reverse /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-valid-user-subaddress-with-default-separator.txt" + docker exec mailserver_reverse /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-non-existing-user.txt" + docker exec mailserver_reverse /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-existing-alias.txt" + docker exec mailserver_reverse /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-spam-to-existing-user.txt" + docker exec mailserver_reverse /bin/sh -c "python3 /tmp/tests/smtp-send.py --starttls 0.0.0.0 587 /tmp/tests/email-templates/internal-user-to-existing-user.txt" + # Wait until postfix has delivered everything it accepted + @echo "Waiting for the postfix queue to drain (mailserver_reverse) ..." + @timeout $(WAIT_TIMEOUT) sh -c 'until [ -z "$$(docker exec mailserver_reverse find /var/spool/postfix/incoming /var/spool/postfix/active /var/spool/postfix/maildrop -type f 2>/dev/null)" ]; do sleep 2; done' \ + || { echo "TIMEOUT: postfix queue did not drain"; docker exec mailserver_reverse postqueue -p; exit 1; } run_reverse: ./test/bats/bin/bats test/reverse.bats stop_reverse: @@ -302,7 +327,6 @@ stop_reverse: init_ecdsa: init_redis init_mariadb -docker rm -f \ mailserver_ecdsa || true - sleep 10 docker run \ -d \ --name mailserver_ecdsa \ @@ -386,19 +410,24 @@ fixtures_default: # Wait for rspamd to start (default) docker exec mailserver_default /bin/sh -c "while ! echo PING | nc -z 0.0.0.0 11332 ; do sleep 1 ; done" - docker exec mailserver_default /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-existing-user.txt" - docker exec mailserver_default /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-existing-user-spam-learning.txt" - docker exec mailserver_default /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-valid-user-subaddress.txt" - docker exec mailserver_default /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-non-existing-user.txt" - docker exec mailserver_default /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-existing-alias.txt" - docker exec mailserver_default /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-spam-to-existing-user.txt" - docker exec mailserver_default /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-virus-to-existing-user.txt" - docker exec mailserver_default /bin/sh -c "openssl s_client -ign_eof -connect 0.0.0.0:587 -starttls smtp < /tmp/tests/email-templates/internal-user-to-existing-user.txt" - docker exec mailserver_default /bin/sh -c "openssl s_client -ign_eof -connect 0.0.0.0:587 -starttls smtp < /tmp/tests/email-templates/internal-rejected-user-to-existing-user.txt" - sleep 2 + docker exec mailserver_default /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-existing-user.txt" + docker exec mailserver_default /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-existing-user-spam-learning.txt" + docker exec mailserver_default /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-valid-user-subaddress.txt" + docker exec mailserver_default /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-non-existing-user.txt" + docker exec mailserver_default /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-existing-alias.txt" + docker exec mailserver_default /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-spam-to-existing-user.txt" + docker exec mailserver_default /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-virus-to-existing-user.txt" + docker exec mailserver_default /bin/sh -c "python3 /tmp/tests/smtp-send.py --starttls 0.0.0.0 587 /tmp/tests/email-templates/internal-user-to-existing-user.txt" + docker exec mailserver_default /bin/sh -c "python3 /tmp/tests/smtp-send.py --starttls 0.0.0.0 587 /tmp/tests/email-templates/internal-rejected-user-to-existing-user.txt" + # Wait until postfix has delivered everything it accepted + @echo "Waiting for the postfix queue to drain (mailserver_default) ..." + @timeout $(WAIT_TIMEOUT) sh -c 'until [ -z "$$(docker exec mailserver_default find /var/spool/postfix/incoming /var/spool/postfix/active /var/spool/postfix/maildrop -type f 2>/dev/null)" ]; do sleep 2; done' \ + || { echo "TIMEOUT: postfix queue did not drain"; docker exec mailserver_default postqueue -p; exit 1; } docker exec mailserver_default /bin/sh -c "openssl s_client -ign_eof -connect 0.0.0.0:993 < /tmp/tests/sieve/trigger-spam-ham-learning.txt" - # Wait until all mails have been processed - sleep 10 + # Wait until postfix has delivered everything it accepted + @echo "Waiting for the postfix queue to drain (mailserver_default) ..." + @timeout $(WAIT_TIMEOUT) sh -c 'until [ -z "$$(docker exec mailserver_default find /var/spool/postfix/incoming /var/spool/postfix/active /var/spool/postfix/maildrop -type f 2>/dev/null)" ]; do sleep 2; done' \ + || { echo "TIMEOUT: postfix queue did not drain"; docker exec mailserver_default postqueue -p; exit 1; } run_default: ./test/bats/bin/bats test/default.bats stop_default: @@ -439,21 +468,26 @@ fixtures_sieve: # Wait for rspamd to start (sieve) docker exec mailserver_sieve /bin/sh -c "while ! echo PING | nc -z 0.0.0.0 11332 ; do sleep 1 ; done" - docker exec mailserver_sieve /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-existing-user.txt" - docker exec mailserver_sieve /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-existing-user-spam-learning.txt" - docker exec mailserver_sieve /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-valid-user-subaddress.txt" - docker exec mailserver_sieve /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-non-existing-user.txt" - docker exec mailserver_sieve /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-to-existing-alias.txt" - docker exec mailserver_sieve /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-spam-to-existing-user.txt" - docker exec mailserver_sieve /bin/sh -c "nc 0.0.0.0 25 < /tmp/tests/email-templates/external-virus-to-existing-user.txt" - docker exec mailserver_sieve /bin/sh -c "openssl s_client -ign_eof -connect 0.0.0.0:587 -starttls smtp < /tmp/tests/email-templates/internal-user-to-existing-user.txt" - docker exec mailserver_sieve /bin/sh -c "openssl s_client -ign_eof -connect 0.0.0.0:587 -starttls smtp < /tmp/tests/email-templates/internal-rejected-user-to-existing-user.txt" + docker exec mailserver_sieve /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-existing-user.txt" + docker exec mailserver_sieve /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-existing-user-spam-learning.txt" + docker exec mailserver_sieve /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-valid-user-subaddress.txt" + docker exec mailserver_sieve /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-non-existing-user.txt" + docker exec mailserver_sieve /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-to-existing-alias.txt" + docker exec mailserver_sieve /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-spam-to-existing-user.txt" + docker exec mailserver_sieve /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/email-templates/external-virus-to-existing-user.txt" + docker exec mailserver_sieve /bin/sh -c "python3 /tmp/tests/smtp-send.py --starttls 0.0.0.0 587 /tmp/tests/email-templates/internal-user-to-existing-user.txt" + docker exec mailserver_sieve /bin/sh -c "python3 /tmp/tests/smtp-send.py --starttls 0.0.0.0 587 /tmp/tests/email-templates/internal-rejected-user-to-existing-user.txt" - sleep 2 + # Wait until postfix has delivered everything it accepted + @echo "Waiting for the postfix queue to drain (mailserver_sieve) ..." + @timeout $(WAIT_TIMEOUT) sh -c 'until [ -z "$$(docker exec mailserver_sieve find /var/spool/postfix/incoming /var/spool/postfix/active /var/spool/postfix/maildrop -type f 2>/dev/null)" ]; do sleep 2; done' \ + || { echo "TIMEOUT: postfix queue did not drain"; docker exec mailserver_sieve postqueue -p; exit 1; } docker exec mailserver_sieve /bin/sh -c "openssl s_client -ign_eof -connect 0.0.0.0:993 < /tmp/tests/sieve/trigger-spam-ham-learning.txt" - # Wait until all mails have been processed - sleep 10 + # Wait until postfix has delivered everything it accepted + @echo "Waiting for the postfix queue to drain (mailserver_sieve) ..." + @timeout $(WAIT_TIMEOUT) sh -c 'until [ -z "$$(docker exec mailserver_sieve find /var/spool/postfix/incoming /var/spool/postfix/active /var/spool/postfix/maildrop -type f 2>/dev/null)" ]; do sleep 2; done' \ + || { echo "TIMEOUT: postfix queue did not drain"; docker exec mailserver_sieve postqueue -p; exit 1; } run_sieve: ./test/bats/bin/bats test/sieve.bats diff --git a/test/default.bats b/test/default.bats index 10d4c92c..9dd6f14c 100644 --- a/test/default.bats +++ b/test/default.bats @@ -273,42 +273,42 @@ load 'test_helper/bats-assert/load' # echo -ne 'badpassword' | openssl base64 @test "checking smtp (25): STARTTLS AUTH PLAIN works with good password (default configuration)" { - run docker exec mailserver_default /bin/sh -c "openssl s_client -ign_eof -connect 0.0.0.0:25 -starttls smtp < /tmp/tests/auth/smtp-auth-plain.txt 2>&1 | grep -i 'authentication successful'" + run docker exec mailserver_default /bin/sh -c "python3 /tmp/tests/smtp-send.py --starttls 0.0.0.0 25 /tmp/tests/auth/smtp-auth-plain.txt 2>&1 | grep -i 'authentication successful'" assert_success } @test "checking smtp (25): STARTTLS AUTH PLAIN fails with bad password" { - run docker exec mailserver_default /bin/sh -c "openssl s_client -ign_eof -connect 0.0.0.0:25 -starttls smtp < /tmp/tests/auth/smtp-auth-plain-wrong.txt 2>&1 | grep -i 'authentication failed'" + run docker exec mailserver_default /bin/sh -c "python3 /tmp/tests/smtp-send.py --starttls 0.0.0.0 25 /tmp/tests/auth/smtp-auth-plain-wrong.txt 2>&1 | grep -i 'authentication failed'" assert_success } @test "checking smtp (25): clear auth disabled" { - run docker exec mailserver_default /bin/sh -c "nc -w 2 0.0.0.0 25 < /tmp/tests/auth/smtp-auth-plain.txt | grep -i 'authentication not enabled'" + run docker exec mailserver_default /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 25 /tmp/tests/auth/smtp-auth-plain.txt | grep -i 'authentication not enabled'" assert_success } @test "checking submission (587): STARTTLS AUTH LOGIN works with good password (default configuration)" { - run docker exec mailserver_default /bin/sh -c "openssl s_client -ign_eof -connect 0.0.0.0:587 -starttls smtp < /tmp/tests/auth/smtp-auth-login.txt 2>&1 | grep -i 'authentication successful'" + run docker exec mailserver_default /bin/sh -c "python3 /tmp/tests/smtp-send.py --starttls 0.0.0.0 587 /tmp/tests/auth/smtp-auth-login.txt 2>&1 | grep -i 'authentication successful'" assert_success } @test "checking submission (587): STARTTLS AUTH LOGIN fails with bad password" { - run docker exec mailserver_default /bin/sh -c "openssl s_client -ign_eof -connect 0.0.0.0:587 -starttls smtp < /tmp/tests/auth/smtp-auth-login-wrong.txt 2>&1 | grep -i 'authentication failed'" + run docker exec mailserver_default /bin/sh -c "python3 /tmp/tests/smtp-send.py --starttls 0.0.0.0 587 /tmp/tests/auth/smtp-auth-login-wrong.txt 2>&1 | grep -i 'authentication failed'" assert_success } @test "checking submission (587): Auth without STARTTLS fail" { - run docker exec mailserver_default /bin/sh -c "nc -w 2 0.0.0.0 587 < /tmp/tests/auth/smtp-auth-plain.txt | grep -i 'Must issue a STARTTLS command first'" + run docker exec mailserver_default /bin/sh -c "python3 /tmp/tests/smtp-send.py 0.0.0.0 587 /tmp/tests/auth/smtp-auth-plain.txt | grep -i 'Must issue a STARTTLS command first'" assert_success } @test "checking smtps (465): SSL/TLS AUTH LOGIN works with good password (default configuration)" { - run docker exec mailserver_default /bin/sh -c "openssl s_client -ign_eof -connect 0.0.0.0:465 < /tmp/tests/auth/smtp-auth-login.txt 2>&1 | grep -i 'authentication successful'" + run docker exec mailserver_default /bin/sh -c "python3 /tmp/tests/smtp-send.py --tls 0.0.0.0 465 /tmp/tests/auth/smtp-auth-login.txt 2>&1 | grep -i 'authentication successful'" assert_success } @test "checking smtps (465): SSL/TLS AUTH LOGIN fails with bad password" { - run docker exec mailserver_default /bin/sh -c "openssl s_client -ign_eof -connect 0.0.0.0:465 < /tmp/tests/auth/smtp-auth-login-wrong.txt 2>&1 | grep -i 'authentication failed'" + run docker exec mailserver_default /bin/sh -c "python3 /tmp/tests/smtp-send.py --tls 0.0.0.0 465 /tmp/tests/auth/smtp-auth-login-wrong.txt 2>&1 | grep -i 'authentication failed'" assert_success } diff --git a/test/ldap.bats b/test/ldap.bats index 262904b6..89254892 100644 --- a/test/ldap.bats +++ b/test/ldap.bats @@ -245,12 +245,12 @@ load 'test_helper/bats-assert/load' # echo -ne 'badpassword' | openssl base64 @test "checking smtp (25): STARTTLS AUTH PLAIN works with good password (ldap configuration)" { - run docker exec mailserver_ldap /bin/sh -c "openssl s_client -ign_eof -connect 0.0.0.0:25 -starttls smtp < /tmp/tests/auth/smtp-auth-plain.txt 2>&1 | grep -i 'authentication successful'" + run docker exec mailserver_ldap /bin/sh -c "python3 /tmp/tests/smtp-send.py --starttls 0.0.0.0 25 /tmp/tests/auth/smtp-auth-plain.txt 2>&1 | grep -i 'authentication successful'" assert_success } @test "checking smtps (465): SSL/TLS AUTH LOGIN works with good password (ldap configuration)" { - run docker exec mailserver_ldap /bin/sh -c "openssl s_client -ign_eof -connect 0.0.0.0:465 < /tmp/tests/auth/smtp-auth-login.txt 2>&1 | grep -i 'authentication successful'" + run docker exec mailserver_ldap /bin/sh -c "python3 /tmp/tests/smtp-send.py --tls 0.0.0.0 465 /tmp/tests/auth/smtp-auth-login.txt 2>&1 | grep -i 'authentication successful'" assert_success } diff --git a/test/reverse.bats b/test/reverse.bats index e70b5b32..ec2fbd52 100644 --- a/test/reverse.bats +++ b/test/reverse.bats @@ -239,17 +239,17 @@ load 'test_helper/bats-assert/load' # echo -ne 'badpassword' | openssl base64 @test "checking smtp (25): STARTTLS AUTH PLAIN works with good password (reverse configuration)" { - run docker exec mailserver_reverse /bin/sh -c "openssl s_client -ign_eof -connect 0.0.0.0:25 -starttls smtp < /tmp/tests/auth/smtp-auth-plain.txt 2>&1 | grep -i 'authentication successful'" + run docker exec mailserver_reverse /bin/sh -c "python3 /tmp/tests/smtp-send.py --starttls 0.0.0.0 25 /tmp/tests/auth/smtp-auth-plain.txt 2>&1 | grep -i 'authentication successful'" assert_success } @test "checking submission (587): STARTTLS AUTH LOGIN works with good password (reverse configuration)" { - run docker exec mailserver_reverse /bin/sh -c "openssl s_client -ign_eof -connect 0.0.0.0:587 -starttls smtp < /tmp/tests/auth/smtp-auth-login.txt 2>&1 | grep -i 'authentication successful'" + run docker exec mailserver_reverse /bin/sh -c "python3 /tmp/tests/smtp-send.py --starttls 0.0.0.0 587 /tmp/tests/auth/smtp-auth-login.txt 2>&1 | grep -i 'authentication successful'" assert_success } @test "checking smtps (465): SSL/TLS AUTH LOGIN works with good password (reverse configuration)" { - run docker exec mailserver_reverse /bin/sh -c "openssl s_client -ign_eof -connect 0.0.0.0:465 < /tmp/tests/auth/smtp-auth-login.txt 2>&1 | grep -i 'authentication successful'" + run docker exec mailserver_reverse /bin/sh -c "python3 /tmp/tests/smtp-send.py --tls 0.0.0.0 465 /tmp/tests/auth/smtp-auth-login.txt 2>&1 | grep -i 'authentication successful'" assert_success } diff --git a/test/share/tests/smtp-send.py b/test/share/tests/smtp-send.py new file mode 100755 index 00000000..e48df5a6 --- /dev/null +++ b/test/share/tests/smtp-send.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Replay an SMTP dialogue from a file, one command at a time. + +The fixtures used to be piped straight into nc/openssl, which sends every +command without waiting for the server's replies. Postfix rejects that as +unauthorised pipelining (smtpd_forbid_unauth_pipelining, on by default since +postfix 3.9) with "554 5.5.0 Error: SMTP protocol synchronization", so no mail +was accepted at all. This reads each reply before sending the next command. + +The exit status reflects whether the dialogue could be started, not whether the +server accepted the mail: several fixtures deliberately test a rejection, and +postfix ends those dialogues by hanging up. +""" +import argparse +import socket +import ssl +import sys + +TIMEOUT = 30 +HANGUP = (BrokenPipeError, ConnectionResetError, ssl.SSLEOFError, ssl.SSLZeroReturnError) + + +class Hangup(Exception): + """The server ended the dialogue.""" + + +def log(prefix, text): + print(f"{prefix} {text}") + sys.stdout.flush() + + +class Dialogue: + def __init__(self, host, port): + self.sock = socket.create_connection((host, port), TIMEOUT) + self.sock.settimeout(TIMEOUT) + self.f = self.sock.makefile("rb") + self.host = host + + def read_reply(self): + """Read one SMTP reply, following multi-line continuations.""" + last = "" + while True: + try: + line = self.f.readline() + except HANGUP as exc: + raise Hangup(str(exc)) from exc + if not line: + raise Hangup("connection closed") + last = line.rstrip(b"\r\n").decode("utf-8", "replace") + log("S:", last) + # a space in the fourth column marks the final line of a reply + if len(last) < 4 or last[3] == " ": + return last + + def send(self, line, expect_reply=True): + log("C:", line) + try: + self.sock.sendall(line.encode() + b"\r\n") + except HANGUP as exc: + raise Hangup(str(exc)) from exc + return self.read_reply() if expect_reply else None + + def wrap_tls(self): + ctx = ssl.create_default_context() + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + self.sock = ctx.wrap_socket(self.sock, server_hostname=self.host) + self.sock.settimeout(TIMEOUT) + self.f = self.sock.makefile("rb") + + def starttls(self): + self.send("EHLO smtp-send") + self.send("STARTTLS") + self.wrap_tls() + + def replay(self, lines): + i, in_data = 0, False + while i < len(lines): + line = lines[i] + if in_data: + # inside DATA the server stays silent until the terminating dot + self.send(line, expect_reply=False) + if line == ".": + self.read_reply() + in_data = False + else: + reply = self.send(line) + if line.upper().startswith("DATA"): + if reply.startswith("354"): + in_data = True + else: + # DATA refused (a fixture testing a rejection): skip the + # body instead of sending it as commands, which would + # trip the server's error limit + while i + 1 < len(lines) and lines[i + 1] != ".": + i += 1 + i += 1 + i += 1 + + def close(self): + try: + self.sock.close() + except OSError: + pass + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("host") + ap.add_argument("port", type=int) + ap.add_argument("dialogue") + ap.add_argument("--starttls", action="store_true", + help="negotiate TLS with STARTTLS before the dialogue") + ap.add_argument("--tls", action="store_true", + help="wrap the connection in TLS immediately (smtps)") + args = ap.parse_args() + + with open(args.dialogue, "rb") as fh: + lines = [l.rstrip(b"\r\n").decode("utf-8", "replace") + for l in fh.read().split(b"\n")] + while lines and lines[-1] == "": + lines.pop() + + d = Dialogue(args.host, args.port) + try: + if args.tls: + d.wrap_tls() + d.read_reply() # 220 greeting + if args.starttls: + d.starttls() + d.replay(lines) + except Hangup as exc: + log("S:", f"") + finally: + d.close() + return 0 + + +if __name__ == "__main__": + sys.exit(main())