Fatal error when trying to communicate over TLS

Hello!

I am currently trying to set up communication over TLS (over Ethernet) between two NUCLEO-F439ZI microcontrollers. I currently have a curious bug that I can’t seem to diagnose. I can get the TLS handshake to complete but when I try to read/write over TLS using mbedtls_ssl_read() and mbedtls_ssl_write() I get a fatal error. The fatal error appears to occur when the client/server connect to each other, before the handshake, even though the read/write is after the handshake.

Interestingly, I also get a stack overflow error during the TLS handshake if I set the mbedtls debug level to 3 or higher – seemingly when they try to print out a certificate in the debug output.

I can’t really get my head around what could be causing this so any help/thoughts would be greatly appreciated. Server and client test programs are attached below (they are modified versions of the ssl_server.c and ssl_client1.c files included in Mbed TLS) with the error causing code commented out at the end of each file. If there’s anything else I can provide to help debug this please let me know.

(Currently programming in Mbed Studio with the GCC_ARM compiler and the Debug build profile.)

tls_server/main.c:

#include "mbed.h"
#include "EthernetInterface.h"
#include "mbedtls/x509.h"
#include "mbedtls/certs.h"
#if defined(MBEDTLS_SSL_CACHE_C)
#include "mbedtls/ssl_cache.h"
#endif

#include <cstdio>

TCPSocket *client_sock;  // srv.accept() will return pointer to socket
TCPSocket srv;

#define HTTP_RESPONSE \
    "HTTP/1.0 200 OK\r\nContent-Type: text/html\r\n\r\n" \
    "<h2>mbed TLS Test Server</h2>\r\n" \
    "<p>Successful connection using: %s</p>\r\n"

int sslSend(void *ctx, const unsigned char *buf, size_t len)
{
    int ret = client_sock->send(buf, len);

    if (ret == NSAPI_ERROR_WOULD_BLOCK)
        ret = MBEDTLS_ERR_SSL_WANT_WRITE;
    else if (ret < 0)
        mbedtls_printf("socket.send() returned %d\n", ret);

    return ret;
}

int sslRecv(void *ctx, unsigned char *buf, size_t len)
{
    int ret = client_sock->recv(buf, len);

    if (ret == NSAPI_ERROR_WOULD_BLOCK)
        ret = MBEDTLS_ERR_SSL_WANT_READ;
    else if (ret < 0)
        mbedtls_printf("socket.recv() returned %d\n", ret);

    return ret;
}

void sslDebug(void *ctx, int level, const char *file,
                                int line, const char *str)
{
    (void)ctx;

    const char *p, *basename;

    /* Extract basename from file */
    for (p = basename = file; *p != '\0'; p++) {
        if (*p == '/' || *p == '\\')
            basename = p + 1;
    }

    mbedtls_printf("%s:%d: |%d| %s\r", basename, line, level, str);
}


int main()
{
    printf("TLS server example\n");
    EthernetInterface eth;
    mbedtls_entropy_context entropy;
    mbedtls_ctr_drbg_context ctr_drbg;
    mbedtls_ssl_context ssl;
    mbedtls_ssl_config ssl_conf;
    mbedtls_x509_crt srvcert;
    mbedtls_pk_context pkey;
#if defined(MBEDTLS_SSL_CACHE_C)
    mbedtls_ssl_cache_context cache;
#endif

    int ret = 0;
    if((ret = mbedtls_platform_setup(NULL)) != 0) {
        mbedtls_printf("Platform initialization failed with error %d\r\n", ret);
        return 1;
    }

    const char *DRBG_PERSONALIZED_STR = "teststring";

    mbedtls_ssl_init(&ssl);
    mbedtls_ssl_config_init(&ssl_conf);
#if defined(MBEDTLS_SSL_CACHE_C)
    mbedtls_ssl_cache_init( &cache );
#endif
    mbedtls_x509_crt_init( &srvcert );
    mbedtls_pk_init( &pkey );
    mbedtls_entropy_init(&entropy);
    mbedtls_ctr_drbg_init(&ctr_drbg);

#if defined(MBEDTLS_DEBUG_C)
    mbedtls_debug_set_threshold( 2 );
#endif

    int len;
    unsigned char buf[1024];

    /*
    * 1. Load the certificates and private RSA key
    */
    mbedtls_printf( "\n  . Loading the server cert. and key..." );
    fflush( stdout );

    ret = mbedtls_x509_crt_parse( &srvcert, (const unsigned char *) mbedtls_test_srv_crt,
                        mbedtls_test_srv_crt_len );
    if( ret != 0 )
    {
        mbedtls_printf( " failed\n  !  mbedtls_x509_crt_parse returned %d\n\n", ret );
    }

    ret = mbedtls_x509_crt_parse( &srvcert, (const unsigned char *) mbedtls_test_cas_pem,
                        mbedtls_test_cas_pem_len );
    if( ret != 0 )
    {
        mbedtls_printf( " failed\n  !  mbedtls_x509_crt_parse returned %d\n\n", ret );
    }

    ret =  mbedtls_pk_parse_key( &pkey, (const unsigned char *) mbedtls_test_srv_key,
                        mbedtls_test_srv_key_len, NULL, 0 );
    if( ret != 0 )
    {
        mbedtls_printf( " failed\n  !  mbedtls_pk_parse_key returned %d\n\n", ret );
    }

    mbedtls_printf( " ok\n" );

    /*
    * 2. Setup the listening TCP socket
    */
    SocketAddress ip;
    ip.set_ip_address("10.10.0.1");
    SocketAddress mask; 
    mask.set_ip_address("255.255.255.0");
    SocketAddress gateway; 
    gateway.set_ip_address("10.10.0.0");
    eth.set_network(ip, mask, gateway);
    
    eth.connect();

    SocketAddress a;
    eth.get_ip_address(&a);
    printf("  . Bind on %s ...", a.get_ip_address() ? a.get_ip_address() : "None");

    /* Open the server on ethernet stack */
    srv.open(&eth);
    
    /* Bind the port (TCP 4433) to the server */
    srv.bind(4433);
    mbedtls_printf(" ok\n");

    /*
    * 3. Seed the RNG
    */
    mbedtls_printf("  . Seeding the random number generator...");
    fflush(stdout);
    ret = mbedtls_ctr_drbg_seed(&ctr_drbg, mbedtls_entropy_func, &entropy,
            (const unsigned char *)DRBG_PERSONALIZED_STR,
            strlen(DRBG_PERSONALIZED_STR) + 1);
    if (ret != 0) {
        mbedtls_printf("mbedtls_ctr_drbg_seed() returned -0x%04X\n", -ret);
        return ret;
    }
    mbedtls_printf(" ok\n");
    
    /*
    * 4. Setup stuff
    */
    mbedtls_printf("  . Setting up the SSL data...");
    fflush(stdout);

    ret = mbedtls_ssl_config_defaults(&ssl_conf, MBEDTLS_SSL_IS_SERVER,
                                    MBEDTLS_SSL_TRANSPORT_STREAM,
                                    MBEDTLS_SSL_PRESET_DEFAULT);
    if (ret != 0) {
        mbedtls_printf("mbedtls_ssl_config_defaults() returned -0x%04X\n",
                    -ret);
        return ret;
    }

    mbedtls_ssl_conf_rng(&ssl_conf, mbedtls_ctr_drbg_random, &ctr_drbg);
    mbedtls_ssl_conf_dbg(&ssl_conf, sslDebug, stdout);

#if defined(MBEDTLS_SSL_CACHE_C)
    mbedtls_ssl_conf_session_cache( &ssl_conf, &cache,
                                mbedtls_ssl_cache_get,
                                mbedtls_ssl_cache_set );
#endif

    mbedtls_ssl_conf_ca_chain( &ssl_conf, srvcert.next, NULL );
    if( ( ret = mbedtls_ssl_conf_own_cert( &ssl_conf, &srvcert, &pkey ) ) != 0 )
    {
        mbedtls_printf( " failed\n  ! mbedtls_ssl_conf_own_cert returned %d\n\n", ret );
    }

    if ((ret = mbedtls_ssl_setup( &ssl, &ssl_conf)) != 0) {
        mbedtls_printf("mbedtls_ssl_setup() returned -0x%04X\n", -ret);
        return ret;
    }

    mbedtls_printf(" ok\n");
    
    mbedtls_ssl_session_reset(&ssl);

    /*
    * 3. Wait until a client connects
    */
    mbedtls_printf("  . Waiting for a remote connection ...");
    fflush(stdout);
    srv.listen(1);

    client_sock = srv.accept();  //return pointer of a client socket

    mbedtls_ssl_set_bio(&ssl, static_cast<void *>(&client_sock), sslSend, sslRecv, NULL);
    mbedtls_printf(" ok\n");

    /*
    * 5. Handshake
    */
    mbedtls_printf("  . Performing the TLS handshake...");
    fflush(stdout);
    do {
        ret = mbedtls_ssl_handshake(&ssl);
    } while(ret != 0 &&
            (ret == MBEDTLS_ERR_SSL_WANT_READ ||
            ret == MBEDTLS_ERR_SSL_WANT_WRITE));
    if (ret < 0) {
        mbedtls_printf("failed \n mbedtls_ssl_handshake() returned -0x%04X\n", -ret);
        return ret;
    }

    mbedtls_printf(" ok\n");
    
    /*
    * 6. Read the HTTP Request
    */
    // mbedtls_printf("  < Read from client:");
    // fflush(stdout);

    // do
    // {
    //     len = sizeof(buf) - 1;
    //     memset(buf, 0, sizeof(buf));
    //     ret = mbedtls_ssl_read(&ssl, buf, len);

    //     if (ret == MBEDTLS_ERR_SSL_WANT_READ || ret == MBEDTLS_ERR_SSL_WANT_WRITE)
    //         continue;

    //     if (ret <= 0)
    //     {
    //         switch (ret)
    //         {
    //         case MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY:
    //             mbedtls_printf(" connection was closed gracefully\n");
    //             break;

    //         default:
    //             mbedtls_printf(" mbedtls_ssl_read returned -0x%x\n", (unsigned int)-ret);
    //             break;
    //         }

    //         break;
    //     }

    //     len = ret;
    //     mbedtls_printf(" %d bytes read\n\n%s", len, (char *)buf);

    //     if (ret > 0)
    //         break;
    // } while (1);

    /*
    * 7. Write the 200 Response
    */
    // mbedtls_printf("  > Write to client:");
    // fflush(stdout);

    // len = sprintf((char *)buf, HTTP_RESPONSE,
    //               mbedtls_ssl_get_ciphersuite(&ssl));

    // while ((ret = mbedtls_ssl_write(&ssl, buf, len)) <= 0)
    // {

    //     if (ret != MBEDTLS_ERR_SSL_WANT_READ && ret != MBEDTLS_ERR_SSL_WANT_WRITE)
    //     {
    //         mbedtls_printf(" failed\n  ! mbedtls_ssl_write returned %d\n\n", ret);
    //         goto exit;
    //     }
    // }

    // len = ret;
    // mbedtls_printf(" %d bytes written\n\n%s\n", len, (char *)buf);

    // mbedtls_printf("  . Closing the connection...");

    // while ((ret = mbedtls_ssl_close_notify(&ssl)) < 0)
    // {
    //     if (ret != MBEDTLS_ERR_SSL_WANT_READ &&
    //         ret != MBEDTLS_ERR_SSL_WANT_WRITE)
    //     {
    //         mbedtls_printf(" failed\n  ! mbedtls_ssl_close_notify returned %d\n\n", ret);
    //         goto reset;
    //     }
    // }

    // mbedtls_printf(" ok\n");
    ret = 0;

exit:
    // teardown
    mbedtls_x509_crt_free( &srvcert );
    mbedtls_pk_free( &pkey );
    mbedtls_entropy_free(&entropy);
    mbedtls_ctr_drbg_free(&ctr_drbg);
    mbedtls_ssl_free(&ssl);
    mbedtls_ssl_config_free(&ssl_conf);
    mbedtls_platform_teardown(NULL);
    client_sock->close();
    eth.disconnect();
    printf("Done\n");
    return ret;
}

tls_client/main.c:

#include "mbed.h"
#include "EthernetInterface.h"
#include "mbedtls/debug.h"
#include "mbedtls/certs.h"

#define GET_REQUEST "GET / HTTP/1.0\r\n\r\n"

const char *HTTP_REQUEST_FILE_PATH = "/media/uploads/mbed_official/hello.txt";

TCPSocket socket;

int sslSend(void *ctx, const unsigned char *buf, size_t len)
{
    int ret = socket.send(buf, len);

    if (ret == NSAPI_ERROR_WOULD_BLOCK)
        ret = MBEDTLS_ERR_SSL_WANT_WRITE;
    else if (ret < 0)
        mbedtls_printf("socket.send() returned %d\n", ret);

    return ret;
}

int sslRecv(void *ctx, unsigned char *buf, size_t len)
{
    int ret = socket.recv(buf, len);

    if (ret == NSAPI_ERROR_WOULD_BLOCK)
        ret = MBEDTLS_ERR_SSL_WANT_READ;
    else if (ret < 0)
        mbedtls_printf("socket.recv() returned %d\n", ret);

    return ret;
}

void sslDebug(void *ctx, int level, const char *file,
                                int line, const char *str)
{
    (void)ctx;

    const char *p, *basename;

    /* Extract basename from file */
    for (p = basename = file; *p != '\0'; p++) {
        if (*p == '/' || *p == '\\')
            basename = p + 1;
    }

    mbedtls_printf("%s:%d: |%d| %s\r", basename, line, level, str);
}

int main()
{
    printf("TLS client example\n");
    EthernetInterface eth;
    mbedtls_entropy_context entropy;
    mbedtls_ctr_drbg_context ctr_drbg;
    mbedtls_ssl_context ssl;
    mbedtls_ssl_config ssl_conf;
    mbedtls_x509_crt cacert;
    const char *DRBG_PERSONALIZED_STR = "teststring";

#if defined(MBEDTLS_DEBUG_C)
    mbedtls_debug_set_threshold( 2 );
#endif

    int ret = 0, len;
    unsigned char buf[1024];

    /*
    * 0. Initialize the RNG and the session data
    */
    mbedtls_ssl_init(&ssl);
    mbedtls_ssl_config_init(&ssl_conf);
    mbedtls_x509_crt_init( &cacert );
    mbedtls_ctr_drbg_init(&ctr_drbg);

    mbedtls_printf("\n  . Seeding the random number generator...");
    fflush(stdout);

    mbedtls_entropy_init(&entropy);
    ret = mbedtls_ctr_drbg_seed(&ctr_drbg, mbedtls_entropy_func, &entropy,
            (const unsigned char *)DRBG_PERSONALIZED_STR,
            strlen(DRBG_PERSONALIZED_STR) + 1);
    if (ret != 0) {
        mbedtls_printf("mbedtls_ctr_drbg_seed() returned -0x%04X\n", -ret);
        return ret;
    }

    mbedtls_printf(" ok\n");

    /*
    * 0. Initialize certificates
    */
    mbedtls_printf( "  . Loading the CA root certificate ..." );
    fflush( stdout );

    ret = mbedtls_x509_crt_parse( &cacert, (const unsigned char *) mbedtls_test_cas_pem,
                        mbedtls_test_cas_pem_len );
    if( ret < 0 )
    {
        mbedtls_printf( " failed\n  !  mbedtls_x509_crt_parse returned -0x%x\n\n", (unsigned int) -ret );
    }

    mbedtls_printf( " ok (%d skipped)\n", ret );

    /*
    * 1. Start the connection
    */
    SocketAddress ip;
    ip.set_ip_address("10.10.0.2");
    SocketAddress mask; 
    mask.set_ip_address("255.255.255.0");
    SocketAddress gateway; 
    gateway.set_ip_address("10.10.0.0");
    eth.set_network(ip, mask, gateway);

    eth.connect();

    socket.open(&eth);
    socket.set_blocking(false);

    SocketAddress a;
    a.set_ip_address("10.10.0.1");
    a.set_port(4433);
        printf("  . Connecting on %s:%d ...", a.get_ip_address(), a.get_port());
    fflush(stdout);
    if ((ret = socket.connect(a)) != NSAPI_ERROR_OK) {
        mbedtls_printf("socket.connect() returned %d\n", ret);
        return ret;
    }

    mbedtls_printf(" ok\n");

    /*
    * 2. Setup stuff
    */
    mbedtls_printf("  . Setting up the SSL/TLS structure...");
    fflush(stdout);

    ret = mbedtls_ssl_config_defaults(&ssl_conf, MBEDTLS_SSL_IS_CLIENT,
                                    MBEDTLS_SSL_TRANSPORT_STREAM,
                                    MBEDTLS_SSL_PRESET_DEFAULT);
    if (ret != 0) {
        mbedtls_printf("mbedtls_ssl_config_defaults() returned -0x%04X\n",
                    -ret);
        return ret;
    }
    mbedtls_ssl_conf_authmode( &ssl_conf, MBEDTLS_SSL_VERIFY_OPTIONAL );
    mbedtls_ssl_conf_ca_chain( &ssl_conf, &cacert, NULL );
    mbedtls_ssl_conf_rng(&ssl_conf, mbedtls_ctr_drbg_random, &ctr_drbg);
    mbedtls_ssl_conf_dbg(&ssl_conf, sslDebug, NULL);

    if ((ret = mbedtls_ssl_setup( &ssl, &ssl_conf)) != 0) {
        mbedtls_printf("mbedtls_ssl_setup() returned -0x%04X\n", -ret);
        return ret;
    }
    if( ( ret = mbedtls_ssl_set_hostname( &ssl, "10.10.0.1") ) != 0 )
    {
        mbedtls_printf( " failed\n  ! mbedtls_ssl_set_hostname returned %d\n\n", ret );
        goto exit;
    }

    mbedtls_ssl_set_bio(&ssl, &socket, sslSend, sslRecv, NULL);

    mbedtls_printf(" ok\n");

    /*
    * 4. Handshake
    */
    mbedtls_printf("  . Performing the TLS handshake...");
    fflush(stdout);
    do {
        ret = mbedtls_ssl_handshake(&ssl);
    } while(ret != 0 &&
            (ret == MBEDTLS_ERR_SSL_WANT_READ ||
            ret == MBEDTLS_ERR_SSL_WANT_WRITE));
    if (ret < 0) {
        mbedtls_printf("mbedtls_ssl_handshake() returned -0x%04X\n", -ret);
        return ret;
    }
    mbedtls_printf(" ok\n");


    /*
    * 6. Write the GET request
    */
    // mbedtls_printf("  > Write to server:");
    // fflush(stdout);

    // len = sprintf((char *)buf, GET_REQUEST);

    // while ((ret = mbedtls_ssl_write(&ssl, buf, len)) <= 0)
    // {
    //     if (ret != MBEDTLS_ERR_SSL_WANT_READ && ret != MBEDTLS_ERR_SSL_WANT_WRITE)
    //     {
    //         mbedtls_printf(" failed\n  ! mbedtls_ssl_write returned %d\n\n", ret);
    //         goto exit;
    //     }
    // }

    // len = ret;
    // mbedtls_printf(" %d bytes written\n\n%s", len, (char *)buf);

    /*
    * 7. Read the HTTP response
    */
    // mbedtls_printf("  < Read from server:");
    // fflush(stdout);

    // do
    // {
    //     len = sizeof(buf) - 1;
    //     memset(buf, 0, sizeof(buf));
    //     ret = mbedtls_ssl_read(&ssl, buf, len);

    //     if (ret == MBEDTLS_ERR_SSL_WANT_READ || ret == MBEDTLS_ERR_SSL_WANT_WRITE)
    //         continue;

    //     if (ret == MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY)
    //         break;

    //     if (ret < 0)
    //     {
    //         mbedtls_printf("failed\n  ! mbedtls_ssl_read returned %d\n\n", ret);
    //         break;
    //     }

    //     if (ret == 0)
    //     {
    //         mbedtls_printf("\n\nEOF\n\n");
    //         break;
    //     }

    //     len = ret;
    //     mbedtls_printf(" %d bytes read\n\n%s", len, (char *)buf);
    // } while (1);

    mbedtls_ssl_close_notify(&ssl);


exit:
    // teardown
    mbedtls_entropy_free(&entropy);
    mbedtls_ctr_drbg_free(&ctr_drbg);
    mbedtls_ssl_free(&ssl);
    mbedtls_ssl_config_free(&ssl_conf);
    // Close the socket to return its memory and bring down the network interface
    socket.close();
    // Bring down the ethernet interface
    eth.disconnect();
    printf("Done\n");
    return(ret);
}