Implemented new OpenSSL BIO filter to fix fragmentation issue in DTLS…

… on large certificates (see meetecho#252)

Makefile.am

@@ -77,6 +77,8 @@ janus_SOURCES = \
 	debug.h \
 	dtls.c \
 	dtls.h \
+	dtls-bio.c \
+	dtls-bio.h \
 	ice.c \
 	ice.h \
 	janus.c \

dtls-bio.c

@@ -0,0 +1,129 @@
+/*! \file    dtls-bio.h
+ * \author   Lorenzo Miniero <[email protected]>
+ * \copyright GNU General Public License v3
+ * \brief    OpenSSL BIO filter for fragmentation (headers)
+ * \details  Implementation of an OpenSSL BIO filter to fix the broken
+ * behaviour of fragmented packets when using mem BIOs (as we do in
+ * Janus). See https://mta.openssl.org/pipermail/openssl-users/2015-June/001503.html
+ * and https://github.com/meetecho/janus-gateway/issues/252 for more details. 
+ * 
+ * \ingroup protocols
+ * \ref protocols
+ */
+
+#include <glib.h>
+
+#include "dtls-bio.h"
+#include "debug.h"
+#include "mutex.h"
+
+
+/* We keep the MTU lower thatn 1472 just to stay on the safe side 
+ * NOTE: should we make this configurable in janus.cfg? */
+static int mtu = 1200;
+
+
+/* Filter implementation */
+static BIO_METHOD janus_dtls_bio_filter_methods = {
+	BIO_TYPE_FILTER,
+	"janus filter",
+	janus_dtls_bio_filter_write,
+	NULL,
+	NULL,
+	NULL,
+	janus_dtls_bio_filter_ctrl,
+	janus_dtls_bio_filter_new,
+	janus_dtls_bio_filter_free,
+	NULL
+};
+BIO_METHOD *BIO_janus_dtls_filter(void) {
+	return(&janus_dtls_bio_filter_methods);
+}
+
+
+/* Helper struct to keep the filter state */
+typedef struct janus_dtls_bio_filter {
+	GList *packets;
+	janus_mutex mutex;
+} janus_dtls_bio_filter;
+
+
+int janus_dtls_bio_filter_new(BIO *bio) {
+	/* Create a filter state struct */
+	janus_dtls_bio_filter *filter = g_malloc0(sizeof(janus_dtls_bio_filter));
+	filter->packets = NULL;
+	janus_mutex_init(&filter->mutex);
+
+	/* Set the BIO as initialized */
+	bio->init = 1;
+	bio->ptr = filter;
+	bio->flags = 0;
+
+	return 1;
+}
+
+int janus_dtls_bio_filter_free(BIO *bio) {
+	if(bio == NULL)
+		return 0;
+
+	/* Get rid of the filter state */
+	janus_dtls_bio_filter *filter = (janus_dtls_bio_filter *)bio->ptr;
+	if(filter != NULL) {
+		g_list_free(filter->packets);
+		filter->packets = NULL;
+		g_free(filter);
+	}
+	bio->ptr = NULL;
+	bio->init = 0;
+	bio->flags = 0;
+	return 1;
+}
+
+int janus_dtls_bio_filter_write(BIO *bio, const char *in, int inl) {
+	JANUS_LOG(LOG_HUGE, "janus_dtls_bio_filter_write: %p, %d\n", in, inl);
+	/* Forward data to the write BIO */
+	long ret = BIO_write(bio->next_bio, in, inl);
+	JANUS_LOG(LOG_HUGE, "  -- %ld\n", ret);
+
+	/* Keep track of the packet, as we'll advertize them one by one after a pending check */
+	janus_dtls_bio_filter *filter = (janus_dtls_bio_filter *)bio->ptr;
+	if(filter != NULL) {
+		janus_mutex_lock(&filter->mutex);
+		filter->packets = g_list_append(filter->packets, GINT_TO_POINTER(ret));
+		janus_mutex_unlock(&filter->mutex);
+		JANUS_LOG(LOG_HUGE, "New list length: %d\n", g_list_length(filter->packets));
+	}
+	return ret;
+}
+
+long janus_dtls_bio_filter_ctrl(BIO *bio, int cmd, long num, void *ptr) {
+	switch(cmd) {
+		case BIO_CTRL_FLUSH:
+			/* The OpenSSL library needs this */
+			return 1;
+		case BIO_CTRL_DGRAM_QUERY_MTU:
+			/* Let's force a 1200 MTU */
+			JANUS_LOG(LOG_HUGE, "Advertizing MTU: %d\n", mtu);
+			return mtu;
+		case BIO_CTRL_WPENDING:
+			return 0L;
+		case BIO_CTRL_PENDING: {
+			/* We only advertize one packet at a time, as they may be fragmented */
+			janus_dtls_bio_filter *filter = (janus_dtls_bio_filter *)bio->ptr;
+			if(filter == NULL || g_list_length(filter->packets) == 0)
+				return 0;
+			janus_mutex_lock(&filter->mutex);
+			/* Get the first packet that hasn't been read yet */
+			GList *first = g_list_first(filter->packets);
+			filter->packets = g_list_remove_link(filter->packets, first);
+			int pending = GPOINTER_TO_INT(first->data);
+			g_list_free(first);
+			janus_mutex_unlock(&filter->mutex);
+			/* We return its size so that only part of the buffer is read from the write BIO */
+			return pending;
+		}
+		default:
+			JANUS_LOG(LOG_HUGE, "janus_dtls_bio_filter_ctrl: %d\n", cmd);
+	}
+	return 0;
+}

dtls-bio.h

@@ -0,0 +1,28 @@
+/*! \file    dtls-bio.h
+ * \author   Lorenzo Miniero <[email protected]>
+ * \copyright GNU General Public License v3
+ * \brief    OpenSSL BIO filter for fragmentation (headers)
+ * \details  Implementation of an OpenSSL BIO filter to fix the broken
+ * behaviour of fragmented packets when using mem BIOs (as we do in
+ * Janus). See https://mta.openssl.org/pipermail/openssl-users/2015-June/001503.html
+ * and https://github.com/meetecho/janus-gateway/issues/252 for more details. 
+ * 
+ * \ingroup protocols
+ * \ref protocols
+ */
+
+#ifndef _JANUS_DTLS_BIO_H
+#define _JANUS_DTLS_BIO_H
+
+#include <openssl/err.h>
+#include <openssl/ssl.h>
+
+int janus_dtls_bio_filter_write(BIO *h, const char *buf,int num);
+long janus_dtls_bio_filter_ctrl(BIO *h, int cmd, long arg1, void *arg2);
+int janus_dtls_bio_filter_new(BIO *h);
+int janus_dtls_bio_filter_free(BIO *data);
+
+/*! \brief OpenSSL BIO filter for fragmentation constructor */
+BIO_METHOD *BIO_janus_dtls_filter(void);
+
+#endif

dtls.c

@@ -224,7 +224,17 @@ janus_dtls_srtp *janus_dtls_srtp_create(void *ice_component, janus_dtls_role rol
 		return NULL;
 	}
 	BIO_set_mem_eof_return(dtls->write_bio, -1);
-	SSL_set_bio(dtls->ssl, dtls->read_bio, dtls->write_bio);
+	/* The write BIO needs our custom filter, or fragmentation won't work */
+	dtls->filter_bio = BIO_new(BIO_janus_dtls_filter());
+	if(!dtls->filter_bio) {
+		JANUS_LOG(LOG_ERR, "[%"SCNu64"]   Error creating filter BIO!\n", handle->handle_id);
+		janus_dtls_srtp_destroy(dtls);
+		return NULL;
+	}
+	/* Chain filter and write BIOs */
+	BIO_push(dtls->filter_bio, dtls->write_bio);
+	/* Set the filter as the BIO to use for outgoing data */
+	SSL_set_bio(dtls->ssl, dtls->read_bio, dtls->filter_bio);
 	dtls->dtls_role = role;
 	if(dtls->dtls_role == JANUS_DTLS_ROLE_CLIENT) {
 		JANUS_LOG(LOG_VERB, "[%"SCNu64"]   Setting connect state (DTLS client)\n", handle->handle_id);
@@ -503,6 +513,7 @@ void janus_dtls_srtp_destroy(janus_dtls_srtp *dtls) {
 	/* BIOs are destroyed by SSL_free */
 	dtls->read_bio = NULL;
 	dtls->write_bio = NULL;
+	dtls->filter_bio = NULL;
 	if(dtls->srtp_valid) {
 		if(dtls->srtp_in) {
 			srtp_dealloc(dtls->srtp_in);
@@ -592,9 +603,8 @@ void janus_dtls_fd_bridge(janus_dtls_srtp *dtls) {
 		JANUS_LOG(LOG_ERR, "No handle/agent/bio, no DTLS bridge...\n");
 		return;
 	}
-	int pending = BIO_ctrl_pending(dtls->write_bio);
-	JANUS_LOG(LOG_HUGE, "[%"SCNu64"] DTLS check pending: %d\n", handle->handle_id, pending);
-	if (pending > 0) {
+	int pending = BIO_ctrl_pending(dtls->filter_bio);
+	while(pending > 0) {
 		JANUS_LOG(LOG_HUGE, "[%"SCNu64"] >> Going to send DTLS data: %d bytes\n", handle->handle_id, pending);
 		char outgoing[pending];
 		int out = BIO_read(dtls->write_bio, outgoing, sizeof(outgoing));
@@ -614,6 +624,8 @@ void janus_dtls_fd_bridge(janus_dtls_srtp *dtls) {
 		if(bytes > 0) {
 			component->out_stats.data_bytes += bytes;
 		}
+		/* Check if there's anything left to send (e.g., fragmented packets) */
+		pending = BIO_ctrl_pending(dtls->filter_bio);
 	}
 }
 

dtls.h

@@ -17,11 +17,10 @@
 
 #include <inttypes.h>
 #include <glib.h>
-#include <openssl/err.h>
-#include <openssl/ssl.h>
 #include <srtp/srtp.h>
 
 #include "sctp.h"
+#include "dtls-bio.h"
 
 /*! \brief DTLS stuff initialization
  * @param[in] server_pem Path to the certificate to use
@@ -63,6 +62,8 @@ typedef struct janus_dtls_srtp {
 	BIO *read_bio;
 	/*! \brief Write BIO (outgoing DTLS data) */
 	BIO *write_bio;
+	/*! \brief Filter BIO (fix MTU fragmentation on outgoing DTLS data, if required) */
+	BIO *filter_bio;
 	/*! \brief Whether SRTP has been correctly set up for this component or not */
 	gint srtp_valid;
 	/*! \brief libsrtp context for incoming SRTP packets */