[tarantool-patches] [PATCH 2/2] swim: implement and expose transport-level encryption

[Vladislav Shpilevoy <v.shpilevoy@tarantool.org>]

SWIM is going to be used in and between datacenters, which means,
that its packets will go through public networks. Therefore raw
SWIM packets are vulnerable to attacks.

An attacker can do any and all of the following things:

  1) Extract secret information from member payloads, like
     credentials to Tarantool binary ports;

  2) Change UUIDs and addresses in the packets and break a
     topology;

  3) Catch the packets and pretend being a Tarantool instance,
     which could lead to undefined behaviour depending on an
     application logic.

SWIM packets need a protection layer. This commit introduces it.
SWIM transport level allows to choose an encryption algorithm
with a private key to encrypt each packet with that key.

Besides, each packet is encrypted using a random public key
prepended to the packet.

SWIM now provides a public API to choose an encryption algorithm
and a private key.

Part of #3234
---
 src/lib/swim/CMakeLists.txt |   2 +-
 src/lib/swim/swim.c         |   6 ++
 src/lib/swim/swim.h         |  15 ++++
 src/lib/swim/swim_ev.h      |   2 +
 src/lib/swim/swim_io.c      | 133 +++++++++++++++++++++++++++++++-----
 src/lib/swim/swim_io.h      |  26 ++++++-
 src/lib/swim/swim_proto.h   |   5 ++
 test/unit/swim.c            |  47 ++++++++++++-
 test/unit/swim.result       |   9 ++-
 test/unit/swim_test_utils.c |  15 ++--
 test/unit/swim_test_utils.h |   8 +++
 11 files changed, 241 insertions(+), 27 deletions(-)

diff --git a/src/lib/swim/CMakeLists.txt b/src/lib/swim/CMakeLists.txt
index 8ba99d803..11202dce3 100644
--- a/src/lib/swim/CMakeLists.txt
+++ b/src/lib/swim/CMakeLists.txt
@@ -5,7 +5,7 @@ set(lib_swim_ev_sources swim_ev.c)
 set_source_files_compile_flags(${lib_swim_sources} ${lib_swim_udp_sources}
                                ${lib_swim_ev_sources})
 add_library(swim STATIC ${lib_swim_sources})
-target_link_libraries(swim core misc uuid)
+target_link_libraries(swim core misc uuid crypto)
 add_library(swim_udp STATIC ${lib_swim_udp_sources})
 target_link_libraries(swim_udp core)
 add_library(swim_ev STATIC ${lib_swim_ev_sources})
diff --git a/src/lib/swim/swim.c b/src/lib/swim/swim.c
index 746106657..68b87b120 100644
--- a/src/lib/swim/swim.c
+++ b/src/lib/swim/swim.c
@@ -1757,6 +1757,12 @@ swim_cfg(struct swim *swim, const char *uri, double heartbeat_rate,
 	return 0;
 }
 
+int
+swim_set_codec(struct swim *swim, enum crypto_algo algo, const char *key)
+{
+	return swim_scheduler_set_codec(&swim->scheduler, algo, key);
+}
+
 double
 swim_ack_timeout(const struct swim *swim)
 {
diff --git a/src/lib/swim/swim.h b/src/lib/swim/swim.h
index 6a219d131..5abbe8c33 100644
--- a/src/lib/swim/swim.h
+++ b/src/lib/swim/swim.h
@@ -32,6 +32,7 @@
  */
 #include <stdbool.h>
 #include <stdint.h>
+#include "crypto/crypto.h"
 #include "swim_constants.h"
 
 #if defined(__cplusplus)
@@ -108,6 +109,20 @@ swim_ack_timeout(const struct swim *swim);
 int
 swim_set_payload(struct swim *swim, const char *payload, uint16_t payload_size);
 
+/**
+ * Set SWIM codec to encrypt/decrypt messages.
+ * @param swim SWIM instance to set codec for.
+ * @param algo Cipher algorithm.
+ * @param key Private key of the chosen algorithm. It is used to
+ *        encrypt/decrypt messages, and should be the same on all
+ *        cluster nodes. Note that it can be changed, but it shall
+ *        be done on all cluster nodes. Otherwise the nodes will
+ *        not understand each other. There is also a public key
+ *        usually, but it is generated randomly inside SWIM.
+ */
+int
+swim_set_codec(struct swim *swim, enum crypto_algo algo, const char *key);
+
 /**
  * Stop listening and broadcasting messages, cleanup all internal
  * structures, free memory.
diff --git a/src/lib/swim/swim_ev.h b/src/lib/swim/swim_ev.h
index 8f41d43fe..34394ef47 100644
--- a/src/lib/swim/swim_ev.h
+++ b/src/lib/swim/swim_ev.h
@@ -61,4 +61,6 @@ swim_ev_timer_stop(struct ev_loop *loop, struct ev_timer *watcher);
 
 #define swim_ev_io_set ev_io_set
 
+#define swim_ev_set_cb ev_set_cb
+
 #endif /* TARANTOOL_SWIM_EV_H_INCLUDED */
diff --git a/src/lib/swim/swim_io.c b/src/lib/swim/swim_io.c
index 2c1448c1f..d49900a86 100644
--- a/src/lib/swim/swim_io.c
+++ b/src/lib/swim/swim_io.c
@@ -269,31 +269,19 @@ swim_scheduler_fd(const struct swim_scheduler *scheduler)
 	return scheduler->transport.fd;
 }
 
-/**
- * Dispatch a next output event. Build packet meta and send the
- * packet.
- */
-static void
-swim_scheduler_on_output(struct ev_loop *loop, struct ev_io *io, int events);
-
-/**
- * Dispatch a next input event. Unpack meta, forward a packet or
- * propagate further to protocol logic.
- */
-static void
-swim_scheduler_on_input(struct ev_loop *loop, struct ev_io *io, int events);
-
 void
 swim_scheduler_create(struct swim_scheduler *scheduler,
 		      swim_scheduler_on_input_f on_input)
 {
-	swim_ev_init(&scheduler->output, swim_scheduler_on_output);
 	scheduler->output.data = (void *) scheduler;
-	swim_ev_init(&scheduler->input, swim_scheduler_on_input);
 	scheduler->input.data = (void *) scheduler;
 	rlist_create(&scheduler->queue_output);
 	scheduler->on_input = on_input;
 	swim_transport_create(&scheduler->transport);
+	scheduler->codec = NULL;
+	int rc = swim_scheduler_set_codec(scheduler, CRYPTO_NONE, NULL);
+	assert(rc == 0);
+	(void) rc;
 }
 
 int
@@ -335,6 +323,38 @@ swim_scheduler_destroy(struct swim_scheduler *scheduler)
 	swim_scheduler_stop_input(scheduler);
 }
 
+/**
+ * Encrypt data and prepend it with a fresh crypto algorithm's
+ * initial vector.
+ */
+static inline int
+swim_encrypt(struct crypto_codec *c, const char *in, int in_size,
+	     char *out, int out_size)
+{
+	int iv_size = crypto_codec_iv_size(c);
+	const char *iv = crypto_codec_gen_iv(c);
+	assert(out_size >= iv_size);
+	memcpy(out, iv, iv_size);
+	out += iv_size;
+	out_size -= iv_size;
+	int rc = crypto_codec_encrypt(c, in, in_size, out, out_size);
+	if (rc < 0)
+		return -1;
+	return rc + iv_size;
+}
+
+/** Decrypt data prepended with an initial vector. */
+static inline int
+swim_decrypt(struct crypto_codec *c, const char *in, int in_size,
+	     char *out, int out_size)
+{
+	int iv_size = crypto_codec_iv_size(c);
+	const char *iv = in;
+	in += iv_size;
+	in_size -= iv_size;
+	return crypto_codec_decrypt(c, iv, in, in_size, out, out_size);
+}
+
 /**
  * Begin packet transmission. Prepare a next task in the queue to
  * send its packet: build a meta header, pop the task from the
@@ -408,8 +428,34 @@ swim_complete_send(struct swim_scheduler *scheduler, struct swim_task *task,
 		task->complete(task, scheduler, size);
 }
 
+/**
+ * On a new EV_WRITE event send a next packet from the queue
+ * encrypted with the currently chosen algorithm.
+ */
+static void
+swim_on_encrypted_output(struct ev_loop *loop, struct ev_io *io, int events)
+{
+	struct swim_scheduler *scheduler = (struct swim_scheduler *) io->data;
+	const struct sockaddr_in *dst;
+	struct swim_task *task = swim_begin_send(scheduler, loop, io, events,
+						 &dst);
+	if (task == NULL)
+		return;
+	char *buf = static_alloc(UDP_PACKET_SIZE);
+	assert(buf != NULL);
+	ssize_t size = swim_encrypt(scheduler->codec, task->packet.buf,
+				    task->packet.pos - task->packet.buf,
+				    buf, UDP_PACKET_SIZE);
+	if (size > 0)
+		size = swim_do_send(scheduler, buf, size, dst);
+	swim_complete_send(scheduler, task, size);
+}
+
+/**
+ * On a new EV_WRITE event send a next packet without encryption.
+ */
 static void
-swim_scheduler_on_output(struct ev_loop *loop, struct ev_io *io, int events)
+swim_on_plain_output(struct ev_loop *loop, struct ev_io *io, int events)
 {
 	struct swim_scheduler *scheduler = (struct swim_scheduler *) io->data;
 	const struct sockaddr_in *dst;
@@ -512,8 +558,35 @@ error:
 	diag_log();
 }
 
+/**
+ * On a new EV_READ event receive an encrypted packet from the
+ * network.
+ */
 static void
-swim_scheduler_on_input(struct ev_loop *loop, struct ev_io *io, int events)
+swim_on_encrypted_input(struct ev_loop *loop, struct ev_io *io, int events)
+{
+	struct swim_scheduler *scheduler = (struct swim_scheduler *) io->data;
+	/*
+	 * Buffer for decrypted data is on stack, not on static
+	 * memory, because the SWIM code uses static memory as
+	 * well and can accidentally rewrite the packet data.
+	 */
+	char buf[UDP_PACKET_SIZE];
+	swim_begin_recv(scheduler, loop, io, events);
+
+	char *ibuf = static_alloc(UDP_PACKET_SIZE);
+	assert(ibuf != NULL);
+	ssize_t size = swim_do_recv(scheduler, ibuf, UDP_PACKET_SIZE);
+	if (size > 0) {
+		size = swim_decrypt(scheduler->codec, ibuf, size,
+				    buf, UDP_PACKET_SIZE);
+	}
+	swim_complete_recv(scheduler, buf, size);
+}
+
+/** On a new EV_READ event receive a packet from the network. */
+static void
+swim_on_plain_input(struct ev_loop *loop, struct ev_io *io, int events)
 {
 	struct swim_scheduler *scheduler = (struct swim_scheduler *) io->data;
 	char buf[UDP_PACKET_SIZE];
@@ -522,6 +595,30 @@ swim_scheduler_on_input(struct ev_loop *loop, struct ev_io *io, int events)
 	swim_complete_recv(scheduler, buf, size);
 }
 
+int
+swim_scheduler_set_codec(struct swim_scheduler *scheduler,
+			 enum crypto_algo algo, const char *key)
+{
+	if (algo == CRYPTO_NONE) {
+		if (scheduler->codec != NULL) {
+			crypto_codec_delete(scheduler->codec);
+			scheduler->codec = NULL;
+		}
+		swim_ev_set_cb(&scheduler->output, swim_on_plain_output);
+		swim_ev_set_cb(&scheduler->input, swim_on_plain_input);
+		return 0;
+	}
+	struct crypto_codec *newc = crypto_codec_new(algo, key);
+	if (newc == NULL)
+		return -1;
+	if (scheduler->codec != NULL)
+		crypto_codec_delete(scheduler->codec);
+	scheduler->codec = newc;
+	swim_ev_set_cb(&scheduler->output, swim_on_encrypted_output);
+	swim_ev_set_cb(&scheduler->input, swim_on_encrypted_input);
+	return 0;
+}
+
 const char *
 swim_inaddr_str(const struct sockaddr_in *addr)
 {
diff --git a/src/lib/swim/swim_io.h b/src/lib/swim/swim_io.h
index 42d94fc92..397ba4a82 100644
--- a/src/lib/swim/swim_io.h
+++ b/src/lib/swim/swim_io.h
@@ -33,6 +33,7 @@
 #include "tt_static.h"
 #include "small/rlist.h"
 #include "salad/stailq.h"
+#include "crypto/crypto.h"
 #include "swim_transport.h"
 #include "tarantool_ev.h"
 #include "uuid/tt_uuid.h"
@@ -57,13 +58,24 @@ enum {
 	 * configuration.
 	 */
 	UDP_PACKET_SIZE = 1472,
+	/**
+	 * Data can be encrypted, which usually makes it slightly
+	 * bigger in size. Also, to decode data the receiver needs
+	 * two keys: private key and public initial vector. Public
+	 * initial vector is generated randomly for each packet
+	 * and prepends the data. This is why maximal data size is
+	 * reduced by one block and IV sizes.
+	 */
+	MAX_PACKET_SIZE = UDP_PACKET_SIZE - CRYPTO_AES_BLOCK_SIZE -
+			  CRYPTO_AES_IV_SIZE,
+
 };
 
 /**
  * UDP packet. Works as an allocator, allowing to fill its body
  * gradually, while preserving prefix for metadata.
  *
- *          < - - - -UDP_PACKET_SIZE- - - - ->
+ *          < - - - -MAX_PACKET_SIZE- - - - ->
  *          +--------+-----------------------+
  *          |  meta  |    body    |  *free*  |
  *          +--------+-----------------------+
@@ -86,7 +98,7 @@ struct swim_packet {
 	 */
 	char meta[0];
 	/** Packet body buffer. */
-	char buf[UDP_PACKET_SIZE];
+	char buf[MAX_PACKET_SIZE];
 	/**
 	 * Pointer to the end of the buffer. Just sugar to do not
 	 * write 'buf + sizeof(buf)' each time.
@@ -147,6 +159,11 @@ typedef void (*swim_scheduler_on_input_f)(struct swim_scheduler *scheduler,
 struct swim_scheduler {
 	/** Transport to send/receive packets. */
 	struct swim_transport transport;
+	/**
+	 * Codec to encode messages before sending, and decode
+	 * before lifting up to the SWIM core logic.
+	 */
+	struct crypto_codec *codec;
 	/**
 	 * Function called when a packet is received. It takes
 	 * packet body, while meta is handled by transport level
@@ -180,6 +197,11 @@ int
 swim_scheduler_bind(struct swim_scheduler *scheduler,
 		    const struct sockaddr_in *addr);
 
+/** Set a new codec to encrypt/decrypt messages. */
+int
+swim_scheduler_set_codec(struct swim_scheduler *scheduler,
+			 enum crypto_algo algo, const char *key);
+
 /** Stop accepting new packets from the network. */
 void
 swim_scheduler_stop_input(struct swim_scheduler *scheduler);
diff --git a/src/lib/swim/swim_proto.h b/src/lib/swim/swim_proto.h
index beb9bb1fe..482d79fb1 100644
--- a/src/lib/swim/swim_proto.h
+++ b/src/lib/swim/swim_proto.h
@@ -47,6 +47,11 @@ enum {
  * SWIM binary protocol structures and helpers. Below is a picture
  * of a SWIM message template:
  *
+ * +-----------------Public data, not encrypted------------------+
+ * |                                                             |
+ * |      Initial vector, size depends on chosen algorithm.      |
+ * |                   Next data is encrypted.                   |
+ * |                                                             |
  * +----------Meta section, handled by transport level-----------+
  * | {                                                           |
  * |     SWIM_META_TARANTOOL_VERSION: uint, Tarantool version ID,|
diff --git a/test/unit/swim.c b/test/unit/swim.c
index 2deaf138a..5aaae089d 100644
--- a/test/unit/swim.c
+++ b/test/unit/swim.c
@@ -31,6 +31,7 @@
 
 #include "memory.h"
 #include "fiber.h"
+#include "random.h"
 #include "uuid/tt_uuid.h"
 #include "unit.h"
 #include "uri/uri.h"
@@ -870,10 +871,51 @@ swim_test_indirect_ping(void)
 	swim_finish_test();
 }
 
+static void
+swim_test_encryption(void)
+{
+	swim_start_test(3);
+	struct swim_cluster *cluster = swim_cluster_new(2);
+	const char *key = "1234567812345678";
+	swim_cluster_set_codec(cluster, CRYPTO_AES128, key);
+	swim_cluster_add_link(cluster, 0, 1);
+
+	is(swim_cluster_wait_fullmesh(cluster, 2), 0,
+	   "cluster works with encryption");
+	swim_cluster_delete(cluster);
+	/*
+	 * Test that the instances can not interact with different
+	 * encryption keys.
+	 */
+	cluster = swim_cluster_new(2);
+	struct swim *s1 = swim_cluster_member(cluster, 0);
+	int rc = swim_set_codec(s1, CRYPTO_AES128, key);
+	fail_if(rc != 0);
+	struct swim *s2 = swim_cluster_member(cluster, 1);
+	key = "8765432187654321";
+	rc = swim_set_codec(s2, CRYPTO_AES128, key);
+	fail_if(rc != 0);
+	swim_cluster_add_link(cluster, 0, 1);
+	swim_run_for(2);
+	ok(! swim_cluster_is_fullmesh(cluster),
+	   "different encryption keys - can't interact");
+
+	rc = swim_set_codec(s1, CRYPTO_NONE, NULL);
+	fail_if(rc != 0);
+	rc = swim_set_codec(s2, CRYPTO_NONE, NULL);
+	fail_if(rc != 0);
+	is(swim_cluster_wait_fullmesh(cluster, 2), 0,
+	   "cluster works after encryption has been disabled");
+
+	swim_cluster_delete(cluster);
+
+	swim_finish_test();
+}
+
 static int
 main_f(va_list ap)
 {
-	swim_start_test(18);
+	swim_start_test(19);
 
 	(void) ap;
 	swim_test_ev_init();
@@ -897,6 +939,7 @@ main_f(va_list ap)
 	swim_test_payload_basic();
 	swim_test_payload_refutation();
 	swim_test_indirect_ping();
+	swim_test_encryption();
 
 	swim_test_transport_free();
 	swim_test_ev_free();
@@ -909,6 +952,7 @@ main_f(va_list ap)
 int
 main()
 {
+	random_init();
 	time_t seed = time(NULL);
 	srand(seed);
 	memory_init();
@@ -933,6 +977,7 @@ main()
 	say_logger_free();
 	fiber_free();
 	memory_free();
+	random_free();
 
 	return test_result;
 }
\ No newline at end of file
diff --git a/test/unit/swim.result b/test/unit/swim.result
index a450d8427..6e439541a 100644
--- a/test/unit/swim.result
+++ b/test/unit/swim.result
@@ -1,5 +1,5 @@
 	*** main_f ***
-1..18
+1..19
 	*** swim_test_one_link ***
     1..6
     ok 1 - no rounds - no fullmesh
@@ -185,4 +185,11 @@ ok 17 - subtests
     ok 2 - as well as S2 - they communicated via S3
 ok 18 - subtests
 	*** swim_test_indirect_ping: done ***
+	*** swim_test_encryption ***
+    1..3
+    ok 1 - cluster works with encryption
+    ok 2 - different encryption keys - can't interact
+    ok 3 - cluster works after encryption has been disabled
+ok 19 - subtests
+	*** swim_test_encryption: done ***
 	*** main_f: done ***
diff --git a/test/unit/swim_test_utils.c b/test/unit/swim_test_utils.c
index de1bef6e7..77e6e8b7a 100644
--- a/test/unit/swim_test_utils.c
+++ b/test/unit/swim_test_utils.c
@@ -227,9 +227,9 @@ swim_cluster_new(int size)
 	return res;
 }
 
-#define swim_cluster_set_cfg(cluster, ...) ({				\
+#define swim_cluster_set_cfg(cluster, func, ...) ({				\
 	for (int i = 0; i < cluster->size; ++i) {			\
-		int rc = swim_cfg(cluster->node[i].swim, __VA_ARGS__);	\
+		int rc = func(cluster->node[i].swim, __VA_ARGS__);	\
 		assert(rc == 0);					\
 		(void) rc;						\
 	}								\
@@ -238,14 +238,21 @@ swim_cluster_new(int size)
 void
 swim_cluster_set_ack_timeout(struct swim_cluster *cluster, double ack_timeout)
 {
-	swim_cluster_set_cfg(cluster, NULL, -1, ack_timeout, -1, NULL);
+	swim_cluster_set_cfg(cluster, swim_cfg, NULL, -1, ack_timeout, -1, NULL);
 	cluster->ack_timeout = ack_timeout;
 }
 
+void
+swim_cluster_set_codec(struct swim_cluster *cluster, enum crypto_algo algo,
+		       const char *key)
+{
+	swim_cluster_set_cfg(cluster, swim_set_codec, algo, key);
+}
+
 void
 swim_cluster_set_gc(struct swim_cluster *cluster, enum swim_gc_mode gc_mode)
 {
-	swim_cluster_set_cfg(cluster, NULL, -1, -1, gc_mode, NULL);
+	swim_cluster_set_cfg(cluster, swim_cfg, NULL, -1, -1, gc_mode, NULL);
 	cluster->gc_mode = gc_mode;
 }
 
diff --git a/test/unit/swim_test_utils.h b/test/unit/swim_test_utils.h
index c78894820..531b5e92b 100644
--- a/test/unit/swim_test_utils.h
+++ b/test/unit/swim_test_utils.h
@@ -48,6 +48,14 @@ swim_cluster_new(int size);
 void
 swim_cluster_set_ack_timeout(struct swim_cluster *cluster, double ack_timeout);
 
+/**
+ * Set an encryption algorithm and a key for each instance in
+ * @a cluster.
+ */
+void
+swim_cluster_set_codec(struct swim_cluster *cluster, enum crypto_algo algo,
+		       const char *key);
+
 /**
  * Change number of unacknowledged pings to delete a dead member
  * of all the instances in the cluster.
-- 
2.20.1 (Apple Git-117)