[tarantool-patches] [PATCH 0/2] swim crypto

[tarantool-patches] [PATCH 0/2] swim crypto

[Vladislav Shpilevoy]

This patchset is a full duplicate of the previously sent version to help Kostja
to review it.

Note, that these two commits are going to be kept even if lib/crypto will be
totally refactored after Georgy review. Probably a third one will be added to
support arbitrary encryption algorithms and make packet size dynamic.

Branch: http://github.com/tarantool/tarantool/tree/gerold103/gh-3234-swim-crypto
Issue: https://github.com/tarantool/tarantool/issues/3234

Vladislav Shpilevoy (2):
  swim: split send/recv into phases
  swim: implement and expose transport-level encryption

 src/lib/swim/CMakeLists.txt |   2 +-
 src/lib/swim/swim.c         |  12 ++
 src/lib/swim/swim.h         |  15 +++
 src/lib/swim/swim_ev.h      |   2 +
 src/lib/swim/swim_io.c      | 258 ++++++++++++++++++++++++++++++------
 src/lib/swim/swim_io.h      |  26 +++-
 src/lib/swim/swim_proto.h   |   5 +
 test/unit/swim.c            |  47 ++++++-
 test/unit/swim.result       |   9 +-
 test/unit/swim_test_utils.c |  15 ++-
 test/unit/swim_test_utils.h |   8 ++
 11 files changed, 349 insertions(+), 50 deletions(-)

-- 
2.20.1 (Apple Git-117)

[tarantool-patches] [PATCH 1/2] swim: split send/recv into phases

[Vladislav Shpilevoy]

At this moment swim_scheduler_on_output() is a relatively simple
function. It takes a task, builds its meta and flushes a result
into the network. But soon SWIM will be able to encrypt messages.

It means, that in addition to regular preprocessing like building
meta headers a new phase will appear - encryption. What is more -
conditional encryption, because a user may want to do not encrypt
messages.

All the same is about swim_scheduler_on_input() - if a SWIM
instance uses encryption, it should decrypt incoming messages
before forwarding them into the SWIM code logic.

The chosen strategy - lets reuse on_output/on_input virtuality
and create two version of on_input/on_output functions:

    swim_on_plain_input()  | swim_on_encrypted_input()
    swim_on_plain_output() | swim_on_encrypted_output()

One of these pairs will be chosen depending on if the instance
uses encryption.

To make these 4 functions as simple and short as possible this
commit creates two sets of functions, doing all the logic except
encryption:

    swim_begin_send()
    swim_do_send()
    swim_complete_send()

    swim_begin_recv()
    swim_do_recv()
    swim_complete_recv()

These functions will be used by on_input/on_output functions with
different arguments.

Part of #3234
---
 src/lib/swim/swim_io.c | 129 +++++++++++++++++++++++++++++++++--------
 1 file changed, 104 insertions(+), 25 deletions(-)

diff --git a/src/lib/swim/swim_io.c b/src/lib/swim/swim_io.c
index 107f47140..5bfa138df 100644
--- a/src/lib/swim/swim_io.c
+++ b/src/lib/swim/swim_io.c
@@ -338,66 +338,135 @@ swim_scheduler_destroy(struct swim_scheduler *scheduler)
 	swim_scheduler_stop_input(scheduler);
 }
 
-static void
-swim_scheduler_on_output(struct ev_loop *loop, struct ev_io *io, int events)
+/**
+ * Begin packet transmission. Prepare a next task in the queue to
+ * send its packet: build a meta header, pop the task from the
+ * queue.
+ * @param scheduler Scheduler to pop a task from.
+ * @param loop Event loop passed by libev.
+ * @param io Descriptor to send to.
+ * @param events Mask of happened events passed by libev.
+ * @param[out] dst Destination address to send the packet to. Can
+ *        be different from task.dst, for example, if task.proxy
+ *        is specified.
+ *
+ * @retval NULL The queue is empty. Input has been stopped.
+ * @retval not NULL A task ready to be sent.
+ */
+static struct swim_task *
+swim_begin_send(struct swim_scheduler *scheduler, struct ev_loop *loop,
+		struct ev_io *io, int events, const struct sockaddr_in **dst)
 {
 	assert((events & EV_WRITE) != 0);
 	(void) events;
-	struct swim_scheduler *scheduler = (struct swim_scheduler *) io->data;
 	if (rlist_empty(&scheduler->queue_output)) {
 		/*
 		 * Possible, if a member pushed a task and then
 		 * was deleted together with it.
 		 */
 		swim_ev_io_stop(loop, io);
-		return;
+		return NULL;
 	}
 	struct swim_task *task =
 		rlist_shift_entry(&scheduler->queue_output, struct swim_task,
 				  in_queue_output);
 	const struct sockaddr_in *src = &scheduler->transport.addr;
-	const struct sockaddr_in *dst = &task->dst;
-	const char *dst_str = swim_inaddr_str(dst);
+	*dst = &task->dst;
+	const char *dst_str = swim_inaddr_str(*dst);
 	if (! swim_inaddr_is_empty(&task->proxy)) {
-		dst = &task->proxy;
+		*dst = &task->proxy;
 		dst_str = tt_sprintf("%s via %s", dst_str,
-				     swim_inaddr_str(dst));
+				     swim_inaddr_str(*dst));
 		swim_packet_build_meta(&task->packet, src, src, &task->dst);
 	} else {
 		swim_packet_build_meta(&task->packet, src, NULL, NULL);
 	}
 	say_verbose("SWIM %d: send %s to %s", swim_scheduler_fd(scheduler),
 		    task->desc, dst_str);
-	int rc = swim_transport_send(&scheduler->transport, task->packet.buf,
-				     task->packet.pos - task->packet.buf,
-				     (const struct sockaddr *) dst,
-				     sizeof(*dst));
-	if (rc < 0)
+	return task;
+}
+
+/** Send a packet into the network. */
+static inline ssize_t
+swim_do_send(struct swim_scheduler *scheduler, const char *buf, int size,
+	     const struct sockaddr_in *dst)
+{
+	return swim_transport_send(&scheduler->transport, buf, size,
+				   (const struct sockaddr *) dst, sizeof(*dst));
+}
+
+/**
+ * Finalize packet transmission, call the completion callback.
+ * @param scheduler Scheduler owning @a task.
+ * @param task Sent (or failed to be sent) task.
+ * @param size Result of send().
+ */
+static inline void
+swim_complete_send(struct swim_scheduler *scheduler, struct swim_task *task,
+		   ssize_t size)
+{
+	if (size < 0)
 		diag_log();
 	if (task->complete != NULL)
-		task->complete(task, scheduler, rc);
+		task->complete(task, scheduler, size);
 }
 
 static void
-swim_scheduler_on_input(struct ev_loop *loop, struct ev_io *io, int events)
+swim_scheduler_on_output(struct ev_loop *loop, struct ev_io *io, int events)
+{
+	struct swim_scheduler *scheduler = (struct swim_scheduler *) io->data;
+	const struct sockaddr_in *dst;
+	struct swim_task *task = swim_begin_send(scheduler, loop, io, events,
+						 &dst);
+	if (task == NULL)
+		return;
+	ssize_t size = swim_do_send(scheduler, task->packet.buf,
+				    task->packet.pos - task->packet.buf, dst);
+	swim_complete_send(scheduler, task, size);
+}
+
+/**
+ * Begin packet receipt. Note, this function is no-op, and exists
+ * just for consistency with begin/do/complete_send() functions.
+ */
+static inline void
+swim_begin_recv(struct swim_scheduler *scheduler, struct ev_loop *loop,
+		struct ev_io *io, int events)
 {
 	assert((events & EV_READ) != 0);
+	(void) io;
+	(void) scheduler;
 	(void) events;
 	(void) loop;
-	struct swim_scheduler *scheduler = (struct swim_scheduler *) io->data;
+}
+
+/** Receive a packet from the network. */
+static ssize_t
+swim_do_recv(struct swim_scheduler *scheduler, char *buf, int size)
+{
 	struct sockaddr_in src;
 	socklen_t len = sizeof(src);
-	char buf[UDP_PACKET_SIZE];
-	ssize_t size = swim_transport_recv(&scheduler->transport, buf,
-					   sizeof(buf),
-					   (struct sockaddr *) &src, &len);
-	if (size <= 0) {
-		if (size < 0)
-			goto error;
-		return;
-	}
+	ssize_t rc = swim_transport_recv(&scheduler->transport, buf, size,
+					 (struct sockaddr *) &src, &len);
+	if (rc <= 0)
+		return rc;
 	say_verbose("SWIM %d: received from %s", swim_scheduler_fd(scheduler),
 		    swim_inaddr_str(&src));
+	return rc;
+}
+
+/**
+ * Finalize packet receipt, call the SWIM core callbacks, or
+ * forward the packet to a next node.
+ */
+static void
+swim_complete_recv(struct swim_scheduler *scheduler, const char *buf,
+		   ssize_t size)
+{
+	if (size < 0)
+		goto error;
+	if (size == 0)
+		return;
 	struct swim_meta_def meta;
 	const char *pos = buf, *end = pos + size;
 	if (swim_meta_def_decode(&meta, &pos, end) < 0)
@@ -446,6 +515,16 @@ error:
 	diag_log();
 }
 
+static void
+swim_scheduler_on_input(struct ev_loop *loop, struct ev_io *io, int events)
+{
+	struct swim_scheduler *scheduler = (struct swim_scheduler *) io->data;
+	char buf[UDP_PACKET_SIZE];
+	swim_begin_recv(scheduler, loop, io, events);
+	ssize_t size = swim_do_recv(scheduler, buf, UDP_PACKET_SIZE);
+	swim_complete_recv(scheduler, buf, size);
+}
+
 const char *
 swim_inaddr_str(const struct sockaddr_in *addr)
 {
-- 
2.20.1 (Apple Git-117)

[tarantool-patches] [PATCH 2/2] swim: implement and expose transport-level encryption

[Vladislav Shpilevoy]

SWIM is going to be used in and between datacenters, which means,
that its packets will go through public networks. Therefore raw
SWIM packets are vulnerable to attacks.

An attacker can do any and all of the following things:

  1) Extract secret information from member payloads, like
     credentials to Tarantool binary ports;

  2) Change UUIDs and addresses in the packets and break a
     topology;

  3) Catch the packets and pretend being a Tarantool instance,
     which could lead to undefined behaviour depending on an
     application logic.

SWIM packets need a protection layer. This commit introduces it.
SWIM transport level allows to choose an encryption algorithm
with a private key to encrypt each packet with that key.

Besides, each packet is encrypted using a random public key
prepended to the packet.

SWIM now provides a public API to choose an encryption algorithm
and a private key.

Part of #3234
---
 src/lib/swim/CMakeLists.txt |   2 +-
 src/lib/swim/swim.c         |  12 ++++
 src/lib/swim/swim.h         |  15 ++++
 src/lib/swim/swim_ev.h      |   2 +
 src/lib/swim/swim_io.c      | 133 +++++++++++++++++++++++++++++++-----
 src/lib/swim/swim_io.h      |  26 ++++++-
 src/lib/swim/swim_proto.h   |   5 ++
 test/unit/swim.c            |  47 ++++++++++++-
 test/unit/swim.result       |   9 ++-
 test/unit/swim_test_utils.c |  15 ++--
 test/unit/swim_test_utils.h |   8 +++
 11 files changed, 247 insertions(+), 27 deletions(-)

diff --git a/src/lib/swim/CMakeLists.txt b/src/lib/swim/CMakeLists.txt
index 8ba99d803..11202dce3 100644
--- a/src/lib/swim/CMakeLists.txt
+++ b/src/lib/swim/CMakeLists.txt
@@ -5,7 +5,7 @@ set(lib_swim_ev_sources swim_ev.c)
 set_source_files_compile_flags(${lib_swim_sources} ${lib_swim_udp_sources}
                                ${lib_swim_ev_sources})
 add_library(swim STATIC ${lib_swim_sources})
-target_link_libraries(swim core misc uuid)
+target_link_libraries(swim core misc uuid crypto)
 add_library(swim_udp STATIC ${lib_swim_udp_sources})
 target_link_libraries(swim_udp core)
 add_library(swim_ev STATIC ${lib_swim_ev_sources})
diff --git a/src/lib/swim/swim.c b/src/lib/swim/swim.c
index fa6b3a379..801d16886 100644
--- a/src/lib/swim/swim.c
+++ b/src/lib/swim/swim.c
@@ -1818,6 +1818,18 @@ swim_cfg(struct swim *swim, const char *uri, double heartbeat_rate,
 	return 0;
 }
 
+int
+swim_set_codec(struct swim *swim, enum crypto_algo algo, const char *key)
+{
+	return swim_scheduler_set_codec(&swim->scheduler, algo, key);
+}
+
+double
+swim_ack_timeout(const struct swim *swim)
+{
+	return swim->wait_ack_tick.at;
+}
+
 bool
 swim_is_configured(const struct swim *swim)
 {
diff --git a/src/lib/swim/swim.h b/src/lib/swim/swim.h
index 331bd14f0..a3711dc27 100644
--- a/src/lib/swim/swim.h
+++ b/src/lib/swim/swim.h
@@ -32,6 +32,7 @@
  */
 #include <stdbool.h>
 #include <stdint.h>
+#include "crypto/crypto.h"
 #include "swim_constants.h"
 
 #if defined(__cplusplus)
@@ -105,6 +106,20 @@ swim_cfg(struct swim *swim, const char *uri, double heartbeat_rate,
 int
 swim_set_payload(struct swim *swim, const char *payload, uint16_t payload_size);
 
+/**
+ * Set SWIM codec to encrypt/decrypt messages.
+ * @param swim SWIM instance to set codec for.
+ * @param algo Cipher algorithm.
+ * @param key Private key of the chosen algorithm. It is used to
+ *        encrypt/decrypt messages, and should be the same on all
+ *        cluster nodes. Note that it can be changed, but it shall
+ *        be done on all cluster nodes. Otherwise the nodes will
+ *        not understand each other. There is also a public key
+ *        usually, but it is generated randomly inside SWIM.
+ */
+int
+swim_set_codec(struct swim *swim, enum crypto_algo algo, const char *key);
+
 /**
  * Stop listening and broadcasting messages, cleanup all internal
  * structures, free memory.
diff --git a/src/lib/swim/swim_ev.h b/src/lib/swim/swim_ev.h
index b68ed9e19..0fe550523 100644
--- a/src/lib/swim/swim_ev.h
+++ b/src/lib/swim/swim_ev.h
@@ -64,4 +64,6 @@ swim_ev_timer_stop(struct ev_loop *loop, struct ev_timer *watcher);
 
 #define swim_ev_io_set ev_io_set
 
+#define swim_ev_set_cb ev_set_cb
+
 #endif /* TARANTOOL_SWIM_EV_H_INCLUDED */
diff --git a/src/lib/swim/swim_io.c b/src/lib/swim/swim_io.c
index 5bfa138df..ef2f479eb 100644
--- a/src/lib/swim/swim_io.c
+++ b/src/lib/swim/swim_io.c
@@ -269,31 +269,19 @@ swim_scheduler_fd(const struct swim_scheduler *scheduler)
 	return scheduler->transport.fd;
 }
 
-/**
- * Dispatch a next output event. Build packet meta and send the
- * packet.
- */
-static void
-swim_scheduler_on_output(struct ev_loop *loop, struct ev_io *io, int events);
-
-/**
- * Dispatch a next input event. Unpack meta, forward a packet or
- * propagate further to protocol logic.
- */
-static void
-swim_scheduler_on_input(struct ev_loop *loop, struct ev_io *io, int events);
-
 void
 swim_scheduler_create(struct swim_scheduler *scheduler,
 		      swim_scheduler_on_input_f on_input)
 {
-	swim_ev_init(&scheduler->output, swim_scheduler_on_output);
 	scheduler->output.data = (void *) scheduler;
-	swim_ev_init(&scheduler->input, swim_scheduler_on_input);
 	scheduler->input.data = (void *) scheduler;
 	rlist_create(&scheduler->queue_output);
 	scheduler->on_input = on_input;
 	swim_transport_create(&scheduler->transport);
+	scheduler->codec = NULL;
+	int rc = swim_scheduler_set_codec(scheduler, CRYPTO_NONE, NULL);
+	assert(rc == 0);
+	(void) rc;
 }
 
 int
@@ -338,6 +326,38 @@ swim_scheduler_destroy(struct swim_scheduler *scheduler)
 	swim_scheduler_stop_input(scheduler);
 }
 
+/**
+ * Encrypt data and prepend it with a fresh crypto algorithm's
+ * initial vector.
+ */
+static inline int
+swim_encrypt(struct crypto_codec *c, const char *in, int in_size,
+	     char *out, int out_size)
+{
+	int iv_size = crypto_codec_iv_size(c);
+	const char *iv = crypto_codec_gen_iv(c);
+	assert(out_size >= iv_size);
+	memcpy(out, iv, iv_size);
+	out += iv_size;
+	out_size -= iv_size;
+	int rc = crypto_codec_encrypt(c, in, in_size, out, out_size);
+	if (rc < 0)
+		return -1;
+	return rc + iv_size;
+}
+
+/** Decrypt data prepended with an initial vector. */
+static inline int
+swim_decrypt(struct crypto_codec *c, const char *in, int in_size,
+	     char *out, int out_size)
+{
+	int iv_size = crypto_codec_iv_size(c);
+	const char *iv = in;
+	in += iv_size;
+	in_size -= iv_size;
+	return crypto_codec_decrypt(c, iv, in, in_size, out, out_size);
+}
+
 /**
  * Begin packet transmission. Prepare a next task in the queue to
  * send its packet: build a meta header, pop the task from the
@@ -411,8 +431,34 @@ swim_complete_send(struct swim_scheduler *scheduler, struct swim_task *task,
 		task->complete(task, scheduler, size);
 }
 
+/**
+ * On a new EV_WRITE event send a next packet from the queue
+ * encrypted with the currently chosen algorithm.
+ */
 static void
-swim_scheduler_on_output(struct ev_loop *loop, struct ev_io *io, int events)
+swim_on_encrypted_output(struct ev_loop *loop, struct ev_io *io, int events)
+{
+	struct swim_scheduler *scheduler = (struct swim_scheduler *) io->data;
+	const struct sockaddr_in *dst;
+	struct swim_task *task = swim_begin_send(scheduler, loop, io, events,
+						 &dst);
+	if (task == NULL)
+		return;
+	char *buf = static_alloc(UDP_PACKET_SIZE);
+	assert(buf != NULL);
+	ssize_t size = swim_encrypt(scheduler->codec, task->packet.buf,
+				    task->packet.pos - task->packet.buf,
+				    buf, UDP_PACKET_SIZE);
+	if (size > 0)
+		size = swim_do_send(scheduler, buf, size, dst);
+	swim_complete_send(scheduler, task, size);
+}
+
+/**
+ * On a new EV_WRITE event send a next packet without encryption.
+ */
+static void
+swim_on_plain_output(struct ev_loop *loop, struct ev_io *io, int events)
 {
 	struct swim_scheduler *scheduler = (struct swim_scheduler *) io->data;
 	const struct sockaddr_in *dst;
@@ -515,8 +561,35 @@ error:
 	diag_log();
 }
 
+/**
+ * On a new EV_READ event receive an encrypted packet from the
+ * network.
+ */
 static void
-swim_scheduler_on_input(struct ev_loop *loop, struct ev_io *io, int events)
+swim_on_encrypted_input(struct ev_loop *loop, struct ev_io *io, int events)
+{
+	struct swim_scheduler *scheduler = (struct swim_scheduler *) io->data;
+	/*
+	 * Buffer for decrypted data is on stack, not on static
+	 * memory, because the SWIM code uses static memory as
+	 * well and can accidentally rewrite the packet data.
+	 */
+	char buf[UDP_PACKET_SIZE];
+	swim_begin_recv(scheduler, loop, io, events);
+
+	char *ibuf = static_alloc(UDP_PACKET_SIZE);
+	assert(ibuf != NULL);
+	ssize_t size = swim_do_recv(scheduler, ibuf, UDP_PACKET_SIZE);
+	if (size > 0) {
+		size = swim_decrypt(scheduler->codec, ibuf, size,
+				    buf, UDP_PACKET_SIZE);
+	}
+	swim_complete_recv(scheduler, buf, size);
+}
+
+/** On a new EV_READ event receive a packet from the network. */
+static void
+swim_on_plain_input(struct ev_loop *loop, struct ev_io *io, int events)
 {
 	struct swim_scheduler *scheduler = (struct swim_scheduler *) io->data;
 	char buf[UDP_PACKET_SIZE];
@@ -525,6 +598,30 @@ swim_scheduler_on_input(struct ev_loop *loop, struct ev_io *io, int events)
 	swim_complete_recv(scheduler, buf, size);
 }
 
+int
+swim_scheduler_set_codec(struct swim_scheduler *scheduler,
+			 enum crypto_algo algo, const char *key)
+{
+	if (algo == CRYPTO_NONE) {
+		if (scheduler->codec != NULL) {
+			crypto_codec_delete(scheduler->codec);
+			scheduler->codec = NULL;
+		}
+		swim_ev_set_cb(&scheduler->output, swim_on_plain_output);
+		swim_ev_set_cb(&scheduler->input, swim_on_plain_input);
+		return 0;
+	}
+	struct crypto_codec *newc = crypto_codec_new(algo, key);
+	if (newc == NULL)
+		return -1;
+	if (scheduler->codec != NULL)
+		crypto_codec_delete(scheduler->codec);
+	scheduler->codec = newc;
+	swim_ev_set_cb(&scheduler->output, swim_on_encrypted_output);
+	swim_ev_set_cb(&scheduler->input, swim_on_encrypted_input);
+	return 0;
+}
+
 const char *
 swim_inaddr_str(const struct sockaddr_in *addr)
 {
diff --git a/src/lib/swim/swim_io.h b/src/lib/swim/swim_io.h
index 42d94fc92..397ba4a82 100644
--- a/src/lib/swim/swim_io.h
+++ b/src/lib/swim/swim_io.h
@@ -33,6 +33,7 @@
 #include "tt_static.h"
 #include "small/rlist.h"
 #include "salad/stailq.h"
+#include "crypto/crypto.h"
 #include "swim_transport.h"
 #include "tarantool_ev.h"
 #include "uuid/tt_uuid.h"
@@ -57,13 +58,24 @@ enum {
 	 * configuration.
 	 */
 	UDP_PACKET_SIZE = 1472,
+	/**
+	 * Data can be encrypted, which usually makes it slightly
+	 * bigger in size. Also, to decode data the receiver needs
+	 * two keys: private key and public initial vector. Public
+	 * initial vector is generated randomly for each packet
+	 * and prepends the data. This is why maximal data size is
+	 * reduced by one block and IV sizes.
+	 */
+	MAX_PACKET_SIZE = UDP_PACKET_SIZE - CRYPTO_AES_BLOCK_SIZE -
+			  CRYPTO_AES_IV_SIZE,
+
 };
 
 /**
  * UDP packet. Works as an allocator, allowing to fill its body
  * gradually, while preserving prefix for metadata.
  *
- *          < - - - -UDP_PACKET_SIZE- - - - ->
+ *          < - - - -MAX_PACKET_SIZE- - - - ->
  *          +--------+-----------------------+
  *          |  meta  |    body    |  *free*  |
  *          +--------+-----------------------+
@@ -86,7 +98,7 @@ struct swim_packet {
 	 */
 	char meta[0];
 	/** Packet body buffer. */
-	char buf[UDP_PACKET_SIZE];
+	char buf[MAX_PACKET_SIZE];
 	/**
 	 * Pointer to the end of the buffer. Just sugar to do not
 	 * write 'buf + sizeof(buf)' each time.
@@ -147,6 +159,11 @@ typedef void (*swim_scheduler_on_input_f)(struct swim_scheduler *scheduler,
 struct swim_scheduler {
 	/** Transport to send/receive packets. */
 	struct swim_transport transport;
+	/**
+	 * Codec to encode messages before sending, and decode
+	 * before lifting up to the SWIM core logic.
+	 */
+	struct crypto_codec *codec;
 	/**
 	 * Function called when a packet is received. It takes
 	 * packet body, while meta is handled by transport level
@@ -180,6 +197,11 @@ int
 swim_scheduler_bind(struct swim_scheduler *scheduler,
 		    const struct sockaddr_in *addr);
 
+/** Set a new codec to encrypt/decrypt messages. */
+int
+swim_scheduler_set_codec(struct swim_scheduler *scheduler,
+			 enum crypto_algo algo, const char *key);
+
 /** Stop accepting new packets from the network. */
 void
 swim_scheduler_stop_input(struct swim_scheduler *scheduler);
diff --git a/src/lib/swim/swim_proto.h b/src/lib/swim/swim_proto.h
index beb9bb1fe..482d79fb1 100644
--- a/src/lib/swim/swim_proto.h
+++ b/src/lib/swim/swim_proto.h
@@ -47,6 +47,11 @@ enum {
  * SWIM binary protocol structures and helpers. Below is a picture
  * of a SWIM message template:
  *
+ * +-----------------Public data, not encrypted------------------+
+ * |                                                             |
+ * |      Initial vector, size depends on chosen algorithm.      |
+ * |                   Next data is encrypted.                   |
+ * |                                                             |
  * +----------Meta section, handled by transport level-----------+
  * | {                                                           |
  * |     SWIM_META_TARANTOOL_VERSION: uint, Tarantool version ID,|
diff --git a/test/unit/swim.c b/test/unit/swim.c
index 8b4a1f17a..8303e2900 100644
--- a/test/unit/swim.c
+++ b/test/unit/swim.c
@@ -31,6 +31,7 @@
 
 #include "memory.h"
 #include "fiber.h"
+#include "random.h"
 #include "uuid/tt_uuid.h"
 #include "unit.h"
 #include "uri/uri.h"
@@ -881,10 +882,51 @@ swim_test_indirect_ping(void)
 	swim_finish_test();
 }
 
+static void
+swim_test_encryption(void)
+{
+	swim_start_test(3);
+	struct swim_cluster *cluster = swim_cluster_new(2);
+	const char *key = "1234567812345678";
+	swim_cluster_set_codec(cluster, CRYPTO_AES128, key);
+	swim_cluster_add_link(cluster, 0, 1);
+
+	is(swim_cluster_wait_fullmesh(cluster, 2), 0,
+	   "cluster works with encryption");
+	swim_cluster_delete(cluster);
+	/*
+	 * Test that the instances can not interact with different
+	 * encryption keys.
+	 */
+	cluster = swim_cluster_new(2);
+	struct swim *s1 = swim_cluster_member(cluster, 0);
+	int rc = swim_set_codec(s1, CRYPTO_AES128, key);
+	fail_if(rc != 0);
+	struct swim *s2 = swim_cluster_member(cluster, 1);
+	key = "8765432187654321";
+	rc = swim_set_codec(s2, CRYPTO_AES128, key);
+	fail_if(rc != 0);
+	swim_cluster_add_link(cluster, 0, 1);
+	swim_run_for(2);
+	ok(! swim_cluster_is_fullmesh(cluster),
+	   "different encryption keys - can't interact");
+
+	rc = swim_set_codec(s1, CRYPTO_NONE, NULL);
+	fail_if(rc != 0);
+	rc = swim_set_codec(s2, CRYPTO_NONE, NULL);
+	fail_if(rc != 0);
+	is(swim_cluster_wait_fullmesh(cluster, 2), 0,
+	   "cluster works after encryption has been disabled");
+
+	swim_cluster_delete(cluster);
+
+	swim_finish_test();
+}
+
 static int
 main_f(va_list ap)
 {
-	swim_start_test(18);
+	swim_start_test(19);
 
 	(void) ap;
 	swim_test_ev_init();
@@ -908,6 +950,7 @@ main_f(va_list ap)
 	swim_test_payload_basic();
 	swim_test_payload_refutation();
 	swim_test_indirect_ping();
+	swim_test_encryption();
 
 	swim_test_transport_free();
 	swim_test_ev_free();
@@ -920,6 +963,7 @@ main_f(va_list ap)
 int
 main()
 {
+	random_init();
 	time_t seed = time(NULL);
 	srand(seed);
 	memory_init();
@@ -944,6 +988,7 @@ main()
 	say_logger_free();
 	fiber_free();
 	memory_free();
+	random_free();
 
 	return test_result;
 }
\ No newline at end of file
diff --git a/test/unit/swim.result b/test/unit/swim.result
index 0d1fa4a32..24597641c 100644
--- a/test/unit/swim.result
+++ b/test/unit/swim.result
@@ -1,5 +1,5 @@
 	*** main_f ***
-1..18
+1..19
 	*** swim_test_one_link ***
     1..6
     ok 1 - no rounds - no fullmesh
@@ -186,4 +186,11 @@ ok 17 - subtests
     ok 2 - as well as S2 - they communicated via S3
 ok 18 - subtests
 	*** swim_test_indirect_ping: done ***
+	*** swim_test_encryption ***
+    1..3
+    ok 1 - cluster works with encryption
+    ok 2 - different encryption keys - can't interact
+    ok 3 - cluster works after encryption has been disabled
+ok 19 - subtests
+	*** swim_test_encryption: done ***
 	*** main_f: done ***
diff --git a/test/unit/swim_test_utils.c b/test/unit/swim_test_utils.c
index de1bef6e7..77e6e8b7a 100644
--- a/test/unit/swim_test_utils.c
+++ b/test/unit/swim_test_utils.c
@@ -227,9 +227,9 @@ swim_cluster_new(int size)
 	return res;
 }
 
-#define swim_cluster_set_cfg(cluster, ...) ({				\
+#define swim_cluster_set_cfg(cluster, func, ...) ({				\
 	for (int i = 0; i < cluster->size; ++i) {			\
-		int rc = swim_cfg(cluster->node[i].swim, __VA_ARGS__);	\
+		int rc = func(cluster->node[i].swim, __VA_ARGS__);	\
 		assert(rc == 0);					\
 		(void) rc;						\
 	}								\
@@ -238,14 +238,21 @@ swim_cluster_new(int size)
 void
 swim_cluster_set_ack_timeout(struct swim_cluster *cluster, double ack_timeout)
 {
-	swim_cluster_set_cfg(cluster, NULL, -1, ack_timeout, -1, NULL);
+	swim_cluster_set_cfg(cluster, swim_cfg, NULL, -1, ack_timeout, -1, NULL);
 	cluster->ack_timeout = ack_timeout;
 }
 
+void
+swim_cluster_set_codec(struct swim_cluster *cluster, enum crypto_algo algo,
+		       const char *key)
+{
+	swim_cluster_set_cfg(cluster, swim_set_codec, algo, key);
+}
+
 void
 swim_cluster_set_gc(struct swim_cluster *cluster, enum swim_gc_mode gc_mode)
 {
-	swim_cluster_set_cfg(cluster, NULL, -1, -1, gc_mode, NULL);
+	swim_cluster_set_cfg(cluster, swim_cfg, NULL, -1, -1, gc_mode, NULL);
 	cluster->gc_mode = gc_mode;
 }
 
diff --git a/test/unit/swim_test_utils.h b/test/unit/swim_test_utils.h
index c78894820..531b5e92b 100644
--- a/test/unit/swim_test_utils.h
+++ b/test/unit/swim_test_utils.h
@@ -48,6 +48,14 @@ swim_cluster_new(int size);
 void
 swim_cluster_set_ack_timeout(struct swim_cluster *cluster, double ack_timeout);
 
+/**
+ * Set an encryption algorithm and a key for each instance in
+ * @a cluster.
+ */
+void
+swim_cluster_set_codec(struct swim_cluster *cluster, enum crypto_algo algo,
+		       const char *key);
+
 /**
  * Change number of unacknowledged pings to delete a dead member
  * of all the instances in the cluster.
-- 
2.20.1 (Apple Git-117)

[tarantool-patches] [PATCH 2/2] swim: implement and expose transport-level encryption

[Vladislav Shpilevoy]

SWIM is going to be used in and between datacenters, which means,
that its packets will go through public networks. Therefore raw
SWIM packets are vulnerable to attacks.

An attacker can do any and all of the following things:

  1) Extract secret information from member payloads, like
     credentials to Tarantool binary ports;

  2) Change UUIDs and addresses in the packets and break a
     topology;

  3) Catch the packets and pretend being a Tarantool instance,
     which could lead to undefined behaviour depending on an
     application logic.

SWIM packets need a protection layer. This commit introduces it.
SWIM transport level allows to choose an encryption algorithm
with a private key to encrypt each packet with that key.

Besides, each packet is encrypted using a random public key
prepended to the packet.

SWIM now provides a public API to choose an encryption algorithm
and a private key.

Part of #3234
---
 src/lib/swim/CMakeLists.txt |   2 +-
 src/lib/swim/swim.c         |   6 ++
 src/lib/swim/swim.h         |  15 ++++
 src/lib/swim/swim_ev.h      |   2 +
 src/lib/swim/swim_io.c      | 133 +++++++++++++++++++++++++++++++-----
 src/lib/swim/swim_io.h      |  26 ++++++-
 src/lib/swim/swim_proto.h   |   5 ++
 test/unit/swim.c            |  47 ++++++++++++-
 test/unit/swim.result       |   9 ++-
 test/unit/swim_test_utils.c |  15 ++--
 test/unit/swim_test_utils.h |   8 +++
 11 files changed, 241 insertions(+), 27 deletions(-)

diff --git a/src/lib/swim/CMakeLists.txt b/src/lib/swim/CMakeLists.txt
index 8ba99d803..11202dce3 100644
--- a/src/lib/swim/CMakeLists.txt
+++ b/src/lib/swim/CMakeLists.txt
@@ -5,7 +5,7 @@ set(lib_swim_ev_sources swim_ev.c)
 set_source_files_compile_flags(${lib_swim_sources} ${lib_swim_udp_sources}
                                ${lib_swim_ev_sources})
 add_library(swim STATIC ${lib_swim_sources})
-target_link_libraries(swim core misc uuid)
+target_link_libraries(swim core misc uuid crypto)
 add_library(swim_udp STATIC ${lib_swim_udp_sources})
 target_link_libraries(swim_udp core)
 add_library(swim_ev STATIC ${lib_swim_ev_sources})
diff --git a/src/lib/swim/swim.c b/src/lib/swim/swim.c
index 746106657..68b87b120 100644
--- a/src/lib/swim/swim.c
+++ b/src/lib/swim/swim.c
@@ -1757,6 +1757,12 @@ swim_cfg(struct swim *swim, const char *uri, double heartbeat_rate,
 	return 0;
 }
 
+int
+swim_set_codec(struct swim *swim, enum crypto_algo algo, const char *key)
+{
+	return swim_scheduler_set_codec(&swim->scheduler, algo, key);
+}
+
 double
 swim_ack_timeout(const struct swim *swim)
 {
diff --git a/src/lib/swim/swim.h b/src/lib/swim/swim.h
index 6a219d131..5abbe8c33 100644
--- a/src/lib/swim/swim.h
+++ b/src/lib/swim/swim.h
@@ -32,6 +32,7 @@
  */
 #include <stdbool.h>
 #include <stdint.h>
+#include "crypto/crypto.h"
 #include "swim_constants.h"
 
 #if defined(__cplusplus)
@@ -108,6 +109,20 @@ swim_ack_timeout(const struct swim *swim);
 int
 swim_set_payload(struct swim *swim, const char *payload, uint16_t payload_size);
 
+/**
+ * Set SWIM codec to encrypt/decrypt messages.
+ * @param swim SWIM instance to set codec for.
+ * @param algo Cipher algorithm.
+ * @param key Private key of the chosen algorithm. It is used to
+ *        encrypt/decrypt messages, and should be the same on all
+ *        cluster nodes. Note that it can be changed, but it shall
+ *        be done on all cluster nodes. Otherwise the nodes will
+ *        not understand each other. There is also a public key
+ *        usually, but it is generated randomly inside SWIM.
+ */
+int
+swim_set_codec(struct swim *swim, enum crypto_algo algo, const char *key);
+
 /**
  * Stop listening and broadcasting messages, cleanup all internal
  * structures, free memory.
diff --git a/src/lib/swim/swim_ev.h b/src/lib/swim/swim_ev.h
index 8f41d43fe..34394ef47 100644
--- a/src/lib/swim/swim_ev.h
+++ b/src/lib/swim/swim_ev.h
@@ -61,4 +61,6 @@ swim_ev_timer_stop(struct ev_loop *loop, struct ev_timer *watcher);
 
 #define swim_ev_io_set ev_io_set
 
+#define swim_ev_set_cb ev_set_cb
+
 #endif /* TARANTOOL_SWIM_EV_H_INCLUDED */
diff --git a/src/lib/swim/swim_io.c b/src/lib/swim/swim_io.c
index 2c1448c1f..d49900a86 100644
--- a/src/lib/swim/swim_io.c
+++ b/src/lib/swim/swim_io.c
@@ -269,31 +269,19 @@ swim_scheduler_fd(const struct swim_scheduler *scheduler)
 	return scheduler->transport.fd;
 }
 
-/**
- * Dispatch a next output event. Build packet meta and send the
- * packet.
- */
-static void
-swim_scheduler_on_output(struct ev_loop *loop, struct ev_io *io, int events);
-
-/**
- * Dispatch a next input event. Unpack meta, forward a packet or
- * propagate further to protocol logic.
- */
-static void
-swim_scheduler_on_input(struct ev_loop *loop, struct ev_io *io, int events);
-
 void
 swim_scheduler_create(struct swim_scheduler *scheduler,
 		      swim_scheduler_on_input_f on_input)
 {
-	swim_ev_init(&scheduler->output, swim_scheduler_on_output);
 	scheduler->output.data = (void *) scheduler;
-	swim_ev_init(&scheduler->input, swim_scheduler_on_input);
 	scheduler->input.data = (void *) scheduler;
 	rlist_create(&scheduler->queue_output);
 	scheduler->on_input = on_input;
 	swim_transport_create(&scheduler->transport);
+	scheduler->codec = NULL;
+	int rc = swim_scheduler_set_codec(scheduler, CRYPTO_NONE, NULL);
+	assert(rc == 0);
+	(void) rc;
 }
 
 int
@@ -335,6 +323,38 @@ swim_scheduler_destroy(struct swim_scheduler *scheduler)
 	swim_scheduler_stop_input(scheduler);
 }
 
+/**
+ * Encrypt data and prepend it with a fresh crypto algorithm's
+ * initial vector.
+ */
+static inline int
+swim_encrypt(struct crypto_codec *c, const char *in, int in_size,
+	     char *out, int out_size)
+{
+	int iv_size = crypto_codec_iv_size(c);
+	const char *iv = crypto_codec_gen_iv(c);
+	assert(out_size >= iv_size);
+	memcpy(out, iv, iv_size);
+	out += iv_size;
+	out_size -= iv_size;
+	int rc = crypto_codec_encrypt(c, in, in_size, out, out_size);
+	if (rc < 0)
+		return -1;
+	return rc + iv_size;
+}
+
+/** Decrypt data prepended with an initial vector. */
+static inline int
+swim_decrypt(struct crypto_codec *c, const char *in, int in_size,
+	     char *out, int out_size)
+{
+	int iv_size = crypto_codec_iv_size(c);
+	const char *iv = in;
+	in += iv_size;
+	in_size -= iv_size;
+	return crypto_codec_decrypt(c, iv, in, in_size, out, out_size);
+}
+
 /**
  * Begin packet transmission. Prepare a next task in the queue to
  * send its packet: build a meta header, pop the task from the
@@ -408,8 +428,34 @@ swim_complete_send(struct swim_scheduler *scheduler, struct swim_task *task,
 		task->complete(task, scheduler, size);
 }
 
+/**
+ * On a new EV_WRITE event send a next packet from the queue
+ * encrypted with the currently chosen algorithm.
+ */
+static void
+swim_on_encrypted_output(struct ev_loop *loop, struct ev_io *io, int events)
+{
+	struct swim_scheduler *scheduler = (struct swim_scheduler *) io->data;
+	const struct sockaddr_in *dst;
+	struct swim_task *task = swim_begin_send(scheduler, loop, io, events,
+						 &dst);
+	if (task == NULL)
+		return;
+	char *buf = static_alloc(UDP_PACKET_SIZE);
+	assert(buf != NULL);
+	ssize_t size = swim_encrypt(scheduler->codec, task->packet.buf,
+				    task->packet.pos - task->packet.buf,
+				    buf, UDP_PACKET_SIZE);
+	if (size > 0)
+		size = swim_do_send(scheduler, buf, size, dst);
+	swim_complete_send(scheduler, task, size);
+}
+
+/**
+ * On a new EV_WRITE event send a next packet without encryption.
+ */
 static void
-swim_scheduler_on_output(struct ev_loop *loop, struct ev_io *io, int events)
+swim_on_plain_output(struct ev_loop *loop, struct ev_io *io, int events)
 {
 	struct swim_scheduler *scheduler = (struct swim_scheduler *) io->data;
 	const struct sockaddr_in *dst;
@@ -512,8 +558,35 @@ error:
 	diag_log();
 }
 
+/**
+ * On a new EV_READ event receive an encrypted packet from the
+ * network.
+ */
 static void
-swim_scheduler_on_input(struct ev_loop *loop, struct ev_io *io, int events)
+swim_on_encrypted_input(struct ev_loop *loop, struct ev_io *io, int events)
+{
+	struct swim_scheduler *scheduler = (struct swim_scheduler *) io->data;
+	/*
+	 * Buffer for decrypted data is on stack, not on static
+	 * memory, because the SWIM code uses static memory as
+	 * well and can accidentally rewrite the packet data.
+	 */
+	char buf[UDP_PACKET_SIZE];
+	swim_begin_recv(scheduler, loop, io, events);
+
+	char *ibuf = static_alloc(UDP_PACKET_SIZE);
+	assert(ibuf != NULL);
+	ssize_t size = swim_do_recv(scheduler, ibuf, UDP_PACKET_SIZE);
+	if (size > 0) {
+		size = swim_decrypt(scheduler->codec, ibuf, size,
+				    buf, UDP_PACKET_SIZE);
+	}
+	swim_complete_recv(scheduler, buf, size);
+}
+
+/** On a new EV_READ event receive a packet from the network. */
+static void
+swim_on_plain_input(struct ev_loop *loop, struct ev_io *io, int events)
 {
 	struct swim_scheduler *scheduler = (struct swim_scheduler *) io->data;
 	char buf[UDP_PACKET_SIZE];
@@ -522,6 +595,30 @@ swim_scheduler_on_input(struct ev_loop *loop, struct ev_io *io, int events)
 	swim_complete_recv(scheduler, buf, size);
 }
 
+int
+swim_scheduler_set_codec(struct swim_scheduler *scheduler,
+			 enum crypto_algo algo, const char *key)
+{
+	if (algo == CRYPTO_NONE) {
+		if (scheduler->codec != NULL) {
+			crypto_codec_delete(scheduler->codec);
+			scheduler->codec = NULL;
+		}
+		swim_ev_set_cb(&scheduler->output, swim_on_plain_output);
+		swim_ev_set_cb(&scheduler->input, swim_on_plain_input);
+		return 0;
+	}
+	struct crypto_codec *newc = crypto_codec_new(algo, key);
+	if (newc == NULL)
+		return -1;
+	if (scheduler->codec != NULL)
+		crypto_codec_delete(scheduler->codec);
+	scheduler->codec = newc;
+	swim_ev_set_cb(&scheduler->output, swim_on_encrypted_output);
+	swim_ev_set_cb(&scheduler->input, swim_on_encrypted_input);
+	return 0;
+}
+
 const char *
 swim_inaddr_str(const struct sockaddr_in *addr)
 {
diff --git a/src/lib/swim/swim_io.h b/src/lib/swim/swim_io.h
index 42d94fc92..397ba4a82 100644
--- a/src/lib/swim/swim_io.h
+++ b/src/lib/swim/swim_io.h
@@ -33,6 +33,7 @@
 #include "tt_static.h"
 #include "small/rlist.h"
 #include "salad/stailq.h"
+#include "crypto/crypto.h"
 #include "swim_transport.h"
 #include "tarantool_ev.h"
 #include "uuid/tt_uuid.h"
@@ -57,13 +58,24 @@ enum {
 	 * configuration.
 	 */
 	UDP_PACKET_SIZE = 1472,
+	/**
+	 * Data can be encrypted, which usually makes it slightly
+	 * bigger in size. Also, to decode data the receiver needs
+	 * two keys: private key and public initial vector. Public
+	 * initial vector is generated randomly for each packet
+	 * and prepends the data. This is why maximal data size is
+	 * reduced by one block and IV sizes.
+	 */
+	MAX_PACKET_SIZE = UDP_PACKET_SIZE - CRYPTO_AES_BLOCK_SIZE -
+			  CRYPTO_AES_IV_SIZE,
+
 };
 
 /**
  * UDP packet. Works as an allocator, allowing to fill its body
  * gradually, while preserving prefix for metadata.
  *
- *          < - - - -UDP_PACKET_SIZE- - - - ->
+ *          < - - - -MAX_PACKET_SIZE- - - - ->
  *          +--------+-----------------------+
  *          |  meta  |    body    |  *free*  |
  *          +--------+-----------------------+
@@ -86,7 +98,7 @@ struct swim_packet {
 	 */
 	char meta[0];
 	/** Packet body buffer. */
-	char buf[UDP_PACKET_SIZE];
+	char buf[MAX_PACKET_SIZE];
 	/**
 	 * Pointer to the end of the buffer. Just sugar to do not
 	 * write 'buf + sizeof(buf)' each time.
@@ -147,6 +159,11 @@ typedef void (*swim_scheduler_on_input_f)(struct swim_scheduler *scheduler,
 struct swim_scheduler {
 	/** Transport to send/receive packets. */
 	struct swim_transport transport;
+	/**
+	 * Codec to encode messages before sending, and decode
+	 * before lifting up to the SWIM core logic.
+	 */
+	struct crypto_codec *codec;
 	/**
 	 * Function called when a packet is received. It takes
 	 * packet body, while meta is handled by transport level
@@ -180,6 +197,11 @@ int
 swim_scheduler_bind(struct swim_scheduler *scheduler,
 		    const struct sockaddr_in *addr);
 
+/** Set a new codec to encrypt/decrypt messages. */
+int
+swim_scheduler_set_codec(struct swim_scheduler *scheduler,
+			 enum crypto_algo algo, const char *key);
+
 /** Stop accepting new packets from the network. */
 void
 swim_scheduler_stop_input(struct swim_scheduler *scheduler);
diff --git a/src/lib/swim/swim_proto.h b/src/lib/swim/swim_proto.h
index beb9bb1fe..482d79fb1 100644
--- a/src/lib/swim/swim_proto.h
+++ b/src/lib/swim/swim_proto.h
@@ -47,6 +47,11 @@ enum {
  * SWIM binary protocol structures and helpers. Below is a picture
  * of a SWIM message template:
  *
+ * +-----------------Public data, not encrypted------------------+
+ * |                                                             |
+ * |      Initial vector, size depends on chosen algorithm.      |
+ * |                   Next data is encrypted.                   |
+ * |                                                             |
  * +----------Meta section, handled by transport level-----------+
  * | {                                                           |
  * |     SWIM_META_TARANTOOL_VERSION: uint, Tarantool version ID,|
diff --git a/test/unit/swim.c b/test/unit/swim.c
index 2deaf138a..5aaae089d 100644
--- a/test/unit/swim.c
+++ b/test/unit/swim.c
@@ -31,6 +31,7 @@
 
 #include "memory.h"
 #include "fiber.h"
+#include "random.h"
 #include "uuid/tt_uuid.h"
 #include "unit.h"
 #include "uri/uri.h"
@@ -870,10 +871,51 @@ swim_test_indirect_ping(void)
 	swim_finish_test();
 }
 
+static void
+swim_test_encryption(void)
+{
+	swim_start_test(3);
+	struct swim_cluster *cluster = swim_cluster_new(2);
+	const char *key = "1234567812345678";
+	swim_cluster_set_codec(cluster, CRYPTO_AES128, key);
+	swim_cluster_add_link(cluster, 0, 1);
+
+	is(swim_cluster_wait_fullmesh(cluster, 2), 0,
+	   "cluster works with encryption");
+	swim_cluster_delete(cluster);
+	/*
+	 * Test that the instances can not interact with different
+	 * encryption keys.
+	 */
+	cluster = swim_cluster_new(2);
+	struct swim *s1 = swim_cluster_member(cluster, 0);
+	int rc = swim_set_codec(s1, CRYPTO_AES128, key);
+	fail_if(rc != 0);
+	struct swim *s2 = swim_cluster_member(cluster, 1);
+	key = "8765432187654321";
+	rc = swim_set_codec(s2, CRYPTO_AES128, key);
+	fail_if(rc != 0);
+	swim_cluster_add_link(cluster, 0, 1);
+	swim_run_for(2);
+	ok(! swim_cluster_is_fullmesh(cluster),
+	   "different encryption keys - can't interact");
+
+	rc = swim_set_codec(s1, CRYPTO_NONE, NULL);
+	fail_if(rc != 0);
+	rc = swim_set_codec(s2, CRYPTO_NONE, NULL);
+	fail_if(rc != 0);
+	is(swim_cluster_wait_fullmesh(cluster, 2), 0,
+	   "cluster works after encryption has been disabled");
+
+	swim_cluster_delete(cluster);
+
+	swim_finish_test();
+}
+
 static int
 main_f(va_list ap)
 {
-	swim_start_test(18);
+	swim_start_test(19);
 
 	(void) ap;
 	swim_test_ev_init();
@@ -897,6 +939,7 @@ main_f(va_list ap)
 	swim_test_payload_basic();
 	swim_test_payload_refutation();
 	swim_test_indirect_ping();
+	swim_test_encryption();
 
 	swim_test_transport_free();
 	swim_test_ev_free();
@@ -909,6 +952,7 @@ main_f(va_list ap)
 int
 main()
 {
+	random_init();
 	time_t seed = time(NULL);
 	srand(seed);
 	memory_init();
@@ -933,6 +977,7 @@ main()
 	say_logger_free();
 	fiber_free();
 	memory_free();
+	random_free();
 
 	return test_result;
 }
\ No newline at end of file
diff --git a/test/unit/swim.result b/test/unit/swim.result
index a450d8427..6e439541a 100644
--- a/test/unit/swim.result
+++ b/test/unit/swim.result
@@ -1,5 +1,5 @@
 	*** main_f ***
-1..18
+1..19
 	*** swim_test_one_link ***
     1..6
     ok 1 - no rounds - no fullmesh
@@ -185,4 +185,11 @@ ok 17 - subtests
     ok 2 - as well as S2 - they communicated via S3
 ok 18 - subtests
 	*** swim_test_indirect_ping: done ***
+	*** swim_test_encryption ***
+    1..3
+    ok 1 - cluster works with encryption
+    ok 2 - different encryption keys - can't interact
+    ok 3 - cluster works after encryption has been disabled
+ok 19 - subtests
+	*** swim_test_encryption: done ***
 	*** main_f: done ***
diff --git a/test/unit/swim_test_utils.c b/test/unit/swim_test_utils.c
index de1bef6e7..77e6e8b7a 100644
--- a/test/unit/swim_test_utils.c
+++ b/test/unit/swim_test_utils.c
@@ -227,9 +227,9 @@ swim_cluster_new(int size)
 	return res;
 }
 
-#define swim_cluster_set_cfg(cluster, ...) ({				\
+#define swim_cluster_set_cfg(cluster, func, ...) ({				\
 	for (int i = 0; i < cluster->size; ++i) {			\
-		int rc = swim_cfg(cluster->node[i].swim, __VA_ARGS__);	\
+		int rc = func(cluster->node[i].swim, __VA_ARGS__);	\
 		assert(rc == 0);					\
 		(void) rc;						\
 	}								\
@@ -238,14 +238,21 @@ swim_cluster_new(int size)
 void
 swim_cluster_set_ack_timeout(struct swim_cluster *cluster, double ack_timeout)
 {
-	swim_cluster_set_cfg(cluster, NULL, -1, ack_timeout, -1, NULL);
+	swim_cluster_set_cfg(cluster, swim_cfg, NULL, -1, ack_timeout, -1, NULL);
 	cluster->ack_timeout = ack_timeout;
 }
 
+void
+swim_cluster_set_codec(struct swim_cluster *cluster, enum crypto_algo algo,
+		       const char *key)
+{
+	swim_cluster_set_cfg(cluster, swim_set_codec, algo, key);
+}
+
 void
 swim_cluster_set_gc(struct swim_cluster *cluster, enum swim_gc_mode gc_mode)
 {
-	swim_cluster_set_cfg(cluster, NULL, -1, -1, gc_mode, NULL);
+	swim_cluster_set_cfg(cluster, swim_cfg, NULL, -1, -1, gc_mode, NULL);
 	cluster->gc_mode = gc_mode;
 }
 
diff --git a/test/unit/swim_test_utils.h b/test/unit/swim_test_utils.h
index c78894820..531b5e92b 100644
--- a/test/unit/swim_test_utils.h
+++ b/test/unit/swim_test_utils.h
@@ -48,6 +48,14 @@ swim_cluster_new(int size);
 void
 swim_cluster_set_ack_timeout(struct swim_cluster *cluster, double ack_timeout);
 
+/**
+ * Set an encryption algorithm and a key for each instance in
+ * @a cluster.
+ */
+void
+swim_cluster_set_codec(struct swim_cluster *cluster, enum crypto_algo algo,
+		       const char *key);
+
 /**
  * Change number of unacknowledged pings to delete a dead member
  * of all the instances in the cluster.
-- 
2.20.1 (Apple Git-117)