lib: Harden OpenSSL crypto backend

This contains a few improvements and fixes in the OpenSSL backed. We
now zeroize shared secrets with OPENSSL_clear_free. The i2d_PUBKEY
output is bound against CRYPT_KEY_BUFSZ. We now return NULL rather
than silently falling back to SHA-256 when a digest is unknown. FILE
handles are now closed via pthread cleanup so a cancelled thread
cannot leak them. The DHE kex tests now set the KDF NID explicitly,
since the SHA-256 fallback is gone.

Also refactors the PEM string loaders to clean up some code duplication.

Signed-off-by: Dimitri Staessens <dimitri@ouroboros.rocks>
Signed-off-by: Sander Vrijders <sander@ouroboros.rocks>

1 files changed, 4 insertions, 0 deletions

diff --git a/src/lib/tests/kex_test.c b/src/lib/tests/kex_test.c
index 7a4d36d8..786e1977 100644
--- a/src/lib/tests/kex_test.c
+++ b/src/lib/tests/kex_test.c
@@ -241,6 +241,7 @@ static int test_kex_dhe_derive(const char * algo)
 
         memset(&kex, 0, sizeof(kex));
         SET_KEX_ALGO(&kex, algo);
+        SET_KEX_KDF_NID(&kex, NID_sha256);
 
         len = kex_pkp_create(&kex, &pkp1, buf1);
         if (len < 0) {
@@ -352,6 +353,7 @@ static int test_kex_dhe_corrupted_pubkey(const char * algo)
 
         memset(&kex, 0, sizeof(kex));
         SET_KEX_ALGO(&kex, algo);
+        SET_KEX_KDF_NID(&kex, NID_sha256);
 
         len = kex_pkp_create(&kex, &pkp, buf);
         if (len < 0) {
@@ -403,6 +405,8 @@ static int test_kex_dhe_wrong_algo(void)
         memset(&kex2, 0, sizeof(kex2));
         SET_KEX_ALGO(&kex1, algo1);
         SET_KEX_ALGO(&kex2, algo2);
+        SET_KEX_KDF_NID(&kex1, NID_sha256);
+        SET_KEX_KDF_NID(&kex2, NID_sha256);
 
         if (kex_pkp_create(&kex1, &pkp1, buf1) < 0) {
                 printf("Failed to create first key pair.\n");