md5: allow disabling old-style encrypted private keys at build-time

Before this patch, this happened at runtime when using an old (pre-3.0),
FIPS-enabled OpenSSL backend.

This patch makes it possible to disable this via the build-time option
`LIBSSH2_NO_MD5_PEM`.

Also:
- make sure to exclude all MD5 internal APIs when both the above and
  `LIBSSH2_NO_MD5` are enabled.
- fix tests to support build with`LIBSSH2_NO_MD5`, `LIBSSH2_NO_MD5_PEM`
  and `LIBSSH2_NO_3DES`.
- add FIXME to apply this change to `os400qc3.*`.

Old-style encrypted private keys require MD5 and they look like this:
```
-----BEGIN RSA PRIVATE KEY-----
 Proc-Type: 4,ENCRYPTED
 DEK-Info: AES-128-CBC,<MD5-hex>

 <base64>
 -----END RSA PRIVATE KEY-----
```

E.g.: `tests/key_rsa_encrypted`

Ref: libssh2/www#20
Closes libssh2#1181

Author: vszakats

.github/workflows/ci.yml

@@ -322,7 +322,7 @@ jobs:
       - name: 'make build'
         if: ${{ matrix.build == 'make' }}
         env:
-          CPPFLAGS: -DLIBSSH2_NO_MD5 -DLIBSSH2_NO_HMAC_RIPEMD -DLIBSSH2_NO_DSA -DLIBSSH2_NO_AES_CBC -DLIBSSH2_NO_AES_CTR -DLIBSSH2_NO_BLOWFISH -DLIBSSH2_NO_RC4 -DLIBSSH2_NO_CAST -DLIBSSH2_NO_3DES
+          CPPFLAGS: -DLIBSSH2_NO_MD5 -DLIBSSH2_NO_MD5_PEM -DLIBSSH2_NO_HMAC_RIPEMD -DLIBSSH2_NO_DSA -DLIBSSH2_NO_AES_CBC -DLIBSSH2_NO_AES_CTR -DLIBSSH2_NO_BLOWFISH -DLIBSSH2_NO_RC4 -DLIBSSH2_NO_CAST -DLIBSSH2_NO_3DES
           LIBSSH2_CPPFLAGS_LIB: -DLIBSSH2_EXPORTS
           ZLIB_PATH: /${{ matrix.sys }}
           OPENSSL_PATH: /${{ matrix.sys }}

src/Makefile.inc

@@ -29,6 +29,7 @@ HHEADERS =              \
   channel.h             \
   comp.h                \
   crypto.h              \
+  crypto_config.h       \
   libgcrypt.h           \
   libssh2_priv.h        \
   libssh2_setup.h       \

src/crypto.h

@@ -55,71 +55,6 @@
 #error "no cryptography backend selected"
 #endif
 
-#ifdef LIBSSH2_NO_MD5
-#undef LIBSSH2_MD5
-#define LIBSSH2_MD5 0
-#endif
-
-#ifdef LIBSSH2_NO_HMAC_RIPEMD
-#undef LIBSSH2_HMAC_RIPEMD
-#define LIBSSH2_HMAC_RIPEMD 0
-#endif
-
-#ifdef LIBSSH2_NO_DSA
-#undef LIBSSH2_DSA
-#define LIBSSH2_DSA 0
-#endif
-
-#ifdef LIBSSH2_NO_RSA
-#undef LIBSSH2_RSA
-#define LIBSSH2_RSA 0
-#endif
-
-#ifdef LIBSSH2_NO_RSA_SHA1
-#undef LIBSSH2_RSA_SHA1
-#define LIBSSH2_RSA_SHA1 0
-#endif
-
-#ifdef LIBSSH2_NO_ECDSA
-#undef LIBSSH2_ECDSA
-#define LIBSSH2_ECDSA 0
-#endif
-
-#ifdef LIBSSH2_NO_ED25519
-#undef LIBSSH2_ED25519
-#define LIBSSH2_ED25519 0
-#endif
-
-#ifdef LIBSSH2_NO_AES_CTR
-#undef LIBSSH2_AES_CTR
-#define LIBSSH2_AES_CTR 0
-#endif
-
-#ifdef LIBSSH2_NO_AES_CBC
-#undef LIBSSH2_AES_CBC
-#define LIBSSH2_AES_CBC 0
-#endif
-
-#ifdef LIBSSH2_NO_BLOWFISH
-#undef LIBSSH2_BLOWFISH
-#define LIBSSH2_BLOWFISH 0
-#endif
-
-#ifdef LIBSSH2_NO_RC4
-#undef LIBSSH2_RC4
-#define LIBSSH2_RC4 0
-#endif
-
-#ifdef LIBSSH2_NO_CAST
-#undef LIBSSH2_CAST
-#define LIBSSH2_CAST 0
-#endif
-
-#ifdef LIBSSH2_NO_3DES
-#undef LIBSSH2_3DES
-#define LIBSSH2_3DES 0
-#endif
-
 #define LIBSSH2_ED25519_KEY_LEN 32
 #define LIBSSH2_ED25519_PRIVATE_KEY_LEN 64
 #define LIBSSH2_ED25519_SIG_LEN 64

src/crypto_config.h

@@ -0,0 +1,76 @@
+/* Copyright (C) Viktor Szakats
+ *
+ * SPDX-License-Identifier: BSD-3-Clause
+ */
+
+#define LIBSSH2_MD5_PEM LIBSSH2_MD5
+
+#ifdef LIBSSH2_NO_MD5
+#undef LIBSSH2_MD5
+#define LIBSSH2_MD5 0
+#endif
+
+#ifdef LIBSSH2_NO_MD5_PEM
+#undef LIBSSH2_MD5_PEM
+#define LIBSSH2_MD5_PEM 0
+#endif
+
+#ifdef LIBSSH2_NO_HMAC_RIPEMD
+#undef LIBSSH2_HMAC_RIPEMD
+#define LIBSSH2_HMAC_RIPEMD 0
+#endif
+
+#ifdef LIBSSH2_NO_DSA
+#undef LIBSSH2_DSA
+#define LIBSSH2_DSA 0
+#endif
+
+#ifdef LIBSSH2_NO_RSA
+#undef LIBSSH2_RSA
+#define LIBSSH2_RSA 0
+#endif
+
+#ifdef LIBSSH2_NO_RSA_SHA1
+#undef LIBSSH2_RSA_SHA1
+#define LIBSSH2_RSA_SHA1 0
+#endif
+
+#ifdef LIBSSH2_NO_ECDSA
+#undef LIBSSH2_ECDSA
+#define LIBSSH2_ECDSA 0
+#endif
+
+#ifdef LIBSSH2_NO_ED25519
+#undef LIBSSH2_ED25519
+#define LIBSSH2_ED25519 0
+#endif
+
+#ifdef LIBSSH2_NO_AES_CTR
+#undef LIBSSH2_AES_CTR
+#define LIBSSH2_AES_CTR 0
+#endif
+
+#ifdef LIBSSH2_NO_AES_CBC
+#undef LIBSSH2_AES_CBC
+#define LIBSSH2_AES_CBC 0
+#endif
+
+#ifdef LIBSSH2_NO_BLOWFISH
+#undef LIBSSH2_BLOWFISH
+#define LIBSSH2_BLOWFISH 0
+#endif
+
+#ifdef LIBSSH2_NO_RC4
+#undef LIBSSH2_RC4
+#define LIBSSH2_RC4 0
+#endif
+
+#ifdef LIBSSH2_NO_CAST
+#undef LIBSSH2_CAST
+#define LIBSSH2_CAST 0
+#endif
+
+#ifdef LIBSSH2_NO_3DES
+#undef LIBSSH2_3DES
+#define LIBSSH2_3DES 0
+#endif

src/libgcrypt.h

@@ -66,7 +66,11 @@
 #define LIBSSH2_ECDSA 0
 #define LIBSSH2_ED25519 0
 
+#include "crypto_config.h"
+
+#if LIBSSH2_MD5 || LIBSSH2_MD5_PEM
 #define MD5_DIGEST_LENGTH 16
+#endif
 #define SHA_DIGEST_LENGTH 20
 #define SHA256_DIGEST_LENGTH 32
 #define SHA384_DIGEST_LENGTH 48
@@ -124,6 +128,7 @@
 #define libssh2_sha512(message, len, out) \
     gcry_md_hash_buffer(GCRY_MD_SHA512, out, message, len)
 
+#if LIBSSH2_MD5 || LIBSSH2_MD5_PEM
 #define libssh2_md5_ctx gcry_md_hd_t
 
 /* returns 0 in case of failure */
@@ -136,15 +141,18 @@
     memcpy(out, gcry_md_read(ctx, 0), MD5_DIGEST_LENGTH), gcry_md_close(ctx)
 #define libssh2_md5(message, len, out) \
     gcry_md_hash_buffer(GCRY_MD_MD5, out, message, len)
+#endif
 
 #define libssh2_hmac_ctx gcry_md_hd_t
 #define libssh2_hmac_ctx_init(ctx)
 #define libssh2_hmac_sha1_init(ctx, key, keylen) \
     gcry_md_open(ctx, GCRY_MD_SHA1, GCRY_MD_FLAG_HMAC), \
     gcry_md_setkey(*ctx, key, keylen)
+#if LIBSSH2_MD5
 #define libssh2_hmac_md5_init(ctx, key, keylen) \
     gcry_md_open(ctx, GCRY_MD_MD5, GCRY_MD_FLAG_HMAC), \
     gcry_md_setkey(*ctx, key, keylen)
+#endif
 #define libssh2_hmac_ripemd160_init(ctx, key, keylen) \
     gcry_md_open(ctx, GCRY_MD_RMD160, GCRY_MD_FLAG_HMAC), \
     gcry_md_setkey(*ctx, key, keylen)

src/mbedtls.h

@@ -92,7 +92,11 @@
 #endif
 #define LIBSSH2_ED25519         0
 
+#include "crypto_config.h"
+
+#if LIBSSH2_MD5 || LIBSSH2_MD5_PEM
 #define MD5_DIGEST_LENGTH      16
+#endif
 #define SHA_DIGEST_LENGTH      20
 #define SHA256_DIGEST_LENGTH   32
 #define SHA384_DIGEST_LENGTH   48
@@ -134,8 +138,10 @@
 
 #define libssh2_hmac_sha1_init(pctx, key, keylen) \
     _libssh2_mbedtls_hash_init(pctx, MBEDTLS_MD_SHA1, key, keylen)
+#if LIBSSH2_MD5
 #define libssh2_hmac_md5_init(pctx, key, keylen) \
     _libssh2_mbedtls_hash_init(pctx, MBEDTLS_MD_MD5, key, keylen)
+#endif
 #define libssh2_hmac_ripemd160_init(pctx, key, keylen) \
     _libssh2_mbedtls_hash_init(pctx, MBEDTLS_MD_RIPEMD160, key, keylen)
 #define libssh2_hmac_sha256_init(pctx, key, keylen) \
@@ -219,6 +225,7 @@
  * mbedTLS backend: MD5 functions
  */
 
+#if LIBSSH2_MD5 || LIBSSH2_MD5_PEM
 #define libssh2_md5_ctx      mbedtls_md_context_t
 
 #define libssh2_md5_init(pctx) \
@@ -229,7 +236,7 @@
     _libssh2_mbedtls_hash_final(&ctx, hash)
 #define libssh2_md5(data, datalen, hash) \
     _libssh2_mbedtls_hash(data, datalen, MBEDTLS_MD_MD5, hash)
-
+#endif
 
 /*******************************************************************/
 /*

src/openssl.c

@@ -2495,6 +2495,7 @@ _libssh2_sha512(const unsigned char *message, size_t len,
     return 1; /* error */
 }
 
+#if LIBSSH2_MD5 || LIBSSH2_MD5_PEM
 int
 _libssh2_md5_init(libssh2_md5_ctx *ctx)
 {
@@ -2530,6 +2531,7 @@ _libssh2_md5_init(libssh2_md5_ctx *ctx)
     return EVP_DigestInit(ctx, EVP_get_digestbyname("md5"));
 #endif
 }
+#endif
 
 #if LIBSSH2_ECDSA
 

src/openssl.h

@@ -194,6 +194,8 @@
 # define LIBSSH2_3DES 1
 #endif
 
+#include "crypto_config.h"
+
 #define EC_MAX_POINT_LEN ((528 * 2 / 8) + 1)
 
 #define _libssh2_random(buf, len) \
@@ -296,6 +298,7 @@ int _libssh2_sha512(const unsigned char *message, size_t len,
                     unsigned char *out);
 #define libssh2_sha512(x,y,z) _libssh2_sha512(x,y,z)
 
+#if LIBSSH2_MD5 || LIBSSH2_MD5_PEM
 #ifdef HAVE_OPAQUE_STRUCTS
 #define libssh2_md5_ctx EVP_MD_CTX *
 #else
@@ -315,6 +318,7 @@ int _libssh2_md5_init(libssh2_md5_ctx *ctx);
 #define libssh2_md5_update(ctx, data, len) EVP_DigestUpdate(&(ctx), data, len)
 #define libssh2_md5_final(ctx, out) EVP_DigestFinal(&(ctx), out, NULL)
 #endif
+#endif
 
 #ifdef HAVE_OPAQUE_STRUCTS
 #define libssh2_hmac_ctx HMAC_CTX *

src/os400qc3.h

@@ -164,6 +164,8 @@
 #undef Qc3_MK_Pending
 #define Qc3_MK_Pending      '\xF3'      /* '3' */
 
+/* FIXME: Disable MD5 macros/constants and functions when
+          LIBSSH2_MD5 and LIBSSH_MD5_PEM have the value 0. */
 
 /* Define which features are supported. */
 #define LIBSSH2_MD5             1
@@ -186,6 +188,8 @@
 #define LIBSSH2_ECDSA           0
 #define LIBSSH2_ED25519         0
 
+#include "crypto_config.h"
+
 #define MD5_DIGEST_LENGTH       16
 #define SHA_DIGEST_LENGTH       20
 #define SHA256_DIGEST_LENGTH    32

src/pem.c

@@ -106,12 +106,6 @@ static unsigned char hex_decode(char digit)
         ((digit >= 'A') ? (0xA + (digit - 'A')) : (digit - '0'));
 }
 
-/* Hack to fix builds with crypto backends with MD5 support disabled.
-   FIXME: Honor our LIBSSH2_MD5 macro for MD5-dependent logic. */
-#ifdef OPENSSL_NO_MD5
-#define MD5_DIGEST_LENGTH 16
-#endif
-
 int
 _libssh2_pem_parse(LIBSSH2_SESSION * session,
                    const char *headerbegin,
@@ -215,6 +209,7 @@ _libssh2_pem_parse(LIBSSH2_SESSION * session,
     }
 
     if(method) {
+#if LIBSSH2_MD5_PEM
         /* Set up decryption */
         int free_iv = 0, free_secret = 0, len_decrypted = 0, padding = 0;
         int blocksize = method->blocksize;
@@ -292,6 +287,10 @@ _libssh2_pem_parse(LIBSSH2_SESSION * session,
         /* Clean up */
         _libssh2_explicit_zero((char *)secret, sizeof(secret));
         method->dtor(session, &abstract);
+#else
+        ret = -1;
+        goto out;
+#endif
     }
 
     ret = 0;

src/wincng.c

@@ -94,9 +94,11 @@
 #define BCRYPT_RNG_ALGORITHM L"RNG"
 #endif
 
+#if LIBSSH2_MD5 || LIBSSH2_MD5_PEM
 #ifndef BCRYPT_MD5_ALGORITHM
 #define BCRYPT_MD5_ALGORITHM L"MD5"
 #endif
+#endif
 
 #ifndef BCRYPT_SHA1_ALGORITHM
 #define BCRYPT_SHA1_ALGORITHM L"SHA1"
@@ -254,11 +256,13 @@ _libssh2_wincng_init(void)
         _libssh2_wincng.hAlgRNG = NULL;
     }
 
+#if LIBSSH2_MD5 || LIBSSH2_MD5_PEM
     ret = BCryptOpenAlgorithmProvider(&_libssh2_wincng.hAlgHashMD5,
                                       BCRYPT_MD5_ALGORITHM, NULL, 0);
     if(!BCRYPT_SUCCESS(ret)) {
         _libssh2_wincng.hAlgHashMD5 = NULL;
     }
+#endif
     ret = BCryptOpenAlgorithmProvider(&_libssh2_wincng.hAlgHashSHA1,
                                       BCRYPT_SHA1_ALGORITHM, NULL, 0);
     if(!BCRYPT_SUCCESS(ret)) {
@@ -280,12 +284,14 @@ _libssh2_wincng_init(void)
         _libssh2_wincng.hAlgHashSHA512 = NULL;
     }
 
+#if LIBSSH2_MD5
     ret = BCryptOpenAlgorithmProvider(&_libssh2_wincng.hAlgHmacMD5,
                                       BCRYPT_MD5_ALGORITHM, NULL,
                                       BCRYPT_ALG_HANDLE_HMAC_FLAG);
     if(!BCRYPT_SUCCESS(ret)) {
         _libssh2_wincng.hAlgHmacMD5 = NULL;
     }
+#endif
     ret = BCryptOpenAlgorithmProvider(&_libssh2_wincng.hAlgHmacSHA1,
                                       BCRYPT_SHA1_ALGORITHM, NULL,
                                       BCRYPT_ALG_HANDLE_HMAC_FLAG);
@@ -395,8 +401,10 @@ _libssh2_wincng_free(void)
 {
     if(_libssh2_wincng.hAlgRNG)
         (void)BCryptCloseAlgorithmProvider(_libssh2_wincng.hAlgRNG, 0);
+#if LIBSSH2_MD5 || LIBSSH2_MD5_PEM
     if(_libssh2_wincng.hAlgHashMD5)
         (void)BCryptCloseAlgorithmProvider(_libssh2_wincng.hAlgHashMD5, 0);
+#endif
     if(_libssh2_wincng.hAlgHashSHA1)
         (void)BCryptCloseAlgorithmProvider(_libssh2_wincng.hAlgHashSHA1, 0);
     if(_libssh2_wincng.hAlgHashSHA256)
@@ -405,8 +413,10 @@ _libssh2_wincng_free(void)
         (void)BCryptCloseAlgorithmProvider(_libssh2_wincng.hAlgHashSHA384, 0);
     if(_libssh2_wincng.hAlgHashSHA512)
         (void)BCryptCloseAlgorithmProvider(_libssh2_wincng.hAlgHashSHA512, 0);
+#if LIBSSH2_MD5
     if(_libssh2_wincng.hAlgHmacMD5)
         (void)BCryptCloseAlgorithmProvider(_libssh2_wincng.hAlgHmacMD5, 0);
+#endif
     if(_libssh2_wincng.hAlgHmacSHA1)
         (void)BCryptCloseAlgorithmProvider(_libssh2_wincng.hAlgHmacSHA1, 0);
     if(_libssh2_wincng.hAlgHmacSHA256)