crypto_backend: fix type of enc parameter

We had parts of a abstraction, but it wasn't consistent.
GCC 13 now complains about the type mismatch with mbedtls now:

crypto_mbedtls.c:568:1: error:
conflicting types for ‘cipher_ctx_init’ due to enum/integer mismatch;
have ‘void(mbedtls_cipher_context_t *, const uint8_t *, const char *, const mbedtls_operation_t)’
[...] [-Werror=enum-int-mismatch]
crypto_backend.h:341:6: note:
previous declaration of ‘cipher_ctx_init’ with type
‘void(cipher_ctx_t *, const uint8_t *, const char *, int)’ [...]

Previous compiler versions did not complain.

v2:
 - clean solution instead of quick solution. Fix the actual API
   definition

Change-Id: If0dcdde30879fd6185efb2ad31399c1629c04d22
Signed-off-by: Frank Lichtenheld <[email protected]>
Acked-by: Arne Schwabe <[email protected]>
Message-Id: <[email protected]>
URL: https://www.mail-archive.com/[email protected]/msg28498.html
Signed-off-by: Gert Doering <[email protected]>
(cherry picked from commit 4d907bf)

Author: flichtenheld

src/openvpn/crypto_backend.h

@@ -347,10 +347,10 @@ void cipher_ctx_free(cipher_ctx_t *ctx);
  * @param key           Buffer containing the key to use
  * @param ciphername    Ciphername of the cipher to use
  * @param enc           Whether to encrypt or decrypt (either
- *                      \c MBEDTLS_OP_ENCRYPT or \c MBEDTLS_OP_DECRYPT).
+ *                      \c OPENVPN_OP_ENCRYPT or \c OPENVPN_OP_DECRYPT).
  */
 void cipher_ctx_init(cipher_ctx_t *ctx, const uint8_t *key,
-                     const char *cipername, int enc);
+                     const char *cipername, crypto_operation_t enc);
 
 /**
  * Returns the size of the IV used by the cipher, in bytes, or 0 if no IV is

src/openvpn/crypto_mbedtls.c

@@ -566,7 +566,7 @@ cipher_ctx_free(mbedtls_cipher_context_t *ctx)
 
 void
 cipher_ctx_init(mbedtls_cipher_context_t *ctx, const uint8_t *key,
-                const char *ciphername, const mbedtls_operation_t operation)
+                const char *ciphername, crypto_operation_t enc)
 {
     ASSERT(NULL != ciphername && NULL != ctx);
     CLEAR(*ctx);
@@ -580,7 +580,7 @@ cipher_ctx_init(mbedtls_cipher_context_t *ctx, const uint8_t *key,
         msg(M_FATAL, "mbed TLS cipher context init #1");
     }
 
-    if (!mbed_ok(mbedtls_cipher_setkey(ctx, key, (int)key_bitlen, operation)))
+    if (!mbed_ok(mbedtls_cipher_setkey(ctx, key, (int)key_bitlen, enc)))
     {
         msg(M_FATAL, "mbed TLS cipher set key");
     }

src/openvpn/crypto_mbedtls.h

@@ -63,6 +63,8 @@ typedef void provider_t;
 /** Cipher is in GCM mode */
 #define OPENVPN_MODE_GCM        MBEDTLS_MODE_GCM
 
+typedef mbedtls_operation_t crypto_operation_t;
+
 /** Cipher should encrypt */
 #define OPENVPN_OP_ENCRYPT      MBEDTLS_ENCRYPT
 

src/openvpn/crypto_openssl.c

@@ -863,7 +863,7 @@ cipher_ctx_free(EVP_CIPHER_CTX *ctx)
 
 void
 cipher_ctx_init(EVP_CIPHER_CTX *ctx, const uint8_t *key,
-                const char *ciphername, int enc)
+                const char *ciphername, crypto_operation_t enc)
 {
     ASSERT(NULL != ciphername && NULL != ctx);
     evp_cipher_type *kt = cipher_get(ciphername);

src/openvpn/crypto_openssl.h

@@ -85,6 +85,8 @@ typedef EVP_MD evp_md_type;
 /** Cipher is in GCM mode */
 #define OPENVPN_MODE_GCM        EVP_CIPH_GCM_MODE
 
+typedef int crypto_operation_t;
+
 /** Cipher should encrypt */
 #define OPENVPN_OP_ENCRYPT      1