Use openssl_err_t typedef to deal with difference between TLS libraries

AWS-LC and OpenSSL disagree on the type of that errors are reported in.

Instead of having a lot of glue code and casting back and forth, use a
typedef to always use the right type.

Change-Id: I4adbdf0c8b82fd7de309aa5f6f3b0c8157c5ffe7
Signed-off-by: Arne Schwabe <arne@rfc2549.org>
Acked-by: Frank Lichtenheld <frank@lichtenheld.com>
Gerrit URL: https://gerrit.openvpn.net/c/openvpn/+/1578
Message-Id: <20260322111131.8251-1-gert@greenie.muc.de>
URL: https://www.mail-archive.com/openvpn-devel@lists.sourceforge.net/msg36242.html
Signed-off-by: Gert Doering <gert@greenie.muc.de>

src/openvpn/crypto_openssl.c

@@ -229,7 +229,7 @@ crypto_clear_error(void)
 void
 crypto_print_openssl_errors(const unsigned int flags)
 {
-    unsigned long err = 0;
+    openssl_err_t err = 0;
     int line, errflags;
     const char *file, *data, *func;
 

src/openvpn/openssl_compat.h

@@ -45,6 +45,15 @@
 #include <openssl/x509.h>
 #include <openssl/err.h>
 
+/* Define the type of error. This is something that is less
+ * intrusive than casts everywhere */
+#if defined(OPENSSL_IS_AWSLC)
+typedef uint32_t openssl_err_t;
+#else
+typedef unsigned long openssl_err_t;
+#endif
+
+
 /* Functionality missing in 1.1.0 */
 #if OPENSSL_VERSION_NUMBER < 0x10101000L && !defined(ENABLE_CRYPTO_WOLFSSL)
 #define SSL_CTX_set1_groups SSL_CTX_set1_curves
@@ -157,12 +166,12 @@ EVP_MD_free(const EVP_MD *md)
     /* OpenSSL 1.1.1 and lower use only const EVP_MD, nothing to free */
 }
 
-static inline unsigned long
+static inline openssl_err_t
 ERR_get_error_all(const char **file, int *line, const char **func, const char **data, int *flags)
 {
     static const char *empty = "";
     *func = empty;
-    unsigned long err = ERR_get_error_line_data(file, line, data, flags);
+    openssl_err_t err = ERR_get_error_line_data(file, line, data, flags);
     return err;
 }