ssl_mbedtls: Avoid conversion and sign-compare warnings

Change-Id: I777a3a5f4f137432a19746972e2aad1732184feb
Signed-off-by: Frank Lichtenheld <frank@lichtenheld.com>
Acked-by: Gert Doering <gert@greenie.muc.de>
Gerrit URL: https://gerrit.openvpn.net/c/openvpn/+/1559
Message-Id: <20260314170948.2898-1-gert@greenie.muc.de>
URL: https://www.mail-archive.com/openvpn-devel@lists.sourceforge.net/msg36137.html
Signed-off-by: Gert Doering <gert@greenie.muc.de>

src/openvpn/ssl_mbedtls.c

@@ -624,12 +624,6 @@ tls_ctx_load_priv_file(struct tls_root_ctx *ctx, const char *priv_key_file, bool
     return 0;
 }
 
-#if defined(__GNUC__) || defined(__clang__)
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wconversion"
-#pragma GCC diagnostic ignored "-Wsign-compare"
-#endif
-
 #if MBEDTLS_VERSION_NUMBER < 0x04000000
 /**
  * external_pkcs1_sign implements a mbed TLS rsa_sign_func callback, that uses
@@ -656,7 +650,6 @@ external_pkcs1_sign(void *ctx_voidptr, int (*f_rng)(void *, unsigned char *, siz
 {
     struct external_context *const ctx = ctx_voidptr;
     int rv;
-    uint8_t *to_sign = NULL;
     size_t asn_len = 0, oid_size = 0;
     const char *oid = NULL;
 
@@ -688,11 +681,13 @@ external_pkcs1_sign(void *ctx_voidptr, int (*f_rng)(void *, unsigned char *, siz
         asn_len = 10 + oid_size;
     }
 
-    if ((SIZE_MAX - hashlen) < asn_len || ctx->signature_length < (asn_len + hashlen))
+    if (ctx->signature_length < (asn_len + hashlen)
+        || (asn_len + hashlen) > UINT8_MAX)
     {
         return MBEDTLS_ERR_RSA_BAD_INPUT_DATA;
     }
 
+    uint8_t *to_sign = NULL;
     ALLOC_ARRAY_CLEAR(to_sign, uint8_t, asn_len + hashlen);
     uint8_t *p = to_sign;
     if (md_alg != MBEDTLS_MD_NONE)
@@ -707,20 +702,20 @@ external_pkcs1_sign(void *ctx_voidptr, int (*f_rng)(void *, unsigned char *, siz
          * Digest ::= OCTET STRING
          */
         *p++ = MBEDTLS_ASN1_SEQUENCE | MBEDTLS_ASN1_CONSTRUCTED;
-        *p++ = (unsigned char)(0x08 + oid_size + hashlen);
+        *p++ = (uint8_t)(0x08 + oid_size + hashlen);
         *p++ = MBEDTLS_ASN1_SEQUENCE | MBEDTLS_ASN1_CONSTRUCTED;
-        *p++ = (unsigned char)(0x04 + oid_size);
+        *p++ = (uint8_t)(0x04 + oid_size);
         *p++ = MBEDTLS_ASN1_OID;
-        *p++ = oid_size & 0xFF;
+        *p++ = (uint8_t)oid_size;
         memcpy(p, oid, oid_size);
         p += oid_size;
         *p++ = MBEDTLS_ASN1_NULL;
         *p++ = 0x00;
         *p++ = MBEDTLS_ASN1_OCTET_STRING;
-        *p++ = hashlen;
+        *p++ = (uint8_t)hashlen;
 
         /* Double-check ASN length */
-        ASSERT(asn_len == p - to_sign);
+        ASSERT(asn_len == (uintptr_t)(p - to_sign));
     }
 
     /* Copy the hash to be signed */
@@ -810,7 +805,7 @@ management_sign_func(void *sign_ctx, const void *src, size_t src_len, void *dst,
         goto cleanup;
     }
 
-    if (openvpn_base64_decode(dst_b64, dst, (int)dst_len) != dst_len)
+    if (openvpn_base64_decode(dst_b64, dst, (int)dst_len) != (int)dst_len)
     {
         goto cleanup;
     }
@@ -923,6 +918,8 @@ endless_buf_read(endless_buffer *in, unsigned char *out, size_t out_len)
 {
     size_t read_len = 0;
 
+    ASSERT(out_len <= INT_MAX);
+
     if (in->first_block == NULL)
     {
         return MBEDTLS_ERR_SSL_WANT_READ;
@@ -930,7 +927,7 @@ endless_buf_read(endless_buffer *in, unsigned char *out, size_t out_len)
 
     while (in->first_block != NULL && read_len < out_len)
     {
-        int block_len = in->first_block->length - in->data_start;
+        size_t block_len = in->first_block->length - in->data_start;
         if (block_len <= out_len - read_len)
         {
             buffer_entry *cur_entry = in->first_block;
@@ -956,7 +953,7 @@ endless_buf_read(endless_buffer *in, unsigned char *out, size_t out_len)
         }
     }
 
-    return read_len;
+    return (int)read_len;
 }
 
 static int
@@ -975,6 +972,8 @@ endless_buf_write(endless_buffer *out, const unsigned char *in, size_t len)
         return MBEDTLS_ERR_NET_SEND_FAILED;
     }
 
+    ASSERT(len <= INT_MAX);
+
     new_block->length = len;
     new_block->next_block = NULL;
 
@@ -992,7 +991,7 @@ endless_buf_write(endless_buffer *out, const unsigned char *in, size_t len)
 
     out->last_block = new_block;
 
-    return len;
+    return (int)len;
 }
 
 static int
@@ -1012,7 +1011,7 @@ ssl_bio_write(void *ctx, const unsigned char *in, size_t in_len)
 static void
 my_debug(void *ctx, int level, const char *file, int line, const char *str)
 {
-    int my_loglevel = (level < 3) ? D_TLS_DEBUG_MED : D_TLS_DEBUG;
+    msglvl_t my_loglevel = (level < 3) ? D_TLS_DEBUG_MED : D_TLS_DEBUG;
     msg(my_loglevel, "mbed TLS msg (%s:%d): %s", file, line, str);
 }
 
@@ -1048,10 +1047,6 @@ tls_ctx_personalise_random(struct tls_root_ctx *ctx)
 #endif /* MBEDTLS_VERSION_NUMBER < 0x040000 */
 }
 
-#if defined(__GNUC__) || defined(__clang__)
-#pragma GCC diagnostic pop
-#endif
-
 int
 tls_version_max(void)
 {

src/openvpn/ssl_verify_mbedtls.c

@@ -565,11 +565,6 @@ x509_setenv_track(const struct x509_track *xt, struct env_set *es, const int dep
     gc_free(&gc);
 }
 
-#if defined(__GNUC__) || defined(__clang__)
-#pragma GCC diagnostic push
-#pragma GCC diagnostic ignored "-Wsign-compare"
-#endif
-
 /*
  * Save X509 fields to environment, using the naming convention:
  *
@@ -678,10 +673,6 @@ x509_verify_cert_ku(mbedtls_x509_crt *cert, const unsigned int *const expected_k
     return fFound;
 }
 
-#if defined(__GNUC__) || defined(__clang__)
-#pragma GCC diagnostic pop
-#endif
-
 result_t
 x509_verify_cert_eku(mbedtls_x509_crt *cert, const char *const expected_oid)
 {