From e291429c5bc14a0fbe09ba9f95fba68ba9aed4f2 Mon Sep 17 00:00:00 2001
From: "Pedro F. Giffuni" <pfg@FreeBSD.org>
Date: Sun, 15 Feb 2015 14:31:50 +0000
Subject: [PATCH] ulimit(3): Fix broken check.

The existing implementation had a broken comparison that could
overflow and return confusing values.  Replace this with a check
that avoids the overflow before it happens.

Consistently return a maximum value also on the case of negative
arguments since negative is considered an overflow and means
infinity for our current setrlimit().

New revamped version is credited to Bruce Evans.

CID:		1199295
MFC after:	1 week
---
 lib/libc/gen/ulimit.c | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/lib/libc/gen/ulimit.c b/lib/libc/gen/ulimit.c
index e1bc0206620..db1d4988b71 100644
--- a/lib/libc/gen/ulimit.c
+++ b/lib/libc/gen/ulimit.c
@@ -33,6 +33,7 @@
 #include <errno.h>
 #include <limits.h>
 #include <stdarg.h>
+#include <stdint.h>
 #include <ulimit.h>
 
 long
@@ -40,6 +41,7 @@ ulimit(int cmd, ...)
 {
 	struct rlimit limit;
 	va_list ap;
+	volatile intmax_t targ;
 	long arg;
 
 	if (cmd == UL_GETFSIZE) {
@@ -51,16 +53,18 @@ ulimit(int cmd, ...)
 		return ((long)limit.rlim_cur);
 	} else if (cmd == UL_SETFSIZE) {
 		va_start(ap, cmd);
-		arg = va_arg(ap, long);
+		targ = arg = va_arg(ap, long);
 		va_end(ap);
-		limit.rlim_max = limit.rlim_cur = (rlim_t)arg * 512;
+		if (targ < 0)
+			targ = LONG_MAX;
+		if (targ > RLIM_INFINITY / 512)
+			targ = RLIM_INFINITY / 512;
+		limit.rlim_max = limit.rlim_cur = targ * 512;
 
 		/* The setrlimit() function sets errno to EPERM if needed. */
 		if (setrlimit(RLIMIT_FSIZE, &limit) == -1)
 			return (-1);
-		if (arg * 512 > LONG_MAX)
-			return (LONG_MAX);
-		return (arg);
+		return ((long)targ);
 	} else {
 		errno = EINVAL;
 		return (-1);