10826 use cmp and abs to find larger NaN payload (#10895)

`getNaNPayload(x)` returns a value even if `x` is not NaN, so it is up to
the caller to verify that `x` is a NaN before calling `getNaNPayload(x)`.
This means it is not a good option for determining for determining which
of a set of values has the largest NaN payload when it is only known
that at least one value is NaN. `cmp(x, y)` can be used. since to orders
-NaN as before -infinity and NaN as after infinity. When two NaN with the
same sign are compared, the one with the larger payload is considered
larger. This means that `cmp(abs(x), abx(y))` will find the NaN with the
larger payload when it is only known that at least one of `x` or `y` is
NaN.

Author: tedgin

std/internal/math/gammafunction.d

@@ -656,7 +656,7 @@ real beta(in real x, in real y)
 {
     // When one of the input parameters is NaN, return the NaN with the larger
     // payload. This mimics the behavior of the + operator.
-    if (isNaN(x) || isNaN(y)) return getNaNPayload(x) >= getNaNPayload(y) ? x : y;
+    if (isNaN(x) || isNaN(y)) return cmp(abs(x), abs(y)) >= 0 ? x : y;
 
     real res;
 
@@ -764,7 +764,7 @@ real beta(in real x, in real y)
 @safe unittest
 {
     // Test NaN payload propagation
-    assert(isIdentical(beta(NaN(0xABC), 2), NaN(0xABC)));
+    assert(isIdentical(beta(NaN(0x1), 7), NaN(0x1)));
     assert(isIdentical(beta(2, NaN(0xABC)), NaN(0xABC)));
     assert(isIdentical(beta(NaN(0x1), NaN(0x2)), NaN(0x2)));