Review fix on #361: range-guard before the int64 cast in the hash normalizer

The value-normalizing hash cast the double to int64 in the FIRST
conjunct, before the range checks - UB for NaN, inf, and |x| >= 2^63
(UBSan float-cast-overflow abort at scalar_number.h:117, the very
class this PR removes elsewhere). NaN/inf now short-circuit via the
range comparisons (false for NaN) before any cast. Regression test
hashes 1e300/NaN/inf constants.

Signed-off-by: petlenz <peterlenz89.pl@gmail.com>

Author: petlenz

include/numsim_cas/core/scalar_number.h

@@ -114,8 +114,10 @@ inline void hash_combine(std::size_t &seed, scalar_number const &value) {
       [&](auto const &x) {
         using T = std::decay_t<decltype(x)>;
         if constexpr (std::is_same_v<T, double>) {
-          if (x == static_cast<double>(static_cast<std::int64_t>(x)) &&
-              x >= -9.2e18 && x <= 9.2e18) {
+          // guard BEFORE casting: the int64 cast is UB for NaN/inf and
+          // |x| >= 2^63 (review on #361)
+          if (x >= -9.2e18 && x <= 9.2e18 &&
+              x == static_cast<double>(static_cast<std::int64_t>(x))) {
             hash_combine(seed, static_cast<std::int64_t>(x));
             return;
           }

tests/CoreBugFixTest.h

@@ -1401,7 +1401,23 @@ TEST(HashCombineDouble, NumericallyEqualConstantsHashEqual) {
   // fractional constants distinct
   auto h1 = make_expression<scalar_constant>(0.5);
   auto h2 = make_expression<scalar_constant>(0.9);
-  EXPECT_NE(h1.get().hash_value(), h2.get().hash_value());}
+  EXPECT_NE(h1.get().hash_value(), h2.get().hash_value());
+}
+
+// Review on #361: the int64 cast in the value-normalizing hash ran before
+// its range guard - UB for NaN, inf, and huge doubles.
+TEST(HashCombineDouble, HugeAndNonFiniteConstantsHashSafely) {
+  auto big = make_expression<scalar_constant>(1e300);
+  auto nan = make_expression<scalar_constant>(
+      std::numeric_limits<double>::quiet_NaN());
+  auto inf =
+      make_expression<scalar_constant>(std::numeric_limits<double>::infinity());
+  // must be UB-free under -fsanitize=float-cast-overflow (CI leg, #356)
+  (void)big.get().hash_value();
+  (void)nan.get().hash_value();
+  (void)inf.get().hash_value();
+  EXPECT_NE(big.get().hash_value(), inf.get().hash_value());
+}
 
 } // namespace numsim::cas