104080 custom rsapss salt length

src/libraries/Common/src/Interop/Unix/System.Security.Cryptography.Native/Interop.EvpPkey.Rsa.cs

@@ -124,6 +124,7 @@ private static partial int CryptoNative_RsaSignHash(
             SafeEvpPKeyHandle pkey,
             IntPtr extraHandle,
             RSASignaturePaddingMode paddingMode,
+            int pssSaltLength,
             IntPtr digestAlgorithm,
             ref byte hash,
             int hashLength,
@@ -133,6 +134,7 @@ private static partial int CryptoNative_RsaSignHash(
         internal static int RsaSignHash(
             SafeEvpPKeyHandle pkey,
             RSASignaturePaddingMode paddingMode,
+            int pssSaltLength,
             HashAlgorithmName digestAlgorithm,
             ReadOnlySpan<byte> hash,
             Span<byte> destination)
@@ -145,6 +147,7 @@ internal static int RsaSignHash(
                 pkey,
                 pkey.ExtraHandle,
                 paddingMode,
+                pssSaltLength,
                 digestAlgorithmPtr,
                 ref MemoryMarshal.GetReference(hash),
                 hash.Length,
@@ -165,6 +168,7 @@ private static partial int CryptoNative_RsaVerifyHash(
             SafeEvpPKeyHandle pkey,
             IntPtr extraHandle,
             RSASignaturePaddingMode paddingMode,
+            int pssSaltLength,
             IntPtr digestAlgorithm,
             ref byte hash,
             int hashLength,
@@ -174,6 +178,7 @@ private static partial int CryptoNative_RsaVerifyHash(
         internal static bool RsaVerifyHash(
             SafeEvpPKeyHandle pkey,
             RSASignaturePaddingMode paddingMode,
+            int pssSaltLength,
             HashAlgorithmName digestAlgorithm,
             ReadOnlySpan<byte> hash,
             ReadOnlySpan<byte> signature)
@@ -186,6 +191,7 @@ internal static bool RsaVerifyHash(
                 pkey,
                 pkey.ExtraHandle,
                 paddingMode,
+                pssSaltLength,
                 digestAlgorithmPtr,
                 ref MemoryMarshal.GetReference(hash),
                 hash.Length,

src/libraries/Common/src/Interop/Windows/BCrypt/Interop.BCryptSignHash.cs

@@ -54,6 +54,7 @@ internal static unsafe NTSTATUS BCryptSignHashPss(
             ReadOnlySpan<byte> hash,
             Span<byte> destination,
             string hashAlgorithmName,
+            int saltLength,
             out int bytesWritten)
         {
             fixed (char* pHashAlgorithmName = hashAlgorithmName)
@@ -62,7 +63,7 @@ internal static unsafe NTSTATUS BCryptSignHashPss(
             {
                 BCRYPT_PSS_PADDING_INFO paddingInfo = default;
                 paddingInfo.pszAlgId = (IntPtr)pHashAlgorithmName;
-                paddingInfo.cbSalt = hash.Length;
+                paddingInfo.cbSalt = saltLength;
 
                 return BCryptSignHash(
                     key,

src/libraries/Common/src/Interop/Windows/BCrypt/Interop.BCryptVerifySignature.cs

@@ -61,7 +61,8 @@ internal static unsafe bool BCryptVerifySignaturePss(
             SafeBCryptKeyHandle key,
             ReadOnlySpan<byte> hash,
             ReadOnlySpan<byte> signature,
-            string hashAlgorithmName)
+            string hashAlgorithmName,
+            int saltLength)
         {
 
             NTSTATUS status;
@@ -72,7 +73,7 @@ internal static unsafe bool BCryptVerifySignaturePss(
             {
                 BCRYPT_PSS_PADDING_INFO paddingInfo = default;
                 paddingInfo.pszAlgId = (IntPtr)pHashAlgorithmName;
-                paddingInfo.cbSalt = hash.Length;
+                paddingInfo.cbSalt = saltLength;
 
                 status = BCryptVerifySignature(
                     key,

src/libraries/Common/src/System/Security/Cryptography/Asn1/PssParamsAsn.manual.cs

@@ -41,6 +41,9 @@ internal RSASignaturePadding GetSignaturePadding(
                         HashAlgorithm.Algorithm));
             }
 
+#if NET10_0_OR_GREATER
+            return RSASignaturePadding.CreatePss(SaltLength);
+#else
             int saltSize = digestValueLength.GetValueOrDefault();
 
             if (!digestValueLength.HasValue)
@@ -59,6 +62,7 @@ internal RSASignaturePadding GetSignaturePadding(
 
             // When RSASignaturePadding supports custom salt sizes this return will look different.
             return RSASignaturePadding.Pss;
+#endif
         }
     }
 }

src/libraries/Common/src/System/Security/Cryptography/RSAAndroid.cs

@@ -657,7 +657,7 @@ private bool TrySignHash(
                 Debug.Assert(padding != null);
                 signature = null;
 
-                if (padding != RSASignaturePadding.Pkcs1 && padding != RSASignaturePadding.Pss)
+                if (padding != RSASignaturePadding.Pkcs1 && padding.Mode != RSASignaturePaddingMode.Pss)
                 {
                     throw PaddingModeNotSupported();
                 }
@@ -688,7 +688,7 @@ private bool TrySignHash(
                 }
                 else if (padding.Mode == RSASignaturePaddingMode.Pss)
                 {
-                    RsaPaddingProcessor.EncodePss(hashAlgorithm, hash, encodedBytes, KeySize);
+                    RsaPaddingProcessor.EncodePss(hashAlgorithm, hash, encodedBytes, KeySize, RsaPaddingProcessor.CalculatePssSaltLength(padding.PssSaltLength, KeySize, hashAlgorithm));
                 }
                 else
                 {
@@ -726,7 +726,7 @@ public override bool VerifyHash(ReadOnlySpan<byte> hash, ReadOnlySpan<byte> sign
             {
                 ArgumentException.ThrowIfNullOrEmpty(hashAlgorithm.Name, nameof(hashAlgorithm));
                 ArgumentNullException.ThrowIfNull(padding);
-                if (padding != RSASignaturePadding.Pkcs1 && padding != RSASignaturePadding.Pss)
+                if (padding != RSASignaturePadding.Pkcs1 && padding.Mode != RSASignaturePaddingMode.Pss)
                 {
                     throw PaddingModeNotSupported();
                 }
@@ -772,9 +772,10 @@ public override bool VerifyHash(ReadOnlySpan<byte> hash, ReadOnlySpan<byte> sign
                         CryptoPool.Return(repadRent, requiredBytes);
                         return valid;
                     }
-                    else if (padding == RSASignaturePadding.Pss)
+                    else if (padding.Mode == RSASignaturePaddingMode.Pss)
                     {
-                        return RsaPaddingProcessor.VerifyPss(hashAlgorithm, hash, unwrapped, KeySize);
+                        int saltLength = RsaPaddingProcessor.CalculatePssSaltLength(padding.PssSaltLength, KeySize, hashAlgorithm);
+                        return RsaPaddingProcessor.VerifyPss(hashAlgorithm, hash, unwrapped, KeySize, saltLength);
                     }
                     else
                     {

src/libraries/Common/src/System/Security/Cryptography/RSAAppleCrypto.cs

@@ -397,6 +397,14 @@ public override bool TrySignHash(ReadOnlySpan<byte> hash, Span<byte> destination
                 ArgumentException.ThrowIfNullOrEmpty(hashAlgorithm.Name, nameof(hashAlgorithm));
                 ArgumentNullException.ThrowIfNull(padding);
 
+                // Apple does not support custom salt length for the PSS padding
+                if (padding.Mode == RSASignaturePaddingMode.Pss &&
+                    (padding.PssSaltLength != RSASignaturePadding.PssSaltLengthIsHashLength &&
+                    padding.PssSaltLength != RsaPaddingProcessor.HashLength(hashAlgorithm)))
+                {
+                    throw new CryptographicException(SR.Cryptography_CustomPssSaltLengthNotSupported);
+                }
+
                 ThrowIfDisposed();
 
                 Interop.AppleCrypto.PAL_SignatureAlgorithm signatureAlgorithm = padding.Mode switch
@@ -454,7 +462,7 @@ public override bool TrySignHash(ReadOnlySpan<byte> hash, Span<byte> destination
 
                 byte[] rented = CryptoPool.Rent(rsaSize);
                 Span<byte> buf = new Span<byte>(rented, 0, rsaSize);
-                RsaPaddingProcessor.EncodePss(hashAlgorithm, hash, buf, keySize);
+                RsaPaddingProcessor.EncodePss(hashAlgorithm, hash, buf, keySize, hash.Length);
 
                 try
                 {
@@ -537,9 +545,9 @@ public override bool VerifyHash(ReadOnlySpan<byte> hash, ReadOnlySpan<byte> sign
                         Debug.Fail($"TryRsaVerificationPrimitive with a pre-allocated buffer");
                         throw new CryptographicException();
                     }
-
                     Debug.Assert(bytesWritten == rsaSize);
-                    return RsaPaddingProcessor.VerifyPss(hashAlgorithm, hash, unwrapped, keySize);
+                    int saltLength = RsaPaddingProcessor.CalculatePssSaltLength(padding.PssSaltLength, KeySize, hashAlgorithm);
+                    return RsaPaddingProcessor.VerifyPss(hashAlgorithm, hash, unwrapped, keySize, saltLength);
                 }
                 finally
                 {

src/libraries/Common/src/System/Security/Cryptography/RSACng.SignVerify.cs

@@ -66,7 +66,8 @@ public override byte[] SignHash(byte[] hash, HashAlgorithmName hashAlgorithm, RS
                                 return keyHandle.SignHash(hash, AsymmetricPaddingMode.NCRYPT_PAD_PKCS1_FLAG, &pkcsPaddingInfo, estimatedSize);
 
                             case RSASignaturePaddingMode.Pss:
-                                var pssPaddingInfo = new BCRYPT_PSS_PADDING_INFO() { pszAlgId = namePtr, cbSalt = hash.Length };
+                                int saltLength = RsaPaddingProcessor.CalculatePssSaltLength(padding.PssSaltLength, KeySize, hashAlgorithm);
+                                var pssPaddingInfo = new BCRYPT_PSS_PADDING_INFO() { pszAlgId = namePtr, cbSalt = saltLength };
                                 return keyHandle.SignHash(hash, AsymmetricPaddingMode.NCRYPT_PAD_PSS_FLAG, &pssPaddingInfo, estimatedSize);
 
                             default:
@@ -104,7 +105,11 @@ public override unsafe bool TrySignHash(ReadOnlySpan<byte> hash, Span<byte> dest
                             return keyHandle.TrySignHash(hash, destination, AsymmetricPaddingMode.NCRYPT_PAD_PKCS1_FLAG, &pkcs1PaddingInfo, out bytesWritten);
 
                         case RSASignaturePaddingMode.Pss:
-                            var pssPaddingInfo = new BCRYPT_PSS_PADDING_INFO() { pszAlgId = namePtr, cbSalt = hash.Length };
+                            var pssPaddingInfo = new BCRYPT_PSS_PADDING_INFO()
+                            {
+                                pszAlgId = namePtr,
+                                cbSalt = RsaPaddingProcessor.CalculatePssSaltLength(padding.PssSaltLength, KeySize, hashAlgorithm)
+                            };
                             return keyHandle.TrySignHash(hash, destination, AsymmetricPaddingMode.NCRYPT_PAD_PSS_FLAG, &pssPaddingInfo, out bytesWritten);
 
                         default:
@@ -152,7 +157,11 @@ public override unsafe bool VerifyHash(ReadOnlySpan<byte> hash, ReadOnlySpan<byt
                             return keyHandle.VerifyHash(hash, signature, AsymmetricPaddingMode.NCRYPT_PAD_PKCS1_FLAG, &pkcs1PaddingInfo);
 
                         case RSASignaturePaddingMode.Pss:
-                            var pssPaddingInfo = new BCRYPT_PSS_PADDING_INFO() { pszAlgId = namePtr, cbSalt = hash.Length };
+                            var pssPaddingInfo = new BCRYPT_PSS_PADDING_INFO()
+                            {
+                                pszAlgId = namePtr,
+                                cbSalt = RsaPaddingProcessor.CalculatePssSaltLength(padding.PssSaltLength, KeySize, hashAlgorithm)
+                            };
                             return keyHandle.VerifyHash(hash, signature, AsymmetricPaddingMode.NCRYPT_PAD_PSS_FLAG, &pssPaddingInfo);
 
                         default:

src/libraries/Common/src/System/Security/Cryptography/RSAOpenSsl.cs

@@ -735,7 +735,10 @@ public override byte[] SignHash(byte[] hash, HashAlgorithmName hashAlgorithm, RS
             int bytesRequired = Interop.Crypto.GetEvpPKeySizeBytes(key);
             byte[] signature = new byte[bytesRequired];
 
-            int written = Interop.Crypto.RsaSignHash(key, padding.Mode, hashAlgorithm, hash, signature);
+            int pssSaltLength = padding.Mode == RSASignaturePaddingMode.Pss
+                ? RsaPaddingProcessor.CalculatePssSaltLength(padding.PssSaltLength, KeySize, hashAlgorithm)
+                : 0;
+            int written = Interop.Crypto.RsaSignHash(key, padding.Mode, pssSaltLength, hashAlgorithm, hash, signature);
 
             if (written != signature.Length)
             {
@@ -766,7 +769,10 @@ public override bool TrySignHash(
                 return false;
             }
 
-            bytesWritten = Interop.Crypto.RsaSignHash(key, padding.Mode, hashAlgorithm, hash, destination);
+            int pssSaltLength = padding.Mode == RSASignaturePaddingMode.Pss
+                ? RsaPaddingProcessor.CalculatePssSaltLength(padding.PssSaltLength, KeySize, hashAlgorithm)
+                : 0;
+            bytesWritten = Interop.Crypto.RsaSignHash(key, padding.Mode, pssSaltLength, hashAlgorithm, hash, destination);
             Debug.Assert(bytesWritten == bytesRequired);
             return true;
         }
@@ -791,9 +797,13 @@ public override bool VerifyHash(ReadOnlySpan<byte> hash, ReadOnlySpan<byte> sign
 
             SafeEvpPKeyHandle key = GetKey();
 
+            int pssSaltLength = padding.Mode == RSASignaturePaddingMode.Pss
+                ? RsaPaddingProcessor.CalculatePssSaltLength(padding.PssSaltLength, KeySize, hashAlgorithm)
+                : 0;
             return Interop.Crypto.RsaVerifyHash(
                 key,
                 padding.Mode,
+                pssSaltLength,
                 hashAlgorithm,
                 hash,
                 signature);
@@ -853,20 +863,17 @@ private static void ValidatePadding(RSASignaturePadding padding)
         {
             ArgumentNullException.ThrowIfNull(padding);
 
-            // RSASignaturePadding currently only has the mode property, so
-            // there's no need for a runtime check that PKCS#1 doesn't use
-            // nonsensical options like with RSAEncryptionPadding.
-            //
-            // This would change if we supported PSS with an MGF other than MGF-1,
-            // or with a custom salt size, or with a different MGF digest algorithm
-            // than the data digest algorithm.
+            // PKCS#1 does not currently have anything to validate.
             if (padding.Mode == RSASignaturePaddingMode.Pkcs1)
             {
                 Debug.Assert(padding == RSASignaturePadding.Pkcs1);
             }
             else if (padding.Mode == RSASignaturePaddingMode.Pss)
             {
-                Debug.Assert(padding == RSASignaturePadding.Pss);
+                if (padding.PssSaltLength < RSASignaturePadding.PssSaltLengthMax)
+                {
+                    throw PaddingModeNotSupported();
+                }
             }
             else
             {