104080 custom rsapss salt length

src/libraries/Common/src/System/Security/Cryptography/RsaPaddingProcessor.cs

@@ -281,6 +281,7 @@ internal static void EncodePss(HashAlgorithmName hashAlgorithmName, ReadOnlySpan
         {
             const int MaxStackSaltLength = 128;
 
+            int sLen = saltLength;
             int hLen = HashLength(hashAlgorithmName);
 
             // https://tools.ietf.org/html/rfc3447#section-9.1.1
@@ -296,7 +297,7 @@ internal static void EncodePss(HashAlgorithmName hashAlgorithmName, ReadOnlySpan
             //
             // sLen = hLen in this implementation.
 
-            if (emLen < 2 + hLen + saltLength)
+            if (emLen < 2 + hLen + sLen)
             {
                 throw new CryptographicException(SR.Cryptography_KeyTooSmall);
             }
@@ -323,9 +324,9 @@ internal static void EncodePss(HashAlgorithmName hashAlgorithmName, ReadOnlySpan
                 // 4. Generate a random salt of length sLen
                 Debug.Assert(hLen is >= 0 and <= 64);
 
-                Span<byte> salt = saltLength > MaxStackSaltLength
-                    ? new byte[saltLength]
-                    : stackalloc byte[saltLength];
+                Span<byte> salt = sLen > MaxStackSaltLength
+                    ? new byte[sLen]
+                    : stackalloc byte[sLen];
                 RandomNumberGenerator.Fill(salt);
 
                 // 5. Let M' = an octet string of 8 zeros concat mHash concat salt
@@ -343,7 +344,7 @@ internal static void EncodePss(HashAlgorithmName hashAlgorithmName, ReadOnlySpan
 
                 // 7. Generate PS as zero-valued bytes of length emLen - sLen - hLen - 2.
                 // 8. Let DB = PS || 0x01 || salt
-                int psLen = emLen - saltLength - hLen - 2;
+                int psLen = emLen - sLen - hLen - 2;
                 db.Slice(0, psLen).Clear();
                 db[psLen] = 0x01;
                 salt.CopyTo(db.Slice(psLen + 1));

src/libraries/Common/tests/System/Security/Cryptography/AlgorithmImplementations/RSA/SignVerify.cs

@@ -1392,6 +1392,9 @@ private void VerifyExpectedSignature_Pss(
                 Modulus = keyParameters.Modulus,
                 Exponent = keyParameters.Exponent,
             };
+
+            RSASignaturePadding padding = RSASignaturePadding.Pss;
+
             using (RSA rsaPublic = RSAFactory.Create())
             using (RSA rsaPrivate = RSAFactory.Create())
             {
@@ -1410,27 +1413,27 @@ private void VerifyExpectedSignature_Pss(
                 // Generator for new tests.
                 if (signature == null)
                 {
-                    signature = SignData(rsaPrivate, data, hashAlgorithm, RSASignaturePadding.Pss);
+                    signature = SignData(rsaPrivate, data, hashAlgorithm, padding);
                     Console.WriteLine($"{callerName}: {signature.ByteArrayToHex()}");
                 }
 
                 if (RSAFactory.SupportsPss)
                 {
                     Assert.True(
-                        VerifyData(rsaPublic, data, signature, hashAlgorithm, RSASignaturePadding.Pss),
+                        VerifyData(rsaPublic, data, signature, hashAlgorithm, padding),
                         "Public key verified the signature");
 
                     Assert.True(
-                        VerifyData(rsaPrivate, data, signature, hashAlgorithm, RSASignaturePadding.Pss),
+                        VerifyData(rsaPrivate, data, signature, hashAlgorithm, padding),
                         "Private key verified the signature");
                 }
                 else
                 {
                     Assert.ThrowsAny<CryptographicException>(
-                        () => VerifyData(rsaPublic, data, signature, hashAlgorithm, RSASignaturePadding.Pss));
+                        () => VerifyData(rsaPublic, data, signature, hashAlgorithm, padding));
 
                     Assert.ThrowsAny<CryptographicException>(
-                        () => VerifyData(rsaPrivate, data, signature, hashAlgorithm, RSASignaturePadding.Pss));
+                        () => VerifyData(rsaPrivate, data, signature, hashAlgorithm, padding));
                 }
             }
         }

src/libraries/System.Security.Cryptography.Pkcs/tests/SignedCms/SignedCmsTests.netcoreapp.cs

@@ -732,7 +732,7 @@ public static IEnumerable<object[]> CreateSignature_RsaPssTests() =>
             from oid in (string[])([Oids.Sha256, Oids.Sha384, Oids.Sha512, Oids.Sha1])
             from byCtor in new[] { true, false }
             from saltLength in AreCustomPssSaltLengthsSupported
-                ? new[] { 0, RSASignaturePadding.PssSaltLengthMax, RSASignaturePadding.PssSaltLengthIsHashLength }
+                ? new[] { 0, 63, RSASignaturePadding.PssSaltLengthMax, RSASignaturePadding.PssSaltLengthIsHashLength }
                 : new[] { RSASignaturePadding.PssSaltLengthIsHashLength }
             select new object[] { oid, byCtor, saltLength };
 

src/libraries/System.Security.Cryptography.Pkcs/tests/SignedCms/SignedCmsWholeDocumentTests.cs

@@ -90,7 +90,7 @@ public static void ReadRsaPssDocument(bool fromSpan)
             Assert.Equal(
                 "07849DC26FCBB2F3BD5F57BDF214BAE374575F1BD4E6816482324799417CB379",
                 messageDigestAttr.MessageDigest.ByteArrayToHex());
-            
+
             Assert.IsType<Pkcs9AttributeObject>(signedAttrs[3].Values[0]);
 #if !NET
             Assert.NotSame(signedAttrs[3].Oid, signedAttrs[3].Values[0].Oid);

src/libraries/System.Security.Cryptography/src/System/Security/Cryptography/RSASignaturePadding.cs

@@ -12,16 +12,16 @@ namespace System.Security.Cryptography
     public sealed class RSASignaturePadding : IEquatable<RSASignaturePadding>
     {
         /// <summary>
-        /// Represents a constant value indicating that the salt length should match the hash length.
+        /// A constant value indicating that the PSS salt length should match the hash length.
         /// </summary>
         /// <remarks>This value is typically used in cryptographic operations where the salt length is required to
         /// be the same as the hash length.</remarks>
         public const int PssSaltLengthIsHashLength = RsaPaddingProcessor.PssSaltLengthIsHashLength;
 
         /// <summary>
-        /// Represents the maximum allowable length, in bytes, for a PSS (Probabilistic Signature Scheme) salt.
+        /// A constant value indicating that the PSS salt length should be the maximum allowable
         /// </summary>
-        /// <remarks>This constant is used to define the upper limit for the salt length in PSS-based
+        /// <remarks>A constant is used to define the upper limit for the salt length in PSS-based
         /// cryptographic operations. The maximum length is determined by the hash algorithm's output size.</remarks>
         public const int PssSaltLengthMax = RsaPaddingProcessor.PssSaltLengthMax;
 
@@ -38,7 +38,7 @@ public sealed class RSASignaturePadding : IEquatable<RSASignaturePadding>
         /// </summary>
         /// <param name="saltLength">The length of the salt in bytes, or one of the constants <see cref="PssSaltLengthIsHashLength"/> or <see cref="PssSaltLengthMax" />.</param>
         /// <returns>A new instance of <see cref="RSASignaturePadding"/> configured for PSS padding with the specified salt length.</returns>
-        /// <exception cref="ArgumentOutOfRangeException">The <paramref name="saltLength"/> is negative or not one of the special constants.</exception>
+        /// <exception cref="ArgumentOutOfRangeException">The <paramref name="saltLength"/> has a negative value other than one of the special constants.</exception>
         public static RSASignaturePadding CreatePss(int saltLength)
         {
             switch (saltLength)

src/libraries/System.Security.Cryptography/tests/X509Certificates/CertificateCreation/CertificateRequestChainTests.cs

@@ -490,7 +490,7 @@ private static void CreateAndTestChain(
             }
         }
 
-        [ConditionalTheory(nameof(PlatformSupport.AreCustomSaltLengthsSupportedWithPss))]
+        [ConditionalTheory(nameof(AreCustomSaltLengthsSupportedWithPss))]
         [InlineData(1)]
         [InlineData(RSASignaturePadding.PssSaltLengthMax)]
         [InlineData(RSASignaturePadding.PssSaltLengthIsHashLength)]

src/libraries/System.Security.Cryptography/tests/X509Certificates/CertificateCreation/CertificateRequestLoadTests.cs

@@ -1,6 +1,7 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Collections.Generic;
 using System.Formats.Asn1;
 using System.Net;
 using Test.Cryptography;
@@ -374,19 +375,19 @@ public static void VerifySignature_RSA_PKCS1(string hashAlgorithm)
             }
         }
 
+        public static IEnumerable<object[]> AllRsaPssCustomSaltLength()
+        {
+            foreach (string hashAlgorithm in new[] { "SHA256", "SHA384", "SHA512", "SHA1" })
+            {
+                foreach (int saltLength in new[] { 0, 1, RSASignaturePadding.PssSaltLengthMax, RSASignaturePadding.PssSaltLengthIsHashLength })
+                {
+                    yield return new object[] { hashAlgorithm, saltLength };
+                }
+            }
+        }
+
         [ConditionalTheory(typeof(PlatformSupport), nameof(PlatformSupport.AreCustomSaltLengthsSupportedWithPss))]
-        [InlineData("SHA256", 1)]
-        [InlineData("SHA384", 1)]
-        [InlineData("SHA512", 1)]
-        [InlineData("SHA1", 1)] 
-        [InlineData("SHA256", RSASignaturePadding.PssSaltLengthMax)]
-        [InlineData("SHA384", RSASignaturePadding.PssSaltLengthMax)]
-        [InlineData("SHA512", RSASignaturePadding.PssSaltLengthMax)]
-        [InlineData("SHA1", RSASignaturePadding.PssSaltLengthMax)]
-        [InlineData("SHA256", RSASignaturePadding.PssSaltLengthIsHashLength)]
-        [InlineData("SHA384", RSASignaturePadding.PssSaltLengthIsHashLength)]
-        [InlineData("SHA512", RSASignaturePadding.PssSaltLengthIsHashLength)]
-        [InlineData("SHA1", RSASignaturePadding.PssSaltLengthIsHashLength)]
+        [MemberData(nameof(AllRsaPssCustomSaltLength))]
         public static void VerifySignature_RSA_PSS_CustomSaltLength(string hashAlgorithm, int saltLength)
         {
             VerifySignature_RSA_PSSCore(hashAlgorithm, saltLength);

src/libraries/System.Security.Cryptography/tests/X509Certificates/CertificateCreation/CertificateRequestUsageTests.cs

@@ -323,7 +323,7 @@ public static void SelfSign_RSA_PssPadding_CustomSaltLength(int customSaltLength
                 AsnReader sequence = reader.ReadSequence();
                 ReadOnlyMemory<byte> tbsCertificate = sequence.ReadEncodedValue();
                 ReadOnlyMemory<byte> signatureAlgorithm = sequence.ReadEncodedValue();
-                byte[] signature = sequence.ReadBitString(out var _);
+                byte[] signature = sequence.ReadBitString(out _);
 
                 int testSaltLength = customSaltLength switch
                 {

src/libraries/System.Security.Cryptography/tests/X509Certificates/CertificateCreation/RSAPssX509SignatureGeneratorTests.cs

@@ -79,15 +79,15 @@ public static void PssPaddingSaltLengths(int saltLengthToTest)
                 AsnReader rootSequence = asnReader.ReadSequence();
                 Assert.Equal("1.2.840.113549.1.1.10", rootSequence.ReadObjectIdentifier()); // Make sure it's RSASSA-PSS
                 AsnReader pssStructure = rootSequence.ReadSequence();
-                pssStructure.ReadEncodedValue(); // Ignore the hash algorithm OID
-                pssStructure.ReadEncodedValue(); // Ignore the mask generation function OID
+                ReadOnlyMemory<byte> hashAlgorithm = pssStructure.ReadEncodedValue(); // Ignore the hash algorithm OID
+                ReadOnlyMemory<byte> mgf = pssStructure.ReadEncodedValue(); // Ignore the mask generation function OID
                 Asn1Tag saltTag = new Asn1Tag(TagClass.ContextSpecific, 2, true);
 
                 if (pssStructure.HasData && pssStructure.PeekTag().HasSameClassAndValue(saltTag))
                 {
-                    var saltEntry = pssStructure.ReadSequence(saltTag);
-                    var actualSaltLength = saltEntry.ReadInteger();
-                    var expectedSaltLength = saltLengthToTest switch
+                    AsnReader saltEntry = pssStructure.ReadSequence(saltTag);
+                    Numerics.BigInteger actualSaltLength = saltEntry.ReadInteger();
+                    int expectedSaltLength = saltLengthToTest switch
                     {
                         RSASignaturePadding.PssSaltLengthIsHashLength => 32,
                         RSASignaturePadding.PssSaltLengthMax => 222,

src/libraries/System.Security.Cryptography/tests/X509Certificates/CertificateCreation/X509Sha1SignatureGenerators.cs

@@ -1,11 +1,9 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Formats.Asn1;
 using System.Security.Cryptography.Asn1;
 using Test.Cryptography;
-#if NET11_0_OR_GREATER
-using System.Formats.Asn1;
-#endif
 
 namespace System.Security.Cryptography.X509Certificates.Tests.CertificateCreation
 {
@@ -77,6 +75,10 @@ public override byte[] GetSignatureAlgorithmIdentifier(HashAlgorithmName hashAlg
                     // sha1WithRSAEncryption with RSASSA-PSS parameters
                     return "300D06092A864886F70D01010A3000".HexToByteArray();
                 }
+                else if (_signaturePadding.PssSaltLength == 0)
+                {
+                    return "303506092a864886f70d01010a3028a009300706052b0e03021aa116301406092a864886f70d010108300706052b0e03021aa203020100".HexToByteArray();
+                }
                 else if (_signaturePadding.PssSaltLength == 1)
                 {
                     return "303506092a864886f70d01010a3028a009300706052b0e03021aa116301406092a864886f70d010108300706052b0e03021aa203020101".HexToByteArray();