Add testing support for Alpine 3.12

eng/pipelines/coreclr/templates/helix-queues-setup.yml

@@ -43,18 +43,19 @@ jobs:
     # Linux musl x64
     - ${{ if eq(parameters.platform, 'Linux_musl_x64') }}:
       - ${{ if eq(variables['System.TeamProject'], 'public') }}:
-        - (Alpine.311.Amd64.Open)Ubuntu.1604.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.11-helix-bfcd90a-20200123191053
+        - (Alpine.312.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.12-helix-20200602002622-e06dc59
       - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
-        - (Alpine.39.Amd64)[email protected]/dotnet-buildtools/prereqs:alpine-3.9-helix-bfcd90a-20200123191053
         - (Alpine.310.Amd64)[email protected]/dotnet-buildtools/prereqs:alpine-3.10-helix-bfcd90a-20200123191054
         - (Alpine.311.Amd64)[email protected]/dotnet-buildtools/prereqs:alpine-3.11-helix-bfcd90a-20200123191053
+        - (Alpine.312.Amd64)[email protected]/dotnet-buildtools/prereqs:alpine-3.12-helix-20200602002622-e06dc59
 
     # Linux musl arm64
     - ${{ if eq(parameters.platform, 'Linux_musl_arm64') }}:
       - ${{ if eq(variables['System.TeamProject'], 'public') }}:
-        - (Alpine.311.Arm64.Open)[email protected]/dotnet-buildtools/prereqs:alpine-3.11-helix-arm64v8-bfcd90a-20200123191027
+        - (Alpine.312.Arm64.Open)[email protected]/dotnet-buildtools/prereqs:alpine-3.12-helix-arm64v8-20200602002604-25f8a3e
       - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
         - (Alpine.311.Arm64)[email protected]/dotnet-buildtools/prereqs:alpine-3.11-helix-arm64v8-bfcd90a-20200123191027
+        - (Alpine.312.Arm64)[email protected]/dotnet-buildtools/prereqs:alpine-3.12-helix-arm64v8-20200602002604-25f8a3e
 
     # Linux x64
     - ${{ if eq(parameters.platform, 'Linux_x64') }}:

eng/pipelines/libraries/helix-queues-setup.yml

@@ -37,16 +37,16 @@ jobs:
     # Linux musl x64
     - ${{ if eq(parameters.platform, 'Linux_musl_x64') }}:
       - ${{ if eq(parameters.jobParameters.isFullMatrix, false) }}:
-        - (Alpine.311.Amd64.Open)ubuntu.1604.amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.11-helix-bfcd90a-20200123191053
+        - (Alpine.312.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.12-helix-20200602002622-e06dc59
       - ${{ if eq(parameters.jobParameters.isFullMatrix, true) }}:
-        - (Alpine.39.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:alpine-3.9-helix-bfcd90a-20200123191053
-        - (Alpine.310.Amd64.Open)ubuntu.1604.amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.10-helix-bfcd90a-20200123191054
-        - (Alpine.311.Amd64.Open)ubuntu.1604.amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.11-helix-bfcd90a-20200123191053
+        - (Alpine.310.Amd64.Open)[email protected]/dotnet-buildtools/prereqs:alpine-3.10-helix-bfcd90a-20200123191054
+        - (Alpine.311.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.11-helix-bfcd90a-20200123191053
+        - (Alpine.312.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.12-helix-20200601195603-e06dc59
 
     # Linux musl arm64
     - ${{ if eq(parameters.platform, 'Linux_musl_arm64') }}:
       - ${{ if eq(parameters.jobParameters.isFullMatrix, true) }}:
-        - (Alpine.38.Arm64.Open)Ubuntu.1804.ArmArch.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.8-helix-arm64v8-a45aeeb-20190620184035
+        - (Alpine.312.Arm64.Open)ubuntu.1804.armarch.open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.12-helix-arm64v8-20200602002604-25f8a3e
 
     # Linux x64
     - ${{ if eq(parameters.platform, 'Linux_x64') }}:

src/libraries/Native/Unix/System.Globalization.Native/pal_idna.c

@@ -18,10 +18,9 @@ enum {
 static const uint32_t AllowUnassigned = 0x1;
 static const uint32_t UseStd3AsciiRules = 0x2;
 
-static uint32_t GetOptions(uint32_t flags)
+static uint32_t GetOptions(uint32_t flags, uint32_t useToAsciiFlags)
 {
-    // Using Nontransitional to Unicode and Check ContextJ to match the current behavior of .NET on Windows
-    uint32_t options = UIDNA_NONTRANSITIONAL_TO_UNICODE | UIDNA_CHECK_CONTEXTJ;
+    uint32_t options = UIDNA_CHECK_CONTEXTJ;
 
     if ((flags & AllowUnassigned) == AllowUnassigned)
     {
@@ -33,6 +32,15 @@ static uint32_t GetOptions(uint32_t flags)
         options |= UIDNA_USE_STD3_RULES;
     }
 
+    if (useToAsciiFlags)
+    {
+        options |=  UIDNA_NONTRANSITIONAL_TO_ASCII;
+    }
+    else
+    {
+        options |=  UIDNA_NONTRANSITIONAL_TO_UNICODE;
+    }
+
     return options;
 }
 
@@ -53,7 +61,7 @@ int32_t GlobalizationNative_ToAscii(
     UErrorCode err = U_ZERO_ERROR;
     UIDNAInfo info = UIDNA_INFO_INITIALIZER;
 
-    UIDNA* pIdna = uidna_openUTS46(GetOptions(flags), &err);
+    UIDNA* pIdna = uidna_openUTS46(GetOptions(flags, /* useToAsciiFlags */ 1), &err);
 
     int32_t asciiStrLen = uidna_nameToASCII(pIdna, lpSrc, cwSrcLength, lpDst, cwDstLength, &info, &err);
 
@@ -82,7 +90,7 @@ int32_t GlobalizationNative_ToUnicode(
     UErrorCode err = U_ZERO_ERROR;
     UIDNAInfo info = UIDNA_INFO_INITIALIZER;
 
-    UIDNA* pIdna = uidna_openUTS46(GetOptions(flags), &err);
+    UIDNA* pIdna = uidna_openUTS46(GetOptions(flags, /* useToAsciiFlags */ 0), &err);
 
     int32_t unicodeStrLen = uidna_nameToUnicode(pIdna, lpSrc, cwSrcLength, lpDst, cwDstLength, &info, &err);
 

src/libraries/System.Globalization.Extensions/System.Globalization.Extensions.sln

@@ -1,4 +1,5 @@
-Microsoft Visual Studio Solution File, Format Version 12.00
+
+Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio Version 16
 VisualStudioVersion = 16.0.29923.206
 MinimumVisualStudioVersion = 10.0.40219.1
@@ -25,6 +26,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "System.Globalization.Extens
 		{2B96AA10-84C0-4927-8611-8D2474B990E8} = {2B96AA10-84C0-4927-8611-8D2474B990E8}
 	EndProjectSection
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "TestUtilities", "..\Common\tests\TestUtilities\TestUtilities.csproj", "{1659274B-9BFC-4709-B2EF-E9F363AC13B0}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -47,6 +50,10 @@ Global
 		{CB60359A-D0ED-474E-B395-9589F1A5656A}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{CB60359A-D0ED-474E-B395-9589F1A5656A}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{CB60359A-D0ED-474E-B395-9589F1A5656A}.Release|Any CPU.Build.0 = Release|Any CPU
+		{1659274B-9BFC-4709-B2EF-E9F363AC13B0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{1659274B-9BFC-4709-B2EF-E9F363AC13B0}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{1659274B-9BFC-4709-B2EF-E9F363AC13B0}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{1659274B-9BFC-4709-B2EF-E9F363AC13B0}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -56,6 +63,7 @@ Global
 		{2B96AA10-84C0-4927-8611-8D2474B990E8} = {E107E9C1-E893-4E87-987E-04EF0DCEAEFD}
 		{3634FAA3-A33E-406A-94EE-5611C6CC2810} = {2E666815-2EDB-464B-9DF6-380BF4789AD4}
 		{CB60359A-D0ED-474E-B395-9589F1A5656A} = {1A2F9F4A-A032-433E-B914-ADD5992BB178}
+		{1659274B-9BFC-4709-B2EF-E9F363AC13B0} = {1A2F9F4A-A032-433E-B914-ADD5992BB178}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {3E550ED4-6053-4B16-97D1-BAADB02924FE}

src/libraries/System.Globalization.Extensions/tests/IdnMapping/Data/ConformanceIdnaTestResult.cs

@@ -7,6 +7,12 @@
 
 namespace System.Globalization.Tests
 {
+    public enum IdnaTestResultType
+    {
+        ToUnicode,
+        ToAscii
+    }
+
     public class ConformanceIdnaTestResult
     {
         /// <summary>
@@ -22,18 +28,65 @@ public class ConformanceIdnaTestResult
         /// </summary>
         public string Value { get; private set; }
 
-        public ConformanceIdnaTestResult(string entry, string fallbackValue)
+        public IdnaTestResultType ResultType { get; private set; }
+
+        public string StatusValue { get; private set; }
+
+        public ConformanceIdnaTestResult(string entry, string fallbackValue, IdnaTestResultType resultType = IdnaTestResultType.ToAscii)
+            : this(entry, fallbackValue, null, null, useValueForStatus: true, resultType)
+        {
+        }
+
+        public ConformanceIdnaTestResult(string entry, string fallbackValue, string statusValue, string statusFallbackValue, IdnaTestResultType resultType = IdnaTestResultType.ToAscii)
+            : this(entry, fallbackValue, statusValue, statusFallbackValue, useValueForStatus: false, resultType)
+        {
+        }
+
+        private ConformanceIdnaTestResult(string entry, string fallbackValue, string statusValue, string statusFallbackValue, bool useValueForStatus, IdnaTestResultType resultType)
         {
-            if (string.IsNullOrWhiteSpace(entry))
-                SetValues(fallbackValue);
-            else
-                SetValues(entry);
+            ResultType = resultType;
+            SetValue(string.IsNullOrEmpty(entry.Trim()) ? fallbackValue : entry);
+            SetSuccess(useValueForStatus ?
+                            Value :
+                            string.IsNullOrEmpty(statusValue.Trim()) ? statusFallbackValue : statusValue);
         }
 
-        private void SetValues(string entry)
+        private void SetValue(string entry)
         {
             Value = entry.Trim();
-            Success = !Value.StartsWith("[", StringComparison.Ordinal);
+        }
+
+        private void SetSuccess(string statusValue)
+        {
+            StatusValue = statusValue.Trim();
+
+            Success = true;
+
+            if (StatusValue.StartsWith('[') && StatusValue != "[]")
+            {
+                if (StatusValue == Value)
+                {
+                    Success = false;
+                    return;
+                }
+
+                string[] statusCodes = StatusValue[1..^1].Split(',');
+                for (int i = 0; i < statusCodes.Length; i++)
+                {
+                    if (!IsIgnoredError(statusCodes[i].Trim()))
+                    {
+                        Success = false;
+                        break;
+                    }
+                }
+            }
+        }
+
+        private bool IsIgnoredError(string statusCode)
+        {
+            // We don't validate for BIDI rule so we can ignore BIDI codes
+            // If we're validating ToAscii we ignore rule V2 (UIDNA_ERROR_HYPHEN_3_4) for compatibility with windows.
+            return statusCode.StartsWith('B') || (ResultType == IdnaTestResultType.ToAscii && statusCode == "V2");
         }
     }
 }

src/libraries/System.Globalization.Extensions/tests/IdnMapping/Data/ConformanceIdnaUnicodeTestResult.cs

@@ -9,7 +9,13 @@ public sealed class ConformanceIdnaUnicodeTestResult : ConformanceIdnaTestResult
         public bool ValidDomainName { get; private set; }
 
         public ConformanceIdnaUnicodeTestResult(string entry, string fallbackValue, bool validDomainName = true)
-            : base(entry, fallbackValue)
+            : base(entry, fallbackValue, IdnaTestResultType.ToUnicode)
+        {
+            ValidDomainName = validDomainName;
+        }
+
+        public ConformanceIdnaUnicodeTestResult(string entry, string fallbackValue, string statusValue, string statusFallbackValue, bool validDomainName = true)
+            : base(entry, fallbackValue, statusValue, statusFallbackValue, IdnaTestResultType.ToUnicode)
         {
             ValidDomainName = validDomainName;
         }

src/libraries/System.Globalization.Extensions/tests/IdnMapping/Data/Factory.cs

@@ -27,7 +27,9 @@ private static string RemoveComment(string line)
         private static Stream GetIdnaTestTxt()
         {
             string fileName = null;
-            if (PlatformDetection.IsWindows7)
+            if (PlatformDetection.ICUVersion >= new Version(66, 0))
+                fileName = "IdnaTest_13.txt";
+            else if (PlatformDetection.IsWindows7)
                 fileName = "IdnaTest_Win7.txt";
             else if (PlatformDetection.IsWindows10Version1903OrGreater)
                 fileName = "IdnaTest_11.txt";
@@ -60,7 +62,9 @@ private static IEnumerable<IConformanceIdnaTest> ParseFile(Stream stream, Func<s
 
         private static IConformanceIdnaTest GetConformanceIdnaTest(string line, int lineCount)
         {
-            if (PlatformDetection.IsWindows7)
+            if (PlatformDetection.ICUVersion >= new Version(66, 0))
+                return new Unicode_13_0_IdnaTest(line, lineCount);
+            else if (PlatformDetection.IsWindows7)
                 return new Unicode_Win7_IdnaTest(line, lineCount);
             else if (PlatformDetection.IsWindows10Version1903OrGreater)
                 return new Unicode_11_0_IdnaTest(line, lineCount);
@@ -83,9 +87,12 @@ private static IConformanceIdnaTest GetConformanceIdnaTest(string line, int line
         /// <returns></returns>
         public static IEnumerable<IConformanceIdnaTest> GetDataset()
         {
+            // Nls is transitional so we filter out non transitional test cases.
+            // Icu is the opposite.
+            IdnType idnFilter = PlatformDetection.IsNlsGlobalization ? IdnType.Nontransitional : IdnType.Transitional;
             foreach (var entry in ParseFile(GetIdnaTestTxt(), GetConformanceIdnaTest))
             {
-                if (entry.Type != IdnType.Nontransitional && entry.Source != string.Empty)
+                if (entry.Type != idnFilter && entry.Source != string.Empty)
                     yield return entry;
             }
         }

src/libraries/System.Globalization.Extensions/tests/IdnMapping/Data/Unicode_11_0/IdnaTest_11.txt

@@ -2111,7 +2111,7 @@ xn--6-gsc039eqq6k.xn--0gb6bxkx18g; ڇ6Ⴔ辘.صيڇ‌; [B2, B3, C1, V6]; xn--6-
 ꡽≯.򻲀򒳄; ꡽≯.򻲀򒳄; [P1, V6]; xn--hdh8193c.xn--5z40cp629b; ; ;  # ≯.
 xn--hdh8193c.xn--5z40cp629b; ꡽≯.򻲀򒳄; [V6]; xn--hdh8193c.xn--5z40cp629b; ; ;  # ≯.
 ςოٻ.ςܔ; ; [B5, B6]; xn--3xa80l26n.xn--3xa41o; ; xn--4xa60l26n.xn--4xa21o;  # ςოٻ.ςܔ
-ΣᲝٻ.Σܔ; σᲝٻ.σܔ; [B5, B6, P1, V6]; xn--4xa60lw94a.xn--4xa21o; ; ;  # σᲝٻ.σܔ
+#ΣᲝٻ.Σܔ; σᲝٻ.σܔ; [B5, B6, P1, V6]; xn--4xa60lw94a.xn--4xa21o; ; ;  # WITH THE FLAGS WE USE IT DOESN'T RETURN ERRORS
 σოٻ.σܔ; ; [B5, B6]; xn--4xa60l26n.xn--4xa21o; ; ;  # σოٻ.σܔ
 Σოٻ.σܔ; σოٻ.σܔ; [B5, B6]; xn--4xa60l26n.xn--4xa21o; ; ;  # σოٻ.σܔ
 xn--4xa60l26n.xn--4xa21o; σოٻ.σܔ; [B5, B6]; xn--4xa60l26n.xn--4xa21o; ; ;  # σოٻ.σܔ

src/libraries/System.Globalization.Extensions/tests/IdnMapping/Data/Unicode_11_0/Unicode_11_0_IdnaTest.cs

@@ -9,7 +9,7 @@ namespace System.Globalization.Tests
 {
     /// <summary>
     /// Class to read data obtained from http://www.unicode.org/Public/idna.  For more information read the information
-    /// contained in Data\Unicode_11_0\IdnaTestV2.txt
+    /// contained in Data\Unicode_11_0\IdnaTest_11.txt
     ///
     /// The structure of the data set is a semicolon delimited list with the following columns:
     ///
@@ -34,52 +34,20 @@ namespace System.Globalization.Tests
     ///
     /// If the value of toUnicode or toAsciiN is the same as source, the column will be blank.
     /// </summary>
-    public class Unicode_11_0_IdnaTest : IConformanceIdnaTest
+    public class Unicode_11_0_IdnaTest : Unicode_IdnaTest
     {
-        public IdnType Type => IdnType.Nontransitional;
-        public string Source { get; set; }
-        public ConformanceIdnaUnicodeTestResult UnicodeResult { get; set; }
-        public ConformanceIdnaTestResult ASCIIResult { get; set; }
-        public int LineNumber { get; set; }
-
         public Unicode_11_0_IdnaTest(string line, int lineNumber)
         {
             var split = line.Split(';');
 
-            Source = EscapedToLiteralString(split[0], lineNumber);
-            UnicodeResult = new ConformanceIdnaUnicodeTestResult(EscapedToLiteralString(split[1], lineNumber), Source);
-            ASCIIResult = new ConformanceIdnaTestResult(EscapedToLiteralString(split[3], lineNumber), UnicodeResult.Value);
-            LineNumber = lineNumber;
-        }
-
-        /// <summary>
-        /// This will convert strings with escaped sequences to literal characters.  The input string is
-        /// expected to have escaped sequences in the form of '\uXXXX'.
-        ///
-        /// Example: "a\u0020b" will be converted to 'a b'.
-        /// </summary>
-        private static string EscapedToLiteralString(string escaped, int lineNumber)
-        {
-            var sb = new StringBuilder();
+            Type = IdnType.Nontransitional;
 
-            for (int i = 0; i < escaped.Length; i++)
-            {
-                if (i + 1 < escaped.Length && escaped[i] == '\\' && escaped[i + 1] == 'u')
-                {
-                    // Verify that the escaped sequence is not malformed
-                    Assert.True(i + 5 < escaped.Length, "There was a problem converting to literal string on Line " + lineNumber);
+            Source = EscapedToLiteralString(split[0], lineNumber);
 
-                    var codepoint = Convert.ToInt32(escaped.Substring(i + 2, 4), 16);
-                    sb.Append((char)codepoint);
-                    i += 5;
-                }
-                else
-                {
-                    sb.Append(escaped[i]);
-                }
-            }
+            UnicodeResult = new ConformanceIdnaUnicodeTestResult(EscapedToLiteralString(split[1], lineNumber), Source, EscapedToLiteralString(split[2], lineNumber), string.Empty);
+            ASCIIResult = new ConformanceIdnaTestResult(EscapedToLiteralString(split[3], lineNumber), UnicodeResult.Value, EscapedToLiteralString(split[4], lineNumber), UnicodeResult.StatusValue);
 
-            return sb.ToString().Trim();
+            LineNumber = lineNumber;
         }
     }
 }