From 0418e7b60814ba4557af6d4314630aac316456f4 Mon Sep 17 00:00:00 2001 From: Dean Ellis Date: Mon, 19 Feb 2024 13:08:56 +0000 Subject: [PATCH 01/18] Debugging MSBuild Tasks (#8730) * Debugging MSBuild Tasks One thing that is very useful is the ability to debug your Tasks while they are being run on a build process. This is possible thanks to the `MSBUILDDEBUGONSTART` environment variable. When set to `2` this will force MSBuild to wait for a debugger connection before continuing. You will see the following prompt. ```dotnetcli Waiting for debugger to attach (dotnet PID 13001). Press enter to continue... ``` You can then use VS or VSCode to attach to this process and debug you tasks. In the case of .NET Android we need to do a couple of thing first though. Firstly we need to disable the use of `ILRepacker` on the `Xamarin.Android.Build.Tasks` assembly. This is because `ILRepacker` does NOT handle debug symbols very well. Assemblies it generates seem to be JIT optimized so the debugger will not load the symbols. A new MSBuild property has been introduced to disable this feature while debugging. `_ILRepackEnabled` can be set as an environment variable which MSBuild will pickup. ```dotnetcli make prepare && _ILRepackEnabled=false make jenkins ``` This will disable the `ILRepacker` for the build. You can then start your test app with the `dotnet-local` script (so it uses your build) ```dotnetcli MSBUILDDEBUGONSTART=2 ~//dotnet-local.sh build -m:1 ``` Once MSBuild starts it will print the following ```dotnetcli Waiting for debugger to attach (dotnet PID xxxx). Press enter to continue... ``` You need to copy the PID value so we can use this in the IDE. For Visual Studio you can use the `Attach to Process` menu option, while you have the Xamarin.Android.sln solution open. For VSCode open the workspace then use the `Debug MSBuild Task` Run and Debug option. You will be prompted for the PID and it will then connect. Once connection go back to your command prompt and press ENTER so that the MSBuild process can continue. You will be able to set breakpoints in Tasks (but not Targets) and step through code from this point on. --- .vscode/launch.json | 14 +++- .vscode/tasks.json | 18 +++-- Documentation/guides/MSBuildBestPractices.md | 68 +++++++++++++++++++ .../Xamarin.Android.Build.Tasks.targets | 2 + 4 files changed, 97 insertions(+), 5 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 777abde3cf6..8917bb93a8a 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -44,6 +44,12 @@ "port": 10000, "preLaunchTask": "run-sample-under-dotnet", }, + { + "name": "Attach to Process", + "type": "coreclr", + "request": "attach", + "processId": "${input:processid}" + } ], "inputs": [ { @@ -52,6 +58,12 @@ "default": "Debug", "description": "The Build Configuration", "options": [ "Debug", "Release"] - } + }, + { + "id": "processid", + "type": "promptString", + "default": "0", + "description": "Enter dotnet build process id reported when setting the env var MSBUILDDEBUGONSTART=2", + }, ] } diff --git a/.vscode/tasks.json b/.vscode/tasks.json index e79dfcacc8d..4b652be3076 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -127,8 +127,8 @@ "label": "build-sample-under-dotnet", "type": "shell", "windows": { "command": "dotnet-local.cmd build ${input:project} -p:Configuration=${input:configuration} -t:${input:target} -bl:${input:target}.binlog", }, - "linux": { "command": "./dotnet-local.sh build ${input:project} -p:Configuration=${input:configuration} -t:${input:target} -bl:${input:target}.binlog",}, - "osx": { "command": "./dotnet-local.sh build ${input:project} -p:Configuration=${input:configuration} -t:${input:target} -bl:${input:target}.binlog",}, + "linux": { "command": "${input:debugbuildtasks} ./dotnet-local.sh build ${input:project} -p:Configuration=${input:configuration} -t:${input:target} -bl:${input:target}.binlog",}, + "osx": { "command": "${input:debugbuildtasks} ./dotnet-local.sh build ${input:project} -p:Configuration=${input:configuration} -t:${input:target} -bl:${input:target}.binlog",}, "group": { "kind": "build", "isDefault": true @@ -141,8 +141,8 @@ "label": "run-sample-under-dotnet", "type": "shell", "windows": { "command": "dotnet-local.cmd build ${input:project} \"-t:Run\" --no-restore -p:TargetFramework=${input:targetframework} -p:Configuration=${input:configuration} -p:AndroidAttachDebugger=${input:attach} -bl:run.binlog", }, - "linux": { "command": "./dotnet-local.sh build ${input:project} \"-t:Run\" --no-restore -p:TargetFramework=${input:targetframework} -p:Configuration=${input:configuration} -p:AndroidAttachDebugger=${input:attach} -bl:run.binlog",}, - "osx": { "command": "./dotnet-local.sh build ${input:project} \"-t:Run\" --no-restore -p:TargetFramework=${input:targetframework} -p:Configuration=${input:configuration} -p:AndroidAttachDebugger=${input:attach} -bl:run.binlog",}, + "linux": { "command": "${input:debugbuildtasks} ./dotnet-local.sh build ${input:project} \"-t:Run\" --no-restore -p:TargetFramework=${input:targetframework} -p:Configuration=${input:configuration} -p:AndroidAttachDebugger=${input:attach} -bl:run.binlog",}, + "osx": { "command": "${input:debugbuildtasks} ./dotnet-local.sh build ${input:project} \"-t:Run\" --no-restore -p:TargetFramework=${input:targetframework} -p:Configuration=${input:configuration} -p:AndroidAttachDebugger=${input:attach} -bl:run.binlog",}, "group": { "kind": "build", "isDefault": true @@ -220,5 +220,15 @@ "Everything", ] }, + { + "id": "debugbuildtasks", + "type": "pickString", + "default": "", + "description": "Debug Build Tasks?", + "options": [ + "", + "MSBUILDDEBUGONSTART=2" + ] + }, ] } \ No newline at end of file diff --git a/Documentation/guides/MSBuildBestPractices.md b/Documentation/guides/MSBuildBestPractices.md index bca45d30b5e..69c1b49d057 100644 --- a/Documentation/guides/MSBuildBestPractices.md +++ b/Documentation/guides/MSBuildBestPractices.md @@ -5,6 +5,74 @@ This guide is a work-in-progress, but really has two main goals: - What are good MSBuild practice in relation to what we already have going on in Xamarin.Android MSBuild targets? +## Debugging MSBuild Tasks + +One thing that is very useful is the ability to debug your Tasks while +they are being run on a build process. This is possible thanks to the +`MSBUILDDEBUGONSTART` environment variable. When set to `2` this will +force MSBuild to wait for a debugger connection before continuing. +You will see the following prompt. + +```dotnetcli +Waiting for debugger to attach (dotnet PID 13001). Press enter to continue... +``` + +You can then use VS or VSCode to attach to this process and debug you tasks. + +In the case of .NET Android we need to do a couple of thing first though. Firstly +we need to disable the use of `ILRepacker` on the `Xamarin.Android.Build.Tasks` +assembly. This is because `ILRepacker` does NOT handle debug symbols very well. +Assemblies it generates seem to be JIT optimized so the debugger will not load +the symbols. A new MSBuild property has been introduced to disable this feature +while debugging. `_ILRepackEnabled` can be set as an environment variable which +MSBuild will pickup. You will also need to build the `Debug` Configuration. + +```dotnetcli +export CONFIGURATION=Debug +make prepare && _ILRepackEnabled=false make jenkins +``` + +This will disable the `ILRepacker` for the build. + +You can then start your test app with the `dotnet-local` script (so it uses your build) + +### [MacOS](#tab/macos) + +```dotnetcli +MSBUILDDEBUGONSTART=2 ~//dotnet-local.sh build -m:1 +``` + +### [Linux](#tab/linux) + +```dotnetcli +MSBUILDDEBUGONSTART=2 ~//dotnet-local.sh build -m:1 +``` + +### [Windows](#tab/windows) + +```dotnetcli +set MSBUILDDEBUGONSTART=2 +~//dotnet-local.cmd build -m:1 +``` + +--- + +Note: the `-m:1` is important as it restricts MSBuild to 1 node. + +Once MSBuild starts it will print the following + +```dotnetcli +Waiting for debugger to attach (dotnet PID xxxx). Press enter to continue... +``` + +You need to copy the PID value so we can use this in the IDE. For Visual Studio you can use the `Attach to Process` menu option, while you have the Xamarin.Android.sln solution open. For VSCode open the workspace then use the `Attach to Process` Run and Debug option. You will be prompted for the PID and it will then connect. + +Once connected go back to your command prompt and press ENTER so that the MSBuild process can continue. + +You will be able to set breakpoints in Tasks (but not Targets) and step through code from this point on. + +If you want to test in-tree using the same the `build-sample-under-dotnet` command will ask you if you want to debug MSBuild tasks and fill in the `MSBUILDDEBUGONSTART` for you. The PID text will appear in the `Terminal` window in VSCode. In addition the `run-sample-under-dotnet` command will ask the same. + ## Naming MSBuild targets, properties, and item groups are prefixed with an diff --git a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets index 081fbb15896..91a88bedf80 100644 --- a/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets +++ b/src/Xamarin.Android.Build.Tasks/Xamarin.Android.Build.Tasks.targets @@ -28,6 +28,7 @@ <_MultiDexAarInAndroidSdk>extras\android\m2repository\com\android\support\multidex\1.0.1\multidex-1.0.1.aar <_SupportLicense Condition="Exists('$(_AndroidSdkLocation)\extras\android\m2repository\NOTICE.txt')">$(_AndroidSdkLocation)\extras\android\m2repository\NOTICE.txt <_SupportLicense Condition="Exists('$(_AndroidSdkLocation)\extras\android\m2repository\m2repository\NOTICE.txt')">$(_AndroidSdkLocation)\extras\android\m2repository\m2repository\NOTICE.txt + <_ILRepackEnabled Condition=" '$(_ILRepackEnabled)' == '' ">true From 73684872b3891dbaf7cc27d7675ac5f1b813160c Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Mon, 19 Feb 2024 15:52:38 +0000 Subject: [PATCH 02/18] Bump NDK to r26c (#8732) Changes: https://github.com/android/ndk/wiki/Changelog-r26#r26c * Updated LLVM to clang-r487747e. See AndroidVersion.txt and clang_source_info.md in the toolchain directory for version information. * [Issue 1928](https://github.com/android/ndk/issues/1928): Fixed Clang crash in instruction selection for 32-bit armv8 floating point. * [Issue 1953](https://github.com/android/ndk/issues/1953): armeabi-v7a libc++ libraries are once again built as thumb. `AndroidVersion.txt` contains: ``` 17.0.2 based on r487747e ``` --- .../xaprepare/ConfigAndData/BuildAndroidPlatforms.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build-tools/xaprepare/xaprepare/ConfigAndData/BuildAndroidPlatforms.cs b/build-tools/xaprepare/xaprepare/ConfigAndData/BuildAndroidPlatforms.cs index 506a56b25e4..b1ef7a0e2d9 100644 --- a/build-tools/xaprepare/xaprepare/ConfigAndData/BuildAndroidPlatforms.cs +++ b/build-tools/xaprepare/xaprepare/ConfigAndData/BuildAndroidPlatforms.cs @@ -5,8 +5,8 @@ namespace Xamarin.Android.Prepare { class BuildAndroidPlatforms { - public const string AndroidNdkVersion = "26b"; - public const string AndroidNdkPkgRevision = "26.1.10909125"; + public const string AndroidNdkVersion = "26c"; + public const string AndroidNdkPkgRevision = "26.2.11394342"; public const int NdkMinimumAPI = 21; public const int NdkMinimumAPILegacy32 = 21; From 82133485ccf418c104b3975c4b257a9245eb74ef Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 21 Feb 2024 08:38:30 +0000 Subject: [PATCH 03/18] [xaprepare] Make 7zip work with "dangerous" symlinks in ZIPs (#8737) From time to time, `7zip` invoked by `xaprepare` with an error similar to: ERROR: Dangerous symbolic link path was ignored : android-ndk-r26c/toolchains/llvm/prebuilt/linux-x86_64/lib/python3.10/site-packages/lldb/lldb-argdumper : ../../../../bin/lldb-argdumper ERROR: Dangerous symbolic link path was ignored : android-ndk-r26c/toolchains/llvm/prebuilt/linux-x86_64/lib/python3.10/site-packages/lldb/_lldb.cpython-310-x86_64-linux-gnu.so : ../../../liblldb.so The problem is that this symlink is **not** a dangerous one, as it doesn't point outside the archived directory tree. This happened on mac, Windows and Linux alike. Try to work around the issue by using an undocumented `-snld` 7zip option when extracting ZIP archives. Since the option is understood only from `7-Zip` 20.x, check the version before using the argument. --- .../Application/RegexProgramVersionParser.cs | 120 +++++++++--------- .../Application/SevenZipVersionParser.cs | 70 ++++++++++ .../xaprepare/Application/VersionFetchers.cs | 9 +- .../xaprepare/ToolRunners/SevenZipRunner.cs | 17 +++ .../xaprepare/xaprepare/xaprepare.csproj | 5 +- 5 files changed, 155 insertions(+), 66 deletions(-) create mode 100644 build-tools/xaprepare/xaprepare/Application/SevenZipVersionParser.cs diff --git a/build-tools/xaprepare/xaprepare/Application/RegexProgramVersionParser.cs b/build-tools/xaprepare/xaprepare/Application/RegexProgramVersionParser.cs index 90295cc8504..67dc9cdc971 100644 --- a/build-tools/xaprepare/xaprepare/Application/RegexProgramVersionParser.cs +++ b/build-tools/xaprepare/xaprepare/Application/RegexProgramVersionParser.cs @@ -12,78 +12,78 @@ namespace Xamarin.Android.Prepare /// is set to any other value only that line is taken into consideration. This is /// done to make processing less ambiguous and faster. /// - class RegexProgramVersionParser : ProgramVersionParser - { - const string VersionGroupName = "Version"; - static readonly char[] LineSeparator = new [] { '\n' }; + class RegexProgramVersionParser : ProgramVersionParser + { + public const string VersionGroupName = "Version"; + public static readonly char[] LineSeparator = new [] { '\n' }; - Regex rx; + Regex rx; - public RegexProgramVersionParser (string programName, string versionArguments, Regex regex, uint versionOutputLine = 0, Log? log = null) - : base (programName, versionArguments, versionOutputLine, log) - { - if (regex == null) - throw new ArgumentNullException (nameof (regex)); - rx = regex; - } + public RegexProgramVersionParser (string programName, string versionArguments, Regex regex, uint versionOutputLine = 0, Log? log = null) + : base (programName, versionArguments, versionOutputLine, log) + { + if (regex == null) + throw new ArgumentNullException (nameof (regex)); + rx = regex; + } - public RegexProgramVersionParser (string programName, string versionArguments, string regex, uint versionOutputLine = 0, Log? log = null) - : base (programName, versionArguments, versionOutputLine, log) - { - if (String.IsNullOrEmpty (regex)) - throw new ArgumentException ("must not be null or empty", nameof (regex)); + public RegexProgramVersionParser (string programName, string versionArguments, string regex, uint versionOutputLine = 0, Log? log = null) + : base (programName, versionArguments, versionOutputLine, log) + { + if (String.IsNullOrEmpty (regex)) + throw new ArgumentException ("must not be null or empty", nameof (regex)); - rx = new Regex (regex, RegexOptions.Compiled); - } + rx = new Regex (regex, RegexOptions.Compiled); + } - protected override string ParseVersion (string programOutput) - { - string output = programOutput.Trim (); - if (String.IsNullOrEmpty (output)) { - Log.WarningLine ($"Unable to parse version of {ProgramName} because version output was empty"); - return DefaultVersionString; - } + protected override string ParseVersion (string programOutput) + { + string output = programOutput.Trim (); + if (String.IsNullOrEmpty (output)) { + Log.WarningLine ($"Unable to parse version of {ProgramName} because version output was empty"); + return DefaultVersionString; + } - string ret = String.Empty; - string[] lines = programOutput.Split (LineSeparator); - if (VersionOutputLine > 0) { - if (lines.Length < VersionOutputLine) { - Log.WarningLine ($"Not enough lines in version output of {ProgramName}: version number was supposed to be found on line {VersionOutputLine} but there are only {lines.Length} lines"); - return DefaultVersionString; - } + string ret = String.Empty; + string[] lines = programOutput.Split (LineSeparator); + if (VersionOutputLine > 0) { + if (lines.Length < VersionOutputLine) { + Log.WarningLine ($"Not enough lines in version output of {ProgramName}: version number was supposed to be found on line {VersionOutputLine} but there are only {lines.Length} lines"); + return DefaultVersionString; + } - if (TryMatch (lines [VersionOutputLine - 1], out ret) && !String.IsNullOrEmpty (ret)) { - return ret; - } + if (TryMatch (rx, lines [VersionOutputLine - 1], out ret) && !String.IsNullOrEmpty (ret)) { + return ret; + } - return DefaultVersionString; - } + return DefaultVersionString; + } - foreach (string line in lines) { - if (TryMatch (line, out ret)) - break; - } + foreach (string line in lines) { + if (TryMatch (rx, line, out ret)) + break; + } - return ret ?? DefaultVersionString; - } + return ret ?? DefaultVersionString; + } - bool TryMatch (string line, out string version) - { - version = String.Empty; + public static bool TryMatch (Regex regex, string line, out string version) + { + version = String.Empty; - Match match = rx.Match (line); - if (!match.Success || match.Groups.Count <= 0) { - return false; - } + Match match = regex.Match (line); + if (!match.Success || match.Groups.Count <= 0) { + return false; + } - foreach (Group group in match.Groups) { - if (String.Compare (group.Name, VersionGroupName, StringComparison.OrdinalIgnoreCase) == 0) { - version = group.Value; - return true; - } - } + foreach (Group group in match.Groups) { + if (String.Compare (group.Name, VersionGroupName, StringComparison.OrdinalIgnoreCase) == 0) { + version = group.Value; + return true; + } + } - return false; - } - } + return false; + } + } } diff --git a/build-tools/xaprepare/xaprepare/Application/SevenZipVersionParser.cs b/build-tools/xaprepare/xaprepare/Application/SevenZipVersionParser.cs new file mode 100644 index 00000000000..86bc518071a --- /dev/null +++ b/build-tools/xaprepare/xaprepare/Application/SevenZipVersionParser.cs @@ -0,0 +1,70 @@ +using System; +using System.Text.RegularExpressions; + +namespace Xamarin.Android.Prepare; + +class SevenZipVersionParser : ProgramVersionParser +{ + const string VersionArgument = "--help"; + readonly Regex fallbackRegex; + readonly Regex modernRegex; + + public SevenZipVersionParser (string programName, Regex fallbackRegex, Log? log = null) + : base (programName, VersionArgument, 0, log) + { + this.fallbackRegex = fallbackRegex; + modernRegex = VersionFetchers.MakeRegex (@"^7-Zip (\(a\) ){0,1}(?[\d]+\.[\d]+)"); + } + + protected override string ParseVersion (string programOutput) + { + string output = programOutput.Trim (); + if (String.IsNullOrEmpty (output)) { + Log.WarningLine ($"Unable to parse version of {ProgramName} because version output was empty"); + return DefaultVersionString; + } + + string ret = String.Empty; + string[] lines = programOutput.Split (RegexProgramVersionParser.LineSeparator); + + // First try to find the official 7zip release version + foreach (string l in lines) { + string line = l.Trim (); + + if (line.Length == 0) { + continue; + } + + if (line.StartsWith ("7-Zip", StringComparison.OrdinalIgnoreCase)) { + // Strings of the form: + // 7-Zip 23.01 (x64) : Copyright (c) 1999-2023 Igor Pavlov : 2023-06-20 + // 7-Zip (a) 23.01 (x64) : Copyright (c) 1999-2023 Igor Pavlov : 2023-06-20 + // 7-Zip (a) 18.01 (x64) : Copyright (c) 1999-2018 Igor Pavlov : 2018-01-28 + // 7-Zip (a) 18.01 (x86) : Copyright (c) 1999-2018 Igor Pavlov : 2018-01-28 + if (RegexProgramVersionParser.TryMatch (modernRegex, line, out ret)) { + return ret; + } + } + + // Since we know we're dealing with `--help` option output, we can short-circuit things + if (line.StartsWith ("Usage:", StringComparison.OrdinalIgnoreCase)) { + break; + } + } + + // Modern version wasn't found, try again with the fallback one + foreach (string l in lines) { + string line = l.Trim (); + + if (line.Length == 0) { + continue; + } + + if (RegexProgramVersionParser.TryMatch (fallbackRegex, line, out ret)) { + return ret; + } + } + + return DefaultVersionString; + } +} diff --git a/build-tools/xaprepare/xaprepare/Application/VersionFetchers.cs b/build-tools/xaprepare/xaprepare/Application/VersionFetchers.cs index 0125ff33b06..7b0a95b639e 100644 --- a/build-tools/xaprepare/xaprepare/Application/VersionFetchers.cs +++ b/build-tools/xaprepare/xaprepare/Application/VersionFetchers.cs @@ -18,7 +18,7 @@ class VersionFetchers public Dictionary Fetchers => GetFetchers (); - static Regex MakeRegex (string regex) + internal static Regex MakeRegex (string regex) { return new Regex (regex, RegexOptions.Compiled | RegexOptions.Singleline); } @@ -30,8 +30,11 @@ static Regex MakeRegex (string regex) return fetchers; fetchers = new Dictionary (Context.Instance.OS.DefaultStringComparer) { - {"7z", "--help", MakeRegex ($"Version {StandardVersionRegex}"), 3}, - {"7za", "--help", MakeRegex ($"Version {StandardVersionRegex}"), 3}, + // Program-specific parsers + {"7z", new SevenZipVersionParser ("7z", MakeRegex ($"Version {StandardVersionRegex}"))}, + {"7za", new SevenZipVersionParser ("7za", MakeRegex ($"Version {StandardVersionRegex}"))}, + + // Regex parsers {"autoconf", "--version", StandardVersionAtEOL, 1}, {"automake", "--version", StandardVersionAtEOL, 1}, {"brew", "--version", MakeRegex ($"^Homebrew {StandardVersionRegex}"), 1}, diff --git a/build-tools/xaprepare/xaprepare/ToolRunners/SevenZipRunner.cs b/build-tools/xaprepare/xaprepare/ToolRunners/SevenZipRunner.cs index 4e025f17239..e27b85d5b26 100644 --- a/build-tools/xaprepare/xaprepare/ToolRunners/SevenZipRunner.cs +++ b/build-tools/xaprepare/xaprepare/ToolRunners/SevenZipRunner.cs @@ -10,6 +10,10 @@ partial class SevenZipRunner : ToolRunner { const double DefaultTimeout = 30; // minutes static readonly Version bsoepMinVersion = new Version (15, 5); + + // Just an educated guess. The official download page had versions 19 and then 23+ available + // and the 19 one didn't support the `-snld` switch + static readonly Version snldMinVersion = new Version (20, 0); Version version; protected override string DefaultToolExecutableName => "7za"; @@ -36,6 +40,7 @@ public async Task Extract (string archivePath, string outputDirectory, Lis throw new ArgumentException ("must not be null or empty", nameof (outputDirectory)); ProcessRunner runner = CreateProcessRunner ("x"); + AddStandardArguments (runner); AddArguments (runner, extraArguments); runner.AddQuotedArgument ($"-o{outputDirectory}"); @@ -114,6 +119,18 @@ protected override TextWriter CreateLogSink (string? logFilePath) void AddStandardArguments (ProcessRunner runner) { + Log.DebugLine ($"7-zip standard arguments, for 7z version {version}"); + + // Ignore some "dangerous" symbolic symlinks in the ZIP archives. This allows 7zip to unpack Android NDK archives + // without error. The option appears to be undocumented, but was mentioned by the 7zip author here: + // + // https://sourceforge.net/p/sevenzip/discussion/45798/thread/187ce54fb0/ + // + if (version >= snldMinVersion) { + Log.DebugLine ("Adding option to ignore dangerous symlinks"); + runner.AddArgument ("-snld"); + } + // Disable progress indicator (doesn't appear to have any effect with some versions of 7z) runner.AddArgument ("-bd"); diff --git a/build-tools/xaprepare/xaprepare/xaprepare.csproj b/build-tools/xaprepare/xaprepare/xaprepare.csproj index 41766677fdd..ffa96b5c34c 100644 --- a/build-tools/xaprepare/xaprepare/xaprepare.csproj +++ b/build-tools/xaprepare/xaprepare/xaprepare.csproj @@ -1,17 +1,16 @@ + $(DotNetStableTargetFramework) Exe - 8.0 + $(LangVersion) Xamarin.Android.Prepare xaprepare true enable - - From 39ce2f94ae7c9c4846d27deaba4fb2173c5097e1 Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Wed, 21 Feb 2024 09:14:27 +0000 Subject: [PATCH 04/18] Bring in changes from PR #8478 (#8727) Context: https://github.com/xamarin/xamarin-android/pull/8478 Changes in this commit were originally made in #8478 but aren't strictly tied to that PR. To make reviews of #8478 easier, a portion of changes were extracted and placed here. Modify the LLVM IR generator to output more readable and compact code: * Use hexadecimal integer notation wherever it makes sense (e.g. hashes, managed token IDs) * Use boolean literals `true` and `false` instead of previously used `1` and `0`, respectively * Improve formatting of arrays * Introduce arrays which can composed from various "sections", each with its own comment and with data provided during the generation process, instead of during the preparation phase. Additional changes: Extract portions of `MonoAndroidHelper.cs` and place them in a separate file, `MonoAndroidHelper.Basic.cs` . Code placed in there provides conversion between RID, and various forms of Android ABI names as well as `xxHash` functions used throughout the repository, as well as methods to handle paths of ZIP archive entries in a uniform way. #8478 uses these extracted methods in tests, in standalone utilities in addition to build tasks. Use maximum LZ4 compression level when compressing assemblies to be placed in the application APK/AAB archives. --- .../Utilities/ApplicationConfig.cs | 8 + ...pplicationConfigNativeAssemblyGenerator.cs | 10 +- .../Utilities/AssemblyCompression.cs | 2 +- .../LlvmIrGenerator/LlvmIrComposer.cs | 15 +- .../LlvmIrGenerator/LlvmIrGenerator.cs | 403 ++++++++++++++---- .../Utilities/LlvmIrGenerator/LlvmIrModule.cs | 8 +- .../LlvmIrGenerator/LlvmIrVariable.cs | 94 +++- .../LlvmIrGenerator/MemberInfoUtilities.cs | 10 + .../NativeAssemblerAttribute.cs | 2 + .../MarshalMethodsNativeAssemblyGenerator.cs | 17 +- .../Utilities/MonoAndroidHelper.Basic.cs | 200 +++++++++ .../Utilities/MonoAndroidHelper.cs | 92 +--- ...peMappingReleaseNativeAssemblyGenerator.cs | 7 +- 13 files changed, 678 insertions(+), 190 deletions(-) create mode 100644 src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Basic.cs diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs index 25550eb3473..02b6c6ba776 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfig.cs @@ -42,11 +42,19 @@ sealed class ApplicationConfig public uint bundled_assembly_name_width; public uint number_of_assembly_store_files; public uint number_of_dso_cache_entries; + + [NativeAssembler (NumberFormat = LLVMIR.LlvmIrVariableNumberFormat.Hexadecimal)] public uint android_runtime_jnienv_class_token; + + [NativeAssembler (NumberFormat = LLVMIR.LlvmIrVariableNumberFormat.Hexadecimal)] public uint jnienv_initialize_method_token; + + [NativeAssembler (NumberFormat = LLVMIR.LlvmIrVariableNumberFormat.Hexadecimal)] public uint jnienv_registerjninatives_method_token; public uint jni_remapping_replacement_type_count; public uint jni_remapping_replacement_method_index_entry_count; + + [NativeAssembler (NumberFormat = LLVMIR.LlvmIrVariableNumberFormat.Hexadecimal)] public uint mono_components_mask; public string android_package_name = String.Empty; } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs index a442da7e221..cee11386b4a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/ApplicationConfigNativeAssemblyGenerator.cs @@ -29,7 +29,7 @@ public override string GetComment (object data, string fieldName) { var dso_entry = EnsureType (data); if (String.Compare ("hash", fieldName, StringComparison.Ordinal) == 0) { - return $" hash 0x{dso_entry.hash:x}, from name: {dso_entry.HashedName}"; + return $" from name: {dso_entry.HashedName}"; } if (String.Compare ("name", fieldName, StringComparison.Ordinal) == 0) { @@ -49,7 +49,7 @@ sealed class DSOCacheEntry [NativeAssembler (Ignore = true)] public string HashedName; - [NativeAssembler (UsesDataProvider = true)] + [NativeAssembler (UsesDataProvider = true, NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)] public ulong hash; public bool ignore; @@ -93,11 +93,11 @@ sealed class AssemblyStoreSingleAssemblyRuntimeData // src/monodroid/jni/xamarin-app.hh AssemblyStoreRuntimeData structure sealed class AssemblyStoreRuntimeData { - [NativePointer] + [NativePointer (IsNull = true)] public byte data_start; public uint assembly_count; - [NativePointer] + [NativePointer (IsNull = true)] public AssemblyStoreAssemblyDescriptor assemblies; } @@ -299,7 +299,7 @@ void HashAndSortDSOCache (LlvmIrVariable variable, LlvmIrModuleTarget target, ob throw new InvalidOperationException ($"Internal error: DSO cache entry has unexpected type {instance.Obj.GetType ()}"); } - entry.hash = GetXxHash (entry.HashedName, is64Bit); + entry.hash = MonoAndroidHelper.GetXxHash (entry.HashedName, is64Bit); } cache.Sort ((StructureInstance a, StructureInstance b) => a.Instance.hash.CompareTo (b.Instance.hash)); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs index 1972dba0d85..39a899bd31a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/AssemblyCompression.cs @@ -78,7 +78,7 @@ public static CompressionResult Compress (AssemblyData data, string outputDirect } destBytes = bytePool.Rent (LZ4Codec.MaximumOutputSize (sourceBytes.Length)); - int encodedLength = LZ4Codec.Encode (sourceBytes, 0, checked((int)fi.Length), destBytes, 0, destBytes.Length, LZ4Level.L09_HC); + int encodedLength = LZ4Codec.Encode (sourceBytes, 0, checked((int)fi.Length), destBytes, 0, destBytes.Length, LZ4Level.L12_MAX); if (encodedLength < 0) return CompressionResult.EncodingFailed; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs index b250bf24085..6209dc2d382 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrComposer.cs @@ -1,7 +1,5 @@ using System; using System.IO; -using System.IO.Hashing; -using System.Text; using Microsoft.Build.Utilities; @@ -41,18 +39,13 @@ public void Generate (LlvmIrModule module, AndroidTargetArch arch, StreamWriter LlvmIrGenerator generator = LlvmIrGenerator.Create (arch, fileName); generator.Generate (output, module); output.Flush (); - } - public static ulong GetXxHash (string str, bool is64Bit) - { - byte[] stringBytes = Encoding.UTF8.GetBytes (str); - if (is64Bit) { - return XxHash64.HashToUInt64 (stringBytes); - } - - return (ulong)XxHash32.HashToUInt32 (stringBytes); + CleanupAfterGeneration (arch); } + protected virtual void CleanupAfterGeneration (AndroidTargetArch arch) + {} + protected LlvmIrGlobalVariable EnsureGlobalVariable (LlvmIrVariable variable) { var gv = variable as LlvmIrGlobalVariable; diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs index bb81b03db0e..af1184bd77e 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrGenerator.cs @@ -29,6 +29,7 @@ sealed class GeneratorWriteContext public readonly LlvmIrMetadataManager MetadataManager; public string CurrentIndent { get; private set; } = String.Empty; public bool InVariableGroup { get; set; } + public LlvmIrVariableNumberFormat NumberFormat { get; set; } = LlvmIrVariableNumberFormat.Default; public GeneratorWriteContext (TextWriter writer, LlvmIrModule module, LlvmIrModuleTarget target, LlvmIrMetadataManager metadataManager) { @@ -80,31 +81,56 @@ sealed class BasicType public readonly string Name; public readonly ulong Size; public readonly bool IsNumeric; + public readonly bool IsUnsigned; + public readonly bool PreferHex; + public readonly string HexFormat; - public BasicType (string name, ulong size, bool isNumeric = true) + public BasicType (string name, ulong size, bool isNumeric = true, bool isUnsigned = false, bool? preferHex = null) { Name = name; Size = size; IsNumeric = isNumeric; + IsUnsigned = isUnsigned; + + // If hex preference isn't specified, we determine whether the type wants to be represented in + // the hexadecimal notation based on signedness. Unsigned types will be represented in hexadecimal, + // but signed types will remain decimal, as it's easier for humans to see the actual value of the + // variable, given this note from LLVM IR manual: + // + // Note that hexadecimal integers are sign extended from the number of active bits, i.e. the bit width minus the number of leading zeros. So ‘s0x0001’ of type ‘i16’ will be -1, not 1. + // + // See: https://llvm.org/docs/LangRef.html#simple-constants + // + if (preferHex.HasValue) { + PreferHex = preferHex.Value; + } else { + PreferHex = isUnsigned; + } + if (!PreferHex) { + HexFormat = String.Empty; + return; + } + + HexFormat = $"x{size * 2}"; } } public const string IRPointerType = "ptr"; static readonly Dictionary basicTypeMap = new Dictionary { - { typeof (bool), new ("i8", 1, isNumeric: false) }, - { typeof (byte), new ("i8", 1) }, - { typeof (char), new ("i16", 2) }, + { typeof (bool), new ("i1", 1, isNumeric: false, isUnsigned: true, preferHex: false) }, + { typeof (byte), new ("i8", 1, isUnsigned: true) }, + { typeof (char), new ("i16", 2, isUnsigned: true, preferHex: false) }, { typeof (sbyte), new ("i8", 1) }, { typeof (short), new ("i16", 2) }, - { typeof (ushort), new ("i16", 2) }, + { typeof (ushort), new ("i16", 2, isUnsigned: true) }, { typeof (int), new ("i32", 4) }, - { typeof (uint), new ("i32", 4) }, + { typeof (uint), new ("i32", 4, isUnsigned: true) }, { typeof (long), new ("i64", 8) }, - { typeof (ulong), new ("i64", 8) }, + { typeof (ulong), new ("i64", 8, isUnsigned: true) }, { typeof (float), new ("float", 4) }, { typeof (double), new ("double", 8) }, - { typeof (void), new ("void", 0, isNumeric: false) }, + { typeof (void), new ("void", 0, isNumeric: false, preferHex: false) }, }; public string FilePath { get; } @@ -191,6 +217,8 @@ void WriteGlobalVariables (GeneratorWriteContext context) } foreach (LlvmIrGlobalVariable gv in context.Module.GlobalVariables) { + context.NumberFormat = gv.NumberFormat; + if (gv is LlvmIrGroupDelimiterVariable groupDelimiter) { if (!context.InVariableGroup && !String.IsNullOrEmpty (groupDelimiter.Comment)) { context.Output.WriteLine (); @@ -205,6 +233,10 @@ void WriteGlobalVariables (GeneratorWriteContext context) continue; } + if (gv.TargetArch.HasValue && gv.TargetArch.Value != target.TargetArch) { + continue; + } + if (gv.BeforeWriteCallback != null) { gv.BeforeWriteCallback (gv, target, gv.BeforeWriteCallbackCallerState); } @@ -240,8 +272,10 @@ void WriteGlobalVariable (GeneratorWriteContext context, LlvmIrGlobalVariable va context.Output.Write (", align "); ulong alignment; - if (typeInfo.IsAggregate) { - ulong count = GetAggregateValueElementCount (variable); + if (variable.Alignment.HasValue) { + alignment = variable.Alignment.Value; + } else if (typeInfo.IsAggregate) { + ulong count = GetAggregateValueElementCount (context, variable); alignment = (ulong)target.GetAggregateAlignment ((int)typeInfo.MaxFieldAlignment, count * typeInfo.Size); } else if (typeInfo.IsStructure) { alignment = (ulong)target.GetAggregateAlignment ((int)typeInfo.MaxFieldAlignment, typeInfo.Size); @@ -280,7 +314,7 @@ void WriteTypeAndValue (GeneratorWriteContext context, LlvmIrVariable variable, return; } - throw new InvalidOperationException ($"Internal error: variable of type {variable.Type} must not have a null value"); + throw new InvalidOperationException ($"Internal error: variable '{variable.Name}'' of type {variable.Type} must not have a null value"); } if (valueType != variable.Type && !LlvmIrModule.NameValueArrayType.IsAssignableFrom (variable.Type)) { @@ -290,9 +324,9 @@ void WriteTypeAndValue (GeneratorWriteContext context, LlvmIrVariable variable, WriteValue (context, valueType, variable); } - ulong GetAggregateValueElementCount (LlvmIrVariable variable) => GetAggregateValueElementCount (variable.Type, variable.Value, variable as LlvmIrGlobalVariable); + ulong GetAggregateValueElementCount (GeneratorWriteContext context, LlvmIrVariable variable) => GetAggregateValueElementCount (context, variable.Type, variable.Value, variable as LlvmIrGlobalVariable); - ulong GetAggregateValueElementCount (Type type, object? value, LlvmIrGlobalVariable? globalVariable = null) + ulong GetAggregateValueElementCount (GeneratorWriteContext context, Type type, object? value, LlvmIrGlobalVariable? globalVariable = null) { if (!type.IsArray ()) { throw new InvalidOperationException ($"Internal error: unknown type {type} when trying to determine aggregate type element count"); @@ -300,6 +334,9 @@ ulong GetAggregateValueElementCount (Type type, object? value, LlvmIrGlobalVaria if (value == null) { if (globalVariable != null) { + if (globalVariable.ArrayDataProvider != null) { + return globalVariable.ArrayDataProvider.GetTotalDataSize (context.Target); + } return globalVariable.ArrayItemCount; } return 0; @@ -373,7 +410,7 @@ void WriteType (GeneratorWriteContext context, Type type, object? value, out Llv { if (IsStructureInstance (type)) { if (value == null) { - throw new ArgumentException ("must not be null for structure instances", nameof (value)); + throw new ArgumentException ($"must not be null for structure instances ({type})", nameof (value)); } WriteStructureType (context, (StructureInstance)value, out typeInfo); @@ -386,9 +423,9 @@ void WriteType (GeneratorWriteContext context, Type type, object? value, out Llv if (type.IsArray ()) { Type elementType = type.GetArrayElementType (); - ulong elementCount = GetAggregateValueElementCount (type, value, globalVariable); + ulong elementCount = GetAggregateValueElementCount (context, type, value, globalVariable); - WriteArrayType (context, elementType, elementCount, out typeInfo); + WriteArrayType (context, elementType, elementCount, globalVariable, out typeInfo); return; } @@ -404,6 +441,11 @@ void WriteType (GeneratorWriteContext context, Type type, object? value, out Llv } void WriteArrayType (GeneratorWriteContext context, Type elementType, ulong elementCount, out LlvmTypeInfo typeInfo) + { + WriteArrayType (context, elementType, elementCount, variable: null, out typeInfo); + } + + void WriteArrayType (GeneratorWriteContext context, Type elementType, ulong elementCount, LlvmIrGlobalVariable? variable, out LlvmTypeInfo typeInfo) { string irType; ulong size; @@ -420,6 +462,35 @@ void WriteArrayType (GeneratorWriteContext context, Type elementType, ulong elem } else { irType = GetIRType (elementType, out size, out isPointer); maxFieldAlignment = size; + + if (elementType.IsArray) { + if (variable == null) { + throw new InvalidOperationException ($"Internal error: array of arrays ({elementType}) requires variable to be defined"); + } + + // For the sake of simpler code, we currently assume that all the element arrays are of the same size, because that's the only scenario + // that we use at this time. + var value = variable.Value as ICollection; + if (value == null) { + throw new InvalidOperationException ($"Internal error: variable '{variable.Name}' of type '{variable.Type}' is required to have a value of type which implements the ICollection interface"); + } + + if (value.Count == 0) { + throw new InvalidOperationException ($"Internal error: variable '{variable.Name}' of type '{variable.Type}' is required to have a value which is a non-empty ICollection"); + } + + Array? firstItem = null; + foreach (object v in value) { + firstItem = (Array)v; + break; + } + + if (firstItem == null) { + throw new InvalidOperationException ($"Internal error: variable '{variable.Name}' of type '{variable.Type}' is required to have a value which is a non-empty ICollection with non-null elements"); + } + + irType = $"[{MonoAndroidHelper.CultureInvariantToString (firstItem.Length)} x {irType}]"; + } } typeInfo = new LlvmTypeInfo ( isPointer: isPointer, @@ -449,12 +520,17 @@ ulong GetStructureMaxFieldAlignment (StructureInfo si) void WriteValue (GeneratorWriteContext context, Type valueType, LlvmIrVariable variable) { + if (variable is LlvmIrGlobalVariable globalVariable && globalVariable.ArrayDataProvider != null) { + WriteStreamedArrayValue (context, globalVariable, globalVariable.ArrayDataProvider); + return; + } + if (variable.Type.IsArray ()) { bool zeroInitialize = false; if (variable is LlvmIrGlobalVariable gv) { zeroInitialize = gv.ZeroInitializeArray || variable.Value == null; } else { - zeroInitialize = GetAggregateValueElementCount (variable) == 0; + zeroInitialize = GetAggregateValueElementCount (context, variable) == 0; } if (zeroInitialize) { @@ -478,6 +554,29 @@ void AssertArraySize (StructureInstance si, StructureMemberInfo smi, ulong lengt throw new InvalidOperationException ($"Invalid array size in field '{smi.Info.Name}' of structure '{si.Info.Name}', expected {expectedLength}, found {length}"); } + void WriteInlineArray (GeneratorWriteContext context, byte[] bytes, bool encodeAsASCII) + { + if (encodeAsASCII) { + context.Output.Write ('c'); + context.Output.Write (QuoteString (bytes, bytes.Length, out _, nullTerminated: false)); + return; + } + + string irType = MapToIRType (typeof(byte)); + bool first = true; + context.Output.Write ("[ "); + foreach (byte b in bytes) { + if (!first) { + context.Output.Write (", "); + } else { + first = false; + } + + context.Output.Write ($"{irType} u0x{b:x02}"); + } + context.Output.Write (" ]"); + } + void WriteValue (GeneratorWriteContext context, StructureInstance structInstance, StructureMemberInfo smi, object? value) { if (smi.IsNativePointer) { @@ -495,8 +594,7 @@ void WriteValue (GeneratorWriteContext context, StructureInstance structInstance // Byte arrays are represented in the same way as strings, without the explicit NUL termination byte AssertArraySize (structInstance, smi, length, smi.ArrayElements); - context.Output.Write ('c'); - context.Output.Write (QuoteString (bytes, bytes.Length, out _, nullTerminated: false)); + WriteInlineArray (context, bytes, encodeAsASCII: false); return; } @@ -549,6 +647,27 @@ bool WriteNativePointerValue (GeneratorWriteContext context, StructureInstance s return false; } + string ToHex (BasicType basicTypeDesc, Type type, object? value) + { + const char prefixSigned = 's'; + const char prefixUnsigned = 'u'; + + string hex; + if (type == typeof(byte)) { + hex = ((byte)value).ToString (basicTypeDesc.HexFormat, CultureInfo.InvariantCulture); + } else if (type == typeof(ushort)) { + hex = ((ushort)value).ToString (basicTypeDesc.HexFormat, CultureInfo.InvariantCulture); + } else if (type == typeof(uint)) { + hex = ((uint)value).ToString (basicTypeDesc.HexFormat, CultureInfo.InvariantCulture); + } else if (type == typeof(ulong)) { + hex = ((ulong)value).ToString (basicTypeDesc.HexFormat, CultureInfo.InvariantCulture); + } else { + throw new NotImplementedException ($"Conversion to hexadecimal from type {type} is not implemented"); + }; + + return $"{(basicTypeDesc.IsUnsigned ? prefixUnsigned : prefixSigned)}0x{hex}"; + } + void WriteValue (GeneratorWriteContext context, Type type, object? value) { if (value is LlvmIrVariable variableRef) { @@ -556,14 +675,26 @@ void WriteValue (GeneratorWriteContext context, Type type, object? value) return; } - if (IsNumeric (type)) { - context.Output.Write (MonoAndroidHelper.CultureInvariantToString (value)); - return; - } + bool isBasic = basicTypeMap.TryGetValue (type, out BasicType basicTypeDesc); + if (isBasic) { + if (basicTypeDesc.IsNumeric) { + bool hex = context.NumberFormat switch { + LlvmIrVariableNumberFormat.Default => basicTypeDesc.PreferHex, + LlvmIrVariableNumberFormat.Decimal => false, + LlvmIrVariableNumberFormat.Hexadecimal => true, + _ => throw new InvalidOperationException ($"Internal error: number format {context.NumberFormat} is unsupported") + }; - if (type == typeof(bool)) { - context.Output.Write ((bool)value ? '1' : '0'); - return; + context.Output.Write ( + hex ? ToHex (basicTypeDesc, type, value) : MonoAndroidHelper.CultureInvariantToString (value) + ); + return; + } + + if (type == typeof(bool)) { + context.Output.Write ((bool)value ? "true" : "false"); + return; + } } if (IsStructureInstance (type)) { @@ -588,8 +719,13 @@ void WriteValue (GeneratorWriteContext context, Type type, object? value) return; } - if (type.IsInlineArray ()) { + if (type.IsArray) { + if (type == typeof(byte[])) { + WriteInlineArray (context, (byte[])value, encodeAsASCII: true); + return; + } + throw new NotSupportedException ($"Internal error: array of type {type} is unsupported"); } throw new NotSupportedException ($"Internal error: value type '{type}' is unsupported"); @@ -616,8 +752,20 @@ void WriteStructureValue (GeneratorWriteContext context, StructureInstance? inst context.Output.Write (' '); object? value = GetTypedMemberValue (context, info, smi, instance, smi.MemberType); + LlvmIrVariableNumberFormat numberFormat = smi.Info.GetNumberFormat (); + LlvmIrVariableNumberFormat? savedNumberFormat = null; + + if (numberFormat != LlvmIrVariableNumberFormat.Default && numberFormat != context.NumberFormat) { + savedNumberFormat = context.NumberFormat; + context.NumberFormat = numberFormat; + } + WriteValue (context, instance, smi, value); + if (savedNumberFormat.HasValue) { + context.NumberFormat = savedNumberFormat.Value; + } + if (i < lastMember) { context.Output.Write (", "); } @@ -628,9 +776,6 @@ void WriteStructureValue (GeneratorWriteContext context, StructureInstance? inst sb.Append (MapManagedTypeToNative (smi)); sb.Append (' '); sb.Append (smi.Info.Name); - if (value != null && smi.MemberType.IsPrimitive && smi.MemberType != typeof(bool)) { - sb.Append ($" (0x{value:x})"); - } comment = sb.ToString (); } WriteCommentLine (context, comment); @@ -641,75 +786,80 @@ void WriteStructureValue (GeneratorWriteContext context, StructureInstance? inst context.Output.Write ('}'); } - void WriteArrayValue (GeneratorWriteContext context, LlvmIrVariable variable) + void WriteArrayValueStart (GeneratorWriteContext context) { - ICollection entries; - if (variable.Type.ImplementsInterface (typeof(IDictionary))) { - var list = new List (); - foreach (var kvp in (IDictionary)variable.Value) { - list.Add (kvp.Key); - list.Add (kvp.Value); - } - entries = list; - } else { - entries = (ICollection)variable.Value; - } - - if (entries.Count == 0) { - context.Output.Write ("zeroinitializer"); - return; - } - context.Output.WriteLine ('['); context.IncreaseIndent (); + } - Type elementType = variable.Type.GetArrayElementType (); - bool writeIndices = (variable.WriteOptions & LlvmIrVariableWriteOptions.ArrayWriteIndexComments) == LlvmIrVariableWriteOptions.ArrayWriteIndexComments; - ulong counter = 0; - string? prevItemComment = null; - uint stride; + void WriteArrayValueEnd (GeneratorWriteContext context) + { + context.DecreaseIndent (); + context.Output.Write (']'); + } + uint GetArrayStride (LlvmIrVariable variable) + { if ((variable.WriteOptions & LlvmIrVariableWriteOptions.ArrayFormatInRows) == LlvmIrVariableWriteOptions.ArrayFormatInRows) { - stride = variable.ArrayStride > 0 ? variable.ArrayStride : 1; - } else { - stride = 1; + return variable.ArrayStride > 0 ? variable.ArrayStride : 1; } + return 1; + } + + void WriteArrayEntries (GeneratorWriteContext context, LlvmIrVariable variable, ICollection? entries, Type elementType, uint stride, bool writeIndices, bool terminateWithComma = false) + { bool first = true; + bool ignoreComments = stride > 1; + string? prevItemComment = null; + ulong counter = 0; - // TODO: implement output in rows - foreach (object entry in entries) { - if (!first) { - context.Output.Write (','); - WritePrevItemCommentOrNewline (); - } else { - first = false; - } + if (entries != null) { + foreach (object entry in entries) { + if (!first) { + context.Output.Write (','); + if (stride == 1 || counter % stride == 0) { + WritePrevItemCommentOrNewline (); + context.Output.Write (context.CurrentIndent); + } else { + context.Output.Write (' '); + } + } else { + context.Output.Write (context.CurrentIndent); + first = false; + } - prevItemComment = null; - if (variable.GetArrayItemCommentCallback != null) { - prevItemComment = variable.GetArrayItemCommentCallback (variable, target, counter, entry, variable.GetArrayItemCommentCallbackCallerState); - } + if (!ignoreComments) { + prevItemComment = null; + if (variable.GetArrayItemCommentCallback != null) { + prevItemComment = variable.GetArrayItemCommentCallback (variable, target, counter, entry, variable.GetArrayItemCommentCallbackCallerState); + } - if (writeIndices && String.IsNullOrEmpty (prevItemComment)) { - prevItemComment = $" {counter}"; - } + if (writeIndices && String.IsNullOrEmpty (prevItemComment)) { + prevItemComment = $" {counter}"; + } + } - counter++; - context.Output.Write (context.CurrentIndent); - WriteType (context, elementType, entry, out _); + counter++; + WriteType (context, elementType, entry, out _); - context.Output.Write (' '); - WriteValue (context, elementType, entry); + context.Output.Write (' '); + WriteValue (context, elementType, entry); + } } - WritePrevItemCommentOrNewline (); - context.DecreaseIndent (); - context.Output.Write (']'); + if (terminateWithComma) { + if (!ignoreComments) { + context.Output.WriteLine (); // must put comma outside the comment + context.Output.Write (context.CurrentIndent); + } + context.Output.Write (','); + } + WritePrevItemCommentOrNewline (); void WritePrevItemCommentOrNewline () { - if (!String.IsNullOrEmpty (prevItemComment)) { + if (!ignoreComments && !String.IsNullOrEmpty (prevItemComment)) { context.Output.Write (' '); WriteCommentLine (context, prevItemComment); } else { @@ -718,6 +868,97 @@ void WritePrevItemCommentOrNewline () } } + bool ArrayWantsToWriteIndices (LlvmIrVariable variable) => (variable.WriteOptions & LlvmIrVariableWriteOptions.ArrayWriteIndexComments) == LlvmIrVariableWriteOptions.ArrayWriteIndexComments; + + void WriteStreamedArrayValue (GeneratorWriteContext context, LlvmIrGlobalVariable variable, LlvmIrStreamedArrayDataProvider dataProvider) + { + ulong dataSizeSoFar = 0; + ulong totalDataSize = dataProvider.GetTotalDataSize (context.Target); + bool first = true; + + WriteArrayValueStart (context); + while (true) { + (LlvmIrStreamedArrayDataProviderState state, ICollection? data) = dataProvider.GetData (context.Target); + if (state == LlvmIrStreamedArrayDataProviderState.NextSectionNoData) { + continue; + } + + bool mustHaveData = state != LlvmIrStreamedArrayDataProviderState.LastSectionNoData; + if (mustHaveData) { + if (data.Count == 0) { + throw new InvalidOperationException ("Data must be provided for streamed arrays"); + } + + dataSizeSoFar += (ulong)data.Count; + if (dataSizeSoFar > totalDataSize) { + throw new InvalidOperationException ($"Data provider {dataProvider} is trying to write more data than declared"); + } + + if (first) { + first = false; + } else { + context.Output.WriteLine (); + } + string comment = dataProvider.GetSectionStartComment (context.Target); + + if (comment.Length > 0) { + context.Output.Write (context.CurrentIndent); + WriteCommentLine (context, comment); + } + } + + bool lastSection = state == LlvmIrStreamedArrayDataProviderState.LastSection || state == LlvmIrStreamedArrayDataProviderState.LastSectionNoData; + WriteArrayEntries ( + context, + variable, + data, + dataProvider.ArrayElementType, + GetArrayStride (variable), + writeIndices: false, + terminateWithComma: !lastSection + ); + + if (lastSection) { + break; + } + + } + WriteArrayValueEnd (context); + } + + void WriteArrayValue (GeneratorWriteContext context, LlvmIrVariable variable) + { + ICollection entries; + if (variable.Type.ImplementsInterface (typeof(IDictionary))) { + var list = new List (); + foreach (var kvp in (IDictionary)variable.Value) { + list.Add (kvp.Key); + list.Add (kvp.Value); + } + entries = list; + } else { + entries = (ICollection)variable.Value; + } + + if (entries.Count == 0) { + context.Output.Write ("zeroinitializer"); + return; + } + + WriteArrayValueStart (context); + + WriteArrayEntries ( + context, + variable, + entries, + variable.Type.GetArrayElementType (), + GetArrayStride (variable), + writeIndices: ArrayWantsToWriteIndices (variable) + ); + + WriteArrayValueEnd (context); + } + void WriteLinkage (GeneratorWriteContext context, LlvmIrLinkage linkage) { if (linkage == LlvmIrLinkage.Default) { @@ -1232,8 +1473,6 @@ static string MapManagedTypeToNative (StructureMemberInfo smi) return $"{nativeType}*"; } - static bool IsNumeric (Type type) => basicTypeMap.TryGetValue (type, out BasicType typeDesc) && typeDesc.IsNumeric; - object? GetTypedMemberValue (GeneratorWriteContext context, StructureInfo info, StructureMemberInfo smi, StructureInstance instance, Type expectedType, object? defaultValue = null) { object? value = smi.GetValue (instance.Obj); diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs index be1570dafe8..9a50f4ee5cf 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrModule.cs @@ -464,7 +464,7 @@ void AddStringArrayGlobalVariable (LlvmIrGlobalVariable variable, string? string Register (kvp.Value); } } else if (typeof(ICollection).IsAssignableFrom (variable.Type)) { - foreach (string s in (ICollection)variable.Value) { + foreach (string s in (ICollection)variable.Value) { Register (s); } } else { @@ -473,8 +473,12 @@ void AddStringArrayGlobalVariable (LlvmIrGlobalVariable variable, string? string AddStandardGlobalVariable (variable); - void Register (string value) + void Register (string? value) { + if (value == null) { + return; + } + RegisterString (value, stringGroupName, stringGroupComment, symbolSuffix); } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs index 7f741490ce0..d66bda3d7c9 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/LlvmIrVariable.cs @@ -1,6 +1,9 @@ using System; +using System.Collections; using System.Globalization; +using Xamarin.Android.Tools; + namespace Xamarin.Android.Tasks.LLVMIR; [Flags] @@ -11,6 +14,13 @@ enum LlvmIrVariableWriteOptions ArrayFormatInRows = 0x0002, } +enum LlvmIrVariableNumberFormat +{ + Default, + Hexadecimal, + Decimal, +} + abstract class LlvmIrVariable : IEquatable { public abstract bool Global { get; } @@ -29,6 +39,8 @@ abstract class LlvmIrVariable : IEquatable public object? Value { get; set; } public string? Comment { get; set; } + public LlvmIrVariableNumberFormat NumberFormat { get; set; } = LlvmIrVariableNumberFormat.Decimal; + /// /// Both global and local variables will want their names to matter in equality checks, but function /// parameters must not take it into account, thus this property. If set to false, @@ -146,6 +158,50 @@ public void AssignNumber (ulong n) } } +enum LlvmIrStreamedArrayDataProviderState +{ + NextSection, + LastSection, + NextSectionNoData, + LastSectionNoData, +} + +abstract class LlvmIrStreamedArrayDataProvider +{ + /// + /// Type of every member of the array returned by . Generator will check + /// every member type against this property, allowing also derived types. + /// + public Type ArrayElementType { get; } + + protected LlvmIrStreamedArrayDataProvider (Type arrayElementType) + { + ArrayElementType = arrayElementType; + } + + /// + /// Whenever returns the generator will call this method to obtain the new section + /// comment, if any, to be output before the actual data. Returning `String.Empty` prevents the comment + /// from being added. + /// + public virtual string GetSectionStartComment (LlvmIrModuleTarget target) => String.Empty; + + /// + /// Provide the next chunk of data for the specified target (ABI). Implementations need to return at least one + /// non-empty collection of data. The returned collection **must** be exactly the size of contained data (e.g. it cannot be + /// a byte array rented from a byte pool, because these can be bigger than requested. When returning the last (or the only) section, + /// must have a value of . + /// Each section may be preceded by a comment, . + /// + public abstract (LlvmIrStreamedArrayDataProviderState status, ICollection data) GetData (LlvmIrModuleTarget target); + + /// + /// Provide the total data size for the specified target (ABI). This needs to be used instead of + /// because a variable instance is created once and shared by all targets, while per-target data sets might have different sizes. + /// + public abstract ulong GetTotalDataSize (LlvmIrModuleTarget target); +} + class LlvmIrGlobalVariable : LlvmIrVariable { /// @@ -162,9 +218,40 @@ class LlvmIrGlobalVariable : LlvmIrVariable /// public virtual LlvmIrVariableOptions? Options { get; set; } + /// + /// There are situations when a variable differs enough between architectures, that the difference cannot be + /// handled with . In such situations one can create a separate variable + /// for each architecture and set this property. + /// + public AndroidTargetArch? TargetArch { get; set; } + + /// + /// If set to `true`, initialize the array with a shortcut zero-initializer statement. Useful when pre-allocating + /// space for runtime use that won't be filled in with any data at the build time. + /// public bool ZeroInitializeArray { get; set; } + + /// + /// Specify number of items in an array. Used in cases when we want to pre-allocate an array without giving it any + /// value, thus making it impossible for the generator to discover the number of items automatically. This is useful + /// when using . This property is used **only** if the variable + /// is `null`. + /// public ulong ArrayItemCount { get; set; } + /// + /// If set, it will override any automatically calculated alignment for this variable + /// + public ulong? Alignment { get; set; } + + /// + /// If set, the provider will be called to obtain all the data to be placed in an array variable. The total amount + /// of data that will be returned by the provider **must** be specified in the property, + /// in order for the generator to properly declare the variable. The generator will verify that the amount of data + /// is exactly that much and throw an exception otherwise. + /// + public LlvmIrStreamedArrayDataProvider? ArrayDataProvider { get; set; } + /// /// Constructs a local variable. is translated to one of the LLVM IR first class types (see /// https://llvm.org/docs/LangRef.html#t-firstclass) only if it's an integral or floating point type. In all other cases it @@ -196,11 +283,16 @@ public LlvmIrGlobalVariable (object value, string name, LlvmIrVariableOptions? o /// generating output for a specific target (e.g. 32-bit vs 64-bit integer variables). If the variable requires such /// type changes, this should be done at generation time from within the method. /// - public void OverrideValueAndType (Type newType, object? newValue) + public void OverrideTypeAndValue (Type newType, object? newValue) { Type = newType; Value = newValue; } + + public void OverrideName (string newName) + { + Name = newName; + } } class LlvmIrStringVariable : LlvmIrGlobalVariable diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/MemberInfoUtilities.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/MemberInfoUtilities.cs index 431d92b6229..713b0c5a1a6 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/MemberInfoUtilities.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/MemberInfoUtilities.cs @@ -71,5 +71,15 @@ public static bool InlineArrayNeedsPadding (this MemberInfo mi) return attr.NeedsPadding; } + + public static LlvmIrVariableNumberFormat GetNumberFormat (this MemberInfo mi) + { + var attr = mi.GetCustomAttribute (); + if (attr == null) { + return LlvmIrVariableNumberFormat.Default; + } + + return attr.NumberFormat; + } } } diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/NativeAssemblerAttribute.cs b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/NativeAssemblerAttribute.cs index 39fc5e094bb..c239255e1ac 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/NativeAssemblerAttribute.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/LlvmIrGenerator/NativeAssemblerAttribute.cs @@ -33,6 +33,8 @@ class NativeAssemblerAttribute : Attribute /// size to which the member must be padded is specified by /// public bool NeedsPadding { get; set; } + + public LLVMIR.LlvmIrVariableNumberFormat NumberFormat { get; set; } = LLVMIR.LlvmIrVariableNumberFormat.Default; } [AttributeUsage (AttributeTargets.Class, Inherited = true)] diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs index c30df45bb8e..ee6fc0e7f28 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MarshalMethodsNativeAssemblyGenerator.cs @@ -108,7 +108,7 @@ public override string GetComment (object data, string fieldName) var klass = EnsureType (data); if (String.Compare ("token", fieldName, StringComparison.Ordinal) == 0) { - return $" token 0x{klass.token:x}; class name: {klass.ClassName}"; + return $" class name: {klass.ClassName}"; } return String.Empty; @@ -118,7 +118,7 @@ public override string GetComment (object data, string fieldName) [NativeAssemblerStructContextDataProvider (typeof(MarshalMethodsManagedClassDataProvider))] sealed class MarshalMethodsManagedClass { - [NativeAssembler (UsesDataProvider = true)] + [NativeAssembler (UsesDataProvider = true, NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)] public uint token; [NativePointer (IsNull = true)] @@ -982,10 +982,10 @@ void AddAssemblyImageCache (LlvmIrModule module, out AssemblyCacheState acs) foreach (string name in uniqueAssemblyNames) { // We must make sure we keep the possible culture prefix, which will be treated as "directory" path here string clippedName = Path.Combine (Path.GetDirectoryName (name) ?? String.Empty, Path.GetFileNameWithoutExtension (name)); - ulong hashFull32 = GetXxHash (name, is64Bit: false); - ulong hashClipped32 = GetXxHash (clippedName, is64Bit: false); - ulong hashFull64 = GetXxHash (name, is64Bit: true); - ulong hashClipped64 = GetXxHash (clippedName, is64Bit: true); + ulong hashFull32 = MonoAndroidHelper.GetXxHash (name, is64Bit: false); + ulong hashClipped32 = MonoAndroidHelper.GetXxHash (clippedName, is64Bit: false); + ulong hashFull64 = MonoAndroidHelper.GetXxHash (name, is64Bit: true); + ulong hashClipped64 = MonoAndroidHelper.GetXxHash (clippedName, is64Bit: true); // // If the number of name forms changes, xamarin-app.hh MUST be updated to set value of the @@ -1021,6 +1021,7 @@ void AddAssemblyImageCache (LlvmIrModule module, out AssemblyCacheState acs) BeforeWriteCallbackCallerState = acs, GetArrayItemCommentCallback = GetAssemblyImageCacheItemComment, GetArrayItemCommentCallbackCallerState = acs, + NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal, }; module.Add (assembly_image_cache_hashes); @@ -1047,7 +1048,7 @@ void UpdateAssemblyImageCacheHashes (LlvmIrVariable variable, LlvmIrModuleTarget } LlvmIrGlobalVariable gv = EnsureGlobalVariable (variable); - gv.OverrideValueAndType (type, value); + gv.OverrideTypeAndValue (type, value); } string? GetAssemblyImageCacheItemComment (LlvmIrVariable v, LlvmIrModuleTarget target, ulong index, object? value, object? callerState) @@ -1081,7 +1082,7 @@ void UpdateAssemblyImageCacheIndices (LlvmIrVariable variable, LlvmIrModuleTarge } LlvmIrGlobalVariable gv = EnsureGlobalVariable (variable); - gv.OverrideValueAndType (variable.Type, value); + gv.OverrideTypeAndValue (variable.Type, value); } AssemblyCacheState EnsureAssemblyCacheState (object? callerState) diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Basic.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Basic.cs new file mode 100644 index 00000000000..3e8fe9a2d8c --- /dev/null +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.Basic.cs @@ -0,0 +1,200 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO.Hashing; +using System.Text; + +using Xamarin.Android.Tools; + +namespace Xamarin.Android.Tasks; + +partial class MonoAndroidHelper +{ + public static class AndroidAbi + { + public const string Arm32 = "armeabi-v7a"; + public const string Arm64 = "arm64-v8a"; + public const string X86 = "x86"; + public const string X64 = "x86_64"; + } + + public static class RuntimeIdentifier + { + public const string Arm32 = "android-arm"; + public const string Arm64 = "android-arm64"; + public const string X86 = "android-x86"; + public const string X64 = "android-x64"; + } + + public static readonly HashSet SupportedTargetArchitectures = new HashSet { + AndroidTargetArch.Arm, + AndroidTargetArch.Arm64, + AndroidTargetArch.X86, + AndroidTargetArch.X86_64, + }; + + static readonly char[] ZipPathTrimmedChars = {'/', '\\'}; + + static readonly Dictionary ClangAbiMap = new Dictionary (StringComparer.OrdinalIgnoreCase) { + {"arm64-v8a", "aarch64"}, + {"armeabi-v7a", "arm"}, + {"x86", "i686"}, + {"x86_64", "x86_64"} + }; + + static readonly Dictionary AbiToArchMap = new Dictionary (StringComparer.OrdinalIgnoreCase) { + { AndroidAbi.Arm32, AndroidTargetArch.Arm }, + { AndroidAbi.Arm64, AndroidTargetArch.Arm64 }, + { AndroidAbi.X86, AndroidTargetArch.X86 }, + { AndroidAbi.X64, AndroidTargetArch.X86_64 }, + }; + + static readonly Dictionary AbiToRidMap = new Dictionary (StringComparer.OrdinalIgnoreCase) { + { AndroidAbi.Arm32, RuntimeIdentifier.Arm32 }, + { AndroidAbi.Arm64, RuntimeIdentifier.Arm64 }, + { AndroidAbi.X86, RuntimeIdentifier.X86 }, + { AndroidAbi.X64, RuntimeIdentifier.X64 }, + }; + + static readonly Dictionary RidToAbiMap = new Dictionary (StringComparer.OrdinalIgnoreCase) { + { RuntimeIdentifier.Arm32, AndroidAbi.Arm32 }, + { RuntimeIdentifier.Arm64, AndroidAbi.Arm64 }, + { RuntimeIdentifier.X86, AndroidAbi.X86 }, + { RuntimeIdentifier.X64, AndroidAbi.X64 }, + }; + + static readonly Dictionary RidToArchMap = new Dictionary (StringComparer.OrdinalIgnoreCase) { + { RuntimeIdentifier.Arm32, AndroidTargetArch.Arm }, + { RuntimeIdentifier.Arm64, AndroidTargetArch.Arm64 }, + { RuntimeIdentifier.X86, AndroidTargetArch.X86 }, + { RuntimeIdentifier.X64, AndroidTargetArch.X86_64 }, + }; + + static readonly Dictionary ArchToRidMap = new Dictionary { + { AndroidTargetArch.Arm, RuntimeIdentifier.Arm32 }, + { AndroidTargetArch.Arm64, RuntimeIdentifier.Arm64 }, + { AndroidTargetArch.X86, RuntimeIdentifier.X86 }, + { AndroidTargetArch.X86_64, RuntimeIdentifier.X64 }, + }; + + static readonly Dictionary ArchToAbiMap = new Dictionary { + { AndroidTargetArch.Arm, AndroidAbi.Arm32 }, + { AndroidTargetArch.Arm64, AndroidAbi.Arm64 }, + { AndroidTargetArch.X86, AndroidAbi.X86 }, + { AndroidTargetArch.X86_64, AndroidAbi.X64 }, + }; + + public static AndroidTargetArch AbiToTargetArch (string abi) + { + if (!AbiToArchMap.TryGetValue (abi, out AndroidTargetArch arch)) { + return AndroidTargetArch.None; + }; + + return arch; + } + + public static string AbiToRid (string abi) + { + if (!AbiToRidMap.TryGetValue (abi, out string rid)) { + throw new NotSupportedException ($"Internal error: unsupported ABI '{abi}'"); + }; + + return rid; + } + + public static string RidToAbi (string rid) + { + if (!RidToAbiMap.TryGetValue (rid, out string abi)) { + throw new NotSupportedException ($"Internal error: unsupported Runtime Identifier '{rid}'"); + }; + + return abi; + } + + public static AndroidTargetArch RidToArch (string rid) + { + if (!RidToArchMap.TryGetValue (rid, out AndroidTargetArch arch)) { + throw new NotSupportedException ($"Internal error: unsupported Runtime Identifier '{rid}'"); + }; + + return arch; + } + + public static string ArchToRid (AndroidTargetArch arch) + { + if (!ArchToRidMap.TryGetValue (arch, out string rid)) { + throw new InvalidOperationException ($"Internal error: unsupported architecture '{arch}'"); + }; + + return rid; + } + + public static string ArchToAbi (AndroidTargetArch arch) + { + if (!ArchToAbiMap.TryGetValue (arch, out string abi)) { + throw new InvalidOperationException ($"Internal error: unsupported architecture '{arch}'"); + }; + + return abi; + } + + public static string? CultureInvariantToString (object? obj) + { + if (obj == null) { + return null; + } + + return Convert.ToString (obj, CultureInfo.InvariantCulture); + } + + public static string MapAndroidAbiToClang (string androidAbi) + { + if (ClangAbiMap.TryGetValue (androidAbi, out string clangAbi)) { + return clangAbi; + } + return null; + } + + public static string MakeZipArchivePath (string part1, params string[]? pathParts) + { + return MakeZipArchivePath (part1, (ICollection?)pathParts); + } + + public static string MakeZipArchivePath (string part1, ICollection? pathParts) + { + var parts = new List (); + if (!String.IsNullOrEmpty (part1)) { + parts.Add (part1.TrimEnd (ZipPathTrimmedChars)); + }; + + if (pathParts != null && pathParts.Count > 0) { + foreach (string p in pathParts) { + if (String.IsNullOrEmpty (p)) { + continue; + } + parts.Add (p.TrimEnd (ZipPathTrimmedChars)); + } + } + + if (parts.Count == 0) { + return String.Empty; + } + + return String.Join ("/", parts); + } + + public static bool IsValidAbi (string abi) => AbiToRidMap.ContainsKey (abi); + + public static byte[] Utf8StringToBytes (string str) => Encoding.UTF8.GetBytes (str); + + public static ulong GetXxHash (string str, bool is64Bit) => GetXxHash (Utf8StringToBytes (str), is64Bit); + + public static ulong GetXxHash (byte[] stringBytes, bool is64Bit) + { + if (is64Bit) { + return XxHash64.HashToUInt64 (stringBytes); + } + + return (ulong)XxHash32.HashToUInt32 (stringBytes); + } +} diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs index 4efceda5141..9c2794a367a 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/MonoAndroidHelper.cs @@ -1,18 +1,16 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.Globalization; using System.Linq; using System.IO; using System.Reflection.Metadata; using System.Reflection.PortableExecutable; -using System.Security.Cryptography; using System.Text; using Xamarin.Android.Tools; using Xamarin.Tools.Zip; -using Microsoft.Android.Build.Tasks; #if MSBUILD +using Microsoft.Android.Build.Tasks; using Microsoft.Build.Framework; using Microsoft.Build.Utilities; #endif @@ -277,22 +275,6 @@ public static void LogWarning (object log, string msg, params object [] args) } #if MSBUILD - static readonly Dictionary ClangAbiMap = new Dictionary (StringComparer.OrdinalIgnoreCase) { - {"arm64-v8a", "aarch64"}, - {"armeabi-v7a", "arm"}, - {"x86", "i686"}, - {"x86_64", "x86_64"} - }; - - public static string MapAndroidAbiToClang (string androidAbi) - { - if (ClangAbiMap.TryGetValue (androidAbi, out string clangAbi)) { - return clangAbi; - } - return null; - } -#endif - public static bool IsMonoAndroidAssembly (ITaskItem assembly) { var tfi = assembly.GetMetadata ("TargetFrameworkIdentifier"); @@ -313,6 +295,7 @@ public static bool HasMonoAndroidReference (ITaskItem assembly) var reader = pe.GetMetadataReader (); return HasMonoAndroidReference (reader); } +#endif public static bool HasMonoAndroidReference (MetadataReader reader) { @@ -387,7 +370,6 @@ internal static IEnumerable GetFrameworkAssembliesToTreatAsUserAssemb return ret; } -#endif public static bool SaveMapFile (IBuildEngine4 engine, string mapFile, Dictionary map) { @@ -448,6 +430,7 @@ public static bool SaveCustomViewMapFile (IBuildEngine4 engine, string mapFile, return Files.CopyIfStreamChanged (writer.BaseStream, mapFile); } } +#endif // MSBUILD public static string [] GetProguardEnvironmentVaribles (string proguardHome) { @@ -482,6 +465,8 @@ public static IEnumerable Executables (string executable) yield return executable; } + +#if MSBUILD public static string TryGetAndroidJarPath (TaskLoggingHelper log, string platform, bool designTimeBuild = false, bool buildingInsideVisualStudio = false, string targetFramework = "", string androidSdkDirectory = "") { var platformPath = MonoAndroidHelper.AndroidSdk.TryGetPlatformDirectoryFromApiLevel (platform, MonoAndroidHelper.SupportedVersions); @@ -503,7 +488,7 @@ public static void SaveResourceCaseMap (IBuildEngine4 engine, Dictionary LoadResourceCaseMap (IBuildEngine4 engine, Func keyCallback) => engine.GetRegisteredTaskObjectAssemblyLocal> (keyCallback (ResourceCaseMapKey), RegisteredTaskObjectLifetime.Build) ?? new Dictionary (0); - +#endif // MSBUILD public static string FixUpAndroidResourcePath (string file, string resourceDirectory, string resourceDirectoryFullPath, Dictionary resource_name_case_map) { string newfile = null; @@ -523,7 +508,7 @@ public static string FixUpAndroidResourcePath (string file, string resourceDirec } static readonly char [] DirectorySeparators = new [] { Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar }; - +#if MSBUILD /// /// Returns the relative path that should be used for an @(AndroidAsset) item /// @@ -535,57 +520,9 @@ public static string GetRelativePathForAndroidAsset (string assetsDirectory, ITa path = head.Length == path.Length ? path : path.Substring ((head.Length == 0 ? 0 : head.Length + 1) + assetsDirectory.Length).TrimStart (DirectorySeparators); return path; } +#endif // MSBUILD - public static AndroidTargetArch AbiToTargetArch (string abi) - { - return abi switch { - "armeabi-v7a" => AndroidTargetArch.Arm, - "arm64-v8a" => AndroidTargetArch.Arm64, - "x86_64" => AndroidTargetArch.X86_64, - "x86" => AndroidTargetArch.X86, - _ => throw new NotSupportedException ($"Internal error: unsupported ABI '{abi}'") - }; - } - - public static string AbiToRid (string abi) - { - switch (abi) { - case "arm64-v8a": - return "android-arm64"; - case "armeabi-v7a": - return "android-arm"; - - case "x86": - return "android-x86"; - - case "x86_64": - return "android-x64"; - - default: - throw new InvalidOperationException ($"Internal error: unsupported ABI '{abi}'"); - } - } - - public static string ArchToRid (AndroidTargetArch arch) - { - return arch switch { - AndroidTargetArch.Arm64 => "android-arm64", - AndroidTargetArch.Arm => "android-arm", - AndroidTargetArch.X86 => "android-x86", - AndroidTargetArch.X86_64 => "android-x64", - _ => throw new InvalidOperationException ($"Internal error: unsupported ABI '{arch}'") - }; - } - - public static string? CultureInvariantToString (object? obj) - { - if (obj == null) { - return null; - } - - return Convert.ToString (obj, CultureInfo.InvariantCulture); - } /// /// Converts $(SupportedOSPlatformVersion) to an API level, as it can be a version (21.0), or an int (21). @@ -604,16 +541,15 @@ public static int ConvertSupportedOSPlatformVersionToApiLevel (string version) return apiLevel; } - public static AndroidTargetArch GetTargetArch (ITaskItem asmItem) +#if MSBUILD + public static string? GetAssemblyAbi (ITaskItem asmItem) { - string? abi = asmItem.GetMetadata ("Abi"); - if (String.IsNullOrEmpty (abi)) { - return AndroidTargetArch.None; - } - - return AbiToTargetArch (abi); + return asmItem.GetMetadata ("Abi"); } + public static AndroidTargetArch GetTargetArch (ITaskItem asmItem) => AbiToTargetArch (GetAssemblyAbi (asmItem)); +#endif // MSBUILD + static string GetToolsRootDirectoryRelativePath (string androidBinUtilsDirectory) { // We need to link against libc and libm, but since NDK is not in use, the linker won't be able to find the actual Android libraries. diff --git a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs index 01f979632d8..61b3f5c314e 100644 --- a/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs +++ b/src/Xamarin.Android.Build.Tasks/Utilities/TypeMappingReleaseNativeAssemblyGenerator.cs @@ -71,6 +71,7 @@ sealed class TypeMapModuleEntry [NativeAssembler (Ignore = true)] public TypeMapJava JavaTypeMapEntry; + [NativeAssembler (NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)] public uint type_token_id; public uint java_map_index; } @@ -128,6 +129,8 @@ sealed class TypeMapJava public ulong JavaNameHash64; public uint module_index; + + [NativeAssembler (NumberFormat = LlvmIrVariableNumberFormat.Hexadecimal)] public uint type_token_id; public uint java_name_index; } @@ -261,7 +264,7 @@ uint GetJavaEntryIndex (TypeMapJava javaEntry) throw new InvalidOperationException ("Internal error: construction state expected but not found"); } - return $" {index}: 0x{value:x} => {cs.JavaMap[(int)index].Instance.JavaName}"; + return $" {index}: {cs.JavaMap[(int)index].Instance.JavaName}"; } void GenerateAndSortJavaHashes (LlvmIrVariable variable, LlvmIrModuleTarget target, object? callerState) @@ -290,7 +293,7 @@ void GenerateAndSortJavaHashes (LlvmIrVariable variable, LlvmIrModuleTarget targ hashes = list; } - gv.OverrideValueAndType (listType, hashes); + gv.OverrideTypeAndValue (listType, hashes); } ConstructionState EnsureConstructionState (object? callerState) From fc5506d8f72351cf46777ba39986b0e00c8319f6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 21 Feb 2024 08:30:50 -0500 Subject: [PATCH 05/18] Bump to xamarin/Java.Interop/main@ae65609 (#8744) Changes: https://github.com/xamarin/java.interop/compare/7d1e7057cf4b0adcf65e7064186326dafce11b72...ae6560976d7a8d20ae18bed96e33ae6a2fc5c754 * xamarin/java.interop@ae656097: [Java.Interop] restore `IL2035` suppression (xamarin/java.interop#1195) * xamarin/java.interop@c6e38933: [Java.Interop] use `Type.GetType()` to find `MarshalMemberBuilder` (xamarin/java.interop#1193) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- external/Java.Interop | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/Java.Interop b/external/Java.Interop index 7d1e7057cf4..ae6560976d7 160000 --- a/external/Java.Interop +++ b/external/Java.Interop @@ -1 +1 @@ -Subproject commit 7d1e7057cf4b0adcf65e7064186326dafce11b72 +Subproject commit ae6560976d7a8d20ae18bed96e33ae6a2fc5c754 From 7680e6ef240d0e9b567cae5cf440c41ac742b92e Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Wed, 21 Feb 2024 10:08:31 -0800 Subject: [PATCH 06/18] Bump to xamarin/monodroid@cb01503327 (#8742) Changes: https://github.com/xamarin/monodroid/compare/848d1277b76a599d8a280d58ec06e95477b4a7e5...cb01503327f7723ec138ec4cc051610fecee1bf7 * xamarin/monodroid@cb0150332: Bump external/xamarin-android from `0c0f1fe` to `7368487` * xamarin/monodroid@52e7c4c09: Bump external/android-sdk-installer from `4127490` to `13b2fb3` * xamarin/monodroid@5003ccb4d: Bump external/android-sdk-installer from `9e143d7` to `4127490` * xamarin/monodroid@d7ff10afb: Bump tools/msbuild/external/androidtools from `15350e7` to `8aeb717` * xamarin/monodroid@ae8c39d59: Bump external/xamarin-android from `80ee320` to `0c0f1fe` * xamarin/monodroid@d76e18694: [tools/msbuild] introduce $(_AndroidAllowJavaDebugging) feature switch --- .external | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.external b/.external index 8d6cb0be54a..8d72a200199 100644 --- a/.external +++ b/.external @@ -1 +1 @@ -xamarin/monodroid:main@848d1277b76a599d8a280d58ec06e95477b4a7e5 +xamarin/monodroid:main@cb01503327f7723ec138ec4cc051610fecee1bf7 From e199d62210bfb666595d95ca60579c5c766be1d6 Mon Sep 17 00:00:00 2001 From: Jonathan Pobst Date: Wed, 21 Feb 2024 08:55:08 -1000 Subject: [PATCH 07/18] Bump to xamarin/Java.Interop/main@c825dcad (#8701) Changes: https://github.com/xamarin/java.interop/compare/ae6560976d7a8d20ae18bed96e33ae6a2fc5c754...c825dcad8e7fe2e1ba5846a592a02f6a578db991 * xamarin/java.interop@c825dcad: [Java.Interop.Tools.JavaCallableWrappers] Refactor (xamarin/java.interop#1174) `Java.Interop.Tools.JavaCallableWrappers.dll` was refactored and now has a significant API changes. Bump to xamarin/Java.Interop@c825dcad and update callsites accordingly. --- external/Java.Interop | 2 +- .../Tasks/GenerateJavaStubs.cs | 25 +++++++++++++------ 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/external/Java.Interop b/external/Java.Interop index ae6560976d7..c825dcad8e7 160000 --- a/external/Java.Interop +++ b/external/Java.Interop @@ -1 +1 @@ -Subproject commit ae6560976d7a8d20ae18bed96e33ae6a2fc5c754 +Subproject commit c825dcad8e7fe2e1ba5846a592a02f6a578db991 diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs index 6a0444e872b..453885d1758 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/GenerateJavaStubs.cs @@ -20,6 +20,7 @@ using Xamarin.Android.Tools; using Microsoft.Android.Build.Tasks; +using Java.Interop.Tools.JavaCallableWrappers.Adapters; namespace Xamarin.Android.Tasks { @@ -463,6 +464,16 @@ bool CreateJavaSources (IEnumerable newJavaTypes, TypeDefinitionCache bool hasExportReference = ResolvedAssemblies.Any (assembly => Path.GetFileName (assembly.ItemSpec) == "Mono.Android.Export.dll"); bool generateOnCreateOverrides = int.Parse (AndroidSdkPlatform) <= 10; + var reader_options = new CallableWrapperReaderOptions { + DefaultApplicationJavaClass = ApplicationJavaClass, + DefaultGenerateOnCreateOverrides = generateOnCreateOverrides, + DefaultMonoRuntimeInitialization = monoInit, + MethodClassifier = classifier, + }; + var writer_options = new CallableWrapperWriterOptions { + CodeGenerationTarget = JavaPeerStyle.XAJavaInterop1 + }; + bool ok = true; foreach (JavaType jt in newJavaTypes) { TypeDefinition t = jt.Type; // JCW generator doesn't care about ABI-specific types or token ids @@ -473,23 +484,21 @@ bool CreateJavaSources (IEnumerable newJavaTypes, TypeDefinitionCache using (var writer = MemoryStreamPool.Shared.CreateStreamWriter ()) { try { - var jti = new JavaCallableWrapperGenerator (t, Log.LogWarning, cache, classifier) { - GenerateOnCreateOverrides = generateOnCreateOverrides, - ApplicationJavaClass = ApplicationJavaClass, - MonoRuntimeInitialization = monoInit, - }; + var jcw_type = CecilImporter.CreateType (t, cache, reader_options); + + jcw_type.Generate (writer, writer_options); - jti.Generate (writer); if (useMarshalMethods) { if (classifier.FoundDynamicallyRegisteredMethods (t)) { Log.LogWarning ($"Type '{t.GetAssemblyQualifiedName (cache)}' will register some of its Java override methods dynamically. This may adversely affect runtime performance. See preceding warnings for names of dynamically registered methods."); } } + writer.Flush (); - var path = jti.GetDestinationPath (outputPath); + var path = jcw_type.GetDestinationPath (outputPath); Files.CopyIfStreamChanged (writer.BaseStream, path); - if (jti.HasExport && !hasExportReference) + if (jcw_type.HasExport && !hasExportReference) Diagnostic.Error (4210, Properties.Resources.XA4210); } catch (XamarinAndroidException xae) { ok = false; From d21f6dbf40953fe7b7e3dbf2104e671b83ce2c25 Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Wed, 21 Feb 2024 14:41:12 -0600 Subject: [PATCH 08/18] [xamarin-android-tools] import $(LibZipSharpVersion) value (#8738) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes: https://github.com/dotnet/maui/issues/20752 Context: d1ba2ccdd1377677a6dd24dc13c7a23fd1b8b930 Context: https://github.com/xamarin/xamarin-android-tools/commit/34e98e2b65917d105169f868b5648f67e68b6784 Context: https://github.com/xamarin/xamarin-android/pull/8746 `dotnet build` (.NET Core) ignores assembly versions. `msbuild.exe` (.NET Framework) does not. Enter d1ba2ccd, which updates the `libZipSharp.dll` assembly version from 3.0.0.0 to 3.1.1.0. However, this change only impacted `Xamarin.Android.Build.Tasks.dll`: % monodis --assemblyref bin/Release/lib/packs/Microsoft.Android.Sdk.Darwin/34.99.0/tools/Xamarin.Android.Build.Tasks.dll … 11: Version=3.1.1.0 Name=libZipSharp Flags=0x00000000 It did *not* impact `Microsoft.Android.Build.BaseTasks.dll`, as it is built by `external/xamarin-android-tools` and was using a `@(PackageReference)` for libZipSharp 3.0.0: % monodis --assemblyref bin/Release/lib/packs/Microsoft.Android.Sdk.Darwin/34.99.0/tools/Microsoft.Android.Build.BaseTasks.dll … 5: Version=3.0.0.0 Name=libZipSharp Flags=0x00000000 The resulting NuGet package only contains *one* `libZipSharp.dll`, the 3.1.1.0 version (what `Xamarin.Android.Build.Tasks.dll` refs). On PR builds and CI, everything was fine, because all the tests we have use `dotnet build`, and .NET Core ignores assembly versions. However, if you use *`msbuild.exe`* to invoke the tasks within `Microsoft.Android.Build.BaseTasks.dll`, things fail: XARLP7028 System.IO.FileNotFoundException: Could not load file or assembly 'libZipSharp, Version=3.0.0.0, Culture=neutral, PublicKeyToken=276db85bc4e20efc' or one of its dependencies. The system cannot find the file specified.File name: 'libZipSharp, Version=3.0.0.0, Culture=neutral, PublicKeyToken=276db85bc4e20efc' at Xamarin.Android.Tasks.ResolveLibraryProjectImports.Extract(IDictionary`2 jars, ICollection`1 resolvedResourceDirectories, ICollection`1 resolvedAssetDirectories, ICollection`1 resolvedEnvironments, ICollection`1 proguardConfigFiles) at Xamarin.Android.Tasks.ResolveLibraryProjectImports.RunTask() at Microsoft.Android.Build.Tasks.AndroidTask.Execute() in /Users/runner/work/1/s/xamarin-android/external/xamarin-android-tools/src/Microsoft.Android.Build.BaseTasks/AndroidTask.cs:line 25 Fix this by adding an `external/xamarin-android-tools.override.props` file (xamarin/xamarin-android-tools@34e98e2b) which imports `Directory.Build.props`, which causes `$(LibZipSharpVersion)` to be set so that the xamarin-android-tools build uses values provided by xamarin-android. TODO: xamarin/xamarin-android#8746 adds a unit test for the "build with `msbuild.exe`" scenario. --- external/xamarin-android-tools.override.props | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 external/xamarin-android-tools.override.props diff --git a/external/xamarin-android-tools.override.props b/external/xamarin-android-tools.override.props new file mode 100644 index 00000000000..f11b5fdc4b9 --- /dev/null +++ b/external/xamarin-android-tools.override.props @@ -0,0 +1,4 @@ + + + + From 7559d12bdfb717d756795bb59d928d160e5eb6fb Mon Sep 17 00:00:00 2001 From: Marek Habersack Date: Thu, 22 Feb 2024 14:06:40 +0000 Subject: [PATCH 09/18] Add a property required by #8478 (#8749) * Add a property required by #8478 Context: https://github.com/xamarin/xamarin-android/pull/8478 Context: https://github.com/xamarin/monodroid/pull/1393 Merging this no-op change will make it easier to orchestrate merging of the two PRs above. It allows us to merge the `monodroid` one before merging #8478 and prevents breaking of any other builds or PRs in between merging of the two PRs. * Don't require it yet * Dummy change to kick-off the build * Undo the former change --- src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs index 0a15a2b8a26..c99ce0db5c5 100644 --- a/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs +++ b/src/Xamarin.Android.Build.Tasks/Tasks/BuildApk.cs @@ -33,6 +33,8 @@ public class BuildApk : AndroidTask [Required] public string ApkOutputPath { get; set; } + public string AppSharedLibrariesDir { get; set; } + [Required] public ITaskItem[] ResolvedUserAssemblies { get; set; } From 87d8914eef013eda16836ddc1532f13796bb91d2 Mon Sep 17 00:00:00 2001 From: Peter Collins Date: Thu, 22 Feb 2024 12:13:38 -0800 Subject: [PATCH 10/18] [ci] Always run the MAUI test job (#8750) A new $(RunMAUITestJob) variable has been added to control execution of the MAUI integration test job. The value of this variable defaults to true and it provides an easy way to disable the job on servicing/older branches if needed. --- build-tools/automation/azure-pipelines.yaml | 2 +- build-tools/automation/yaml-templates/variables.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/build-tools/automation/azure-pipelines.yaml b/build-tools/automation/azure-pipelines.yaml index 7a77f691e7c..611da9535ef 100644 --- a/build-tools/automation/azure-pipelines.yaml +++ b/build-tools/automation/azure-pipelines.yaml @@ -317,7 +317,7 @@ stages: - stage: maui_tests displayName: MAUI Tests dependsOn: mac_build - condition: and(eq(dependencies.mac_build.result, 'Succeeded'), eq(variables['System.PullRequest.TargetBranch'], 'main')) + condition: and(eq(dependencies.mac_build.result, 'Succeeded'), eq(variables['RunMAUITestJob'], 'true')) jobs: # Check - "Xamarin.Android (MAUI Tests MAUI Integration)" - job: maui_tests_integration diff --git a/build-tools/automation/yaml-templates/variables.yaml b/build-tools/automation/yaml-templates/variables.yaml index 07b09b01c8a..13a1f774a18 100644 --- a/build-tools/automation/yaml-templates/variables.yaml +++ b/build-tools/automation/yaml-templates/variables.yaml @@ -52,3 +52,5 @@ variables: value: 34 - name: ExcludedNightlyNUnitCategories value: 'cat != SystemApplication & cat != TimeZoneInfo & cat != Localization' +- name: RunMAUITestJob + value: true From 0665f449948c43675ec8706a5a2807b5d3f599f7 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 23 Feb 2024 08:16:48 -0600 Subject: [PATCH 11/18] Bump to dotnet/installer/main@0a73f814e1 9.0.100-preview.2.24122.3 (#8716) Changes: https://github.com/dotnet/installer/compare/fb7b9a4b9e...0a73f814e1 Changes: https://github.com/dotnet/runtime/compare/d40c654c27...dcc66a7ca2 Changes: https://github.com/dotnet/emsdk/compare/687be2a32a...258b51a8e5 Changes: https://github.com/dotnet/cecil/compare/b8c2293cd1...61250b0ed4 Updates: * Microsoft.Dotnet.Sdk.Internal: from 9.0.100-preview.2.24106.6 to 9.0.100-preview.2.24122.3 * Microsoft.NETCore.App.Ref: from 9.0.0-preview.2.24080.1 to 9.0.0-preview.2.24122.2 * Microsoft.NET.Workload.Emscripten.Current.Manifest-9.0.100.Transport: from 9.0.0-preview.2.24076.1 to 9.0.0-preview.2.24120.1 * Microsoft.NET.ILLink.Tasks: from 9.0.0-preview.2.24080.1 to 9.0.0-preview.2.24122.2 * Microsoft.DotNet.Cecil: from 0.11.4-alpha.24065.1 to 0.11.4-alpha.24119.1 Other changes: * Move *ahead* to newest dotnet/runtime build. Ran `darc update-dependencies --id 213925`, using build ID from: https://maestro-prod.westus2.cloudapp.azure.com/3883/https:%2F%2Fgithub.com%2Fdotnet%2Fruntime/latest/graph This picks up a fix for the AOT compiler: https://github.com/dotnet/runtime/commit/8819b222b3be41736efcf42ddace87a60d09d29e * Fix assertions in `BuildAotApplicationWithSpecialCharactersInProject` test cases There are fixes in the AOT compiler to support special characters. Co-authored-by: Jonathan Peppers --- eng/Version.Details.xml | 20 ++++++------- eng/Versions.props | 10 +++---- .../Xamarin.Android.Build.Tests/AotTests.cs | 30 +++++++------------ 3 files changed, 26 insertions(+), 34 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 1a4130af0be..ace9f2b5381 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,24 +1,24 @@ - + https://github.com/dotnet/installer - fb7b9a4b9e578fa8c9f5fb67e22daf4b0d22668e + 0a73f814e19c9a239371cb732c9d1257e10fb8a2 - + https://github.com/dotnet/runtime - d40c654c274fe4f4afe66328f0599130f3eb2ea6 + dcc66a7ca25696a2326f296c3d8d3ac5a13f0524 - + https://github.com/dotnet/runtime - d40c654c274fe4f4afe66328f0599130f3eb2ea6 + dcc66a7ca25696a2326f296c3d8d3ac5a13f0524 - + https://github.com/dotnet/emsdk - 687be2a32a302aaade82380c0eaafa5af85fb4da + 258b51a8e5f5bea07766ab99d2aaa75582d1ceb9 - + https://github.com/dotnet/cecil - b8c2293cd1cbd9d0fe6f32d7b5befbd526b5a175 + 61250b0ed403b3f9b69a33f7d8f66f311338d6a1 diff --git a/eng/Versions.props b/eng/Versions.props index ea4799ef67c..c096a719b25 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,15 +1,15 @@ - 9.0.100-preview.2.24106.6 - 9.0.0-preview.2.24080.1 - 9.0.0-preview.2.24080.1 + 9.0.100-preview.2.24122.3 + 9.0.0-preview.2.24122.2 + 9.0.0-preview.2.24122.2 7.0.0-beta.22103.1 7.0.0-beta.22103.1 - 9.0.0-preview.2.24076.1 + 9.0.0-preview.2.24120.1 $(MicrosoftNETWorkloadEmscriptenCurrentManifest90100TransportVersion) 7.0.100-rc.1.22410.7 - 0.11.4-alpha.24065.1 + 0.11.4-alpha.24119.1 $(MicrosoftNETCoreAppRefPackageVersion) diff --git a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs index 0b89a082870..e1884c1c5f3 100644 --- a/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs +++ b/src/Xamarin.Android.Build.Tasks/Tests/Xamarin.Android.Build.Tests/AotTests.cs @@ -111,19 +111,17 @@ public void BuildBasicApplicationReleaseProfiledAotWithoutDefaultProfile () } [Test] - [TestCase ("テスト", false, false, true)] - [TestCase ("テスト", true, true, false)] - [TestCase ("テスト", true, false, true)] - [TestCase ("随机生成器", false, false, true)] - [TestCase ("随机生成器", true, true, false)] - [TestCase ("随机生成器", true, false, true)] - [TestCase ("中国", false, false, true)] - [TestCase ("中国", true, true, false)] - [TestCase ("中国", true, false, true)] - public void BuildAotApplicationWithSpecialCharactersInProject (string testName, bool isRelease, bool aot, bool expectedResult) + [TestCase ("テスト", false, false)] + [TestCase ("テスト", true, true)] + [TestCase ("テスト", true, false)] + [TestCase ("随机生成器", false, false)] + [TestCase ("随机生成器", true, true)] + [TestCase ("随机生成器", true, false)] + [TestCase ("中国", false, false)] + [TestCase ("中国", true, true)] + [TestCase ("中国", true, false)] + public void BuildAotApplicationWithSpecialCharactersInProject (string testName, bool isRelease, bool aot) { - if (!IsWindows) - expectedResult = true; var rootPath = Path.Combine (Root, "temp", TestName); var proj = new XamarinAndroidApplicationProject () { ProjectName = testName, @@ -132,13 +130,7 @@ public void BuildAotApplicationWithSpecialCharactersInProject (string testName, }; proj.SetAndroidSupportedAbis ("armeabi-v7a", "arm64-v8a", "x86", "x86_64"); using (var builder = CreateApkBuilder (Path.Combine (rootPath, proj.ProjectName))){ - builder.ThrowOnBuildFailure = false; - Assert.AreEqual (expectedResult, builder.Build (proj), "Build should have succeeded."); - if (!expectedResult) { - var aotFailed = builder.LastBuildOutput.ContainsText ("Precompiling failed"); - var aapt2Failed = builder.LastBuildOutput.ContainsText ("APT2265"); - Assert.IsTrue (aotFailed || aapt2Failed, "Error APT2265 or an AOT error should have been raised."); - } + Assert.IsTrue (builder.Build (proj), "Build should have succeeded."); } } From a82ac4e5608b9745916db320f4ff36921d4d6dee Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Fri, 23 Feb 2024 13:24:18 -0600 Subject: [PATCH 12/18] [Mono.Android] fix a set of the "easiest" trimmer warnings (#8731) Context: https://github.com/xamarin/xamarin-android/issues/5652 Fixing the following trimmer warnings: ~~ TypeConverter ~~ Context: https://source.dot.net/#System.ComponentModel.TypeConverter/System/ComponentModel/TypeConverter.cs,226 src\Mono.Android\System.Drawing\SizeFConverter.cs(121,49): error IL2046: Base member 'System.ComponentModel.TypeConverter.GetProperties(ITypeDescriptorContext, Object, Attribute[])' with 'RequiresUnreferencedCodeAttribute' has a derived member 'System.Drawing.SizeFConverter.GetProperties(ITypeDescriptorContext, Object, Attribute[])' without 'RequiresUnreferencedCodeAttribute'. 'RequiresUnreferencedCodeAttribute' annotations must match across all interface implementations or overrides. Various `TypeConverter` implementations need to specify `[RequiresUnreferencedCode]` to match the base type. ~~ ColorValueMarshaler & IJavaObjectValueMarshaler ~~ Context: https://github.com/xamarin/java.interop/commit/7d1e7057cf4b0adcf65e7064186326dafce11b72 From the trimmer warnings solved in `Java.Interop.dll`, we need to bring these changes forward to any `*Marshaler` types. ~~ AndroidClientHandler ~~ 'this' argument does not satisfy 'DynamicallyAccessedMemberTypes.NonPublicFields' in call to 'System.Type.GetField(String, BindingFlags)'. The return value of method 'System.Collections.IEnumerator.Current.get' does not have matching annotations. The source value must declare at least the same requirements as those declared on the target location it is assigned to. This class had a loop that was not trimming safe: for (var type = GetType (); type != null; type = type.BaseType) { field = type.GetField (fieldName, BindingFlags.Instance | BindingFlags.NonPublic); if (field != null) break; } If we *look* at the actual base types of `AndroidClientHandler`, we can simplify this loop to something that *is* trimming safe: const BindingFlags flags = BindingFlags.Instance | BindingFlags.NonPublic; FieldInfo? field = typeof (HttpClientHandler).GetField (fieldName, flags) ?? typeof (HttpMessageHandler).GetField (fieldName, flags); There should be no need to iterate *beyond* `HttpMessageHandler`, the code is looking for this field: * https://github.com/dotnet/runtime/blob/135fec006e727a31763271984cd712f1659ccbd3/src/libraries/System.Net.Http/src/System/Net/Http/HttpClientHandler.AnyMobile.cs#L25 As `AndroidClientHandler` is `[Obsolete]` and unlikely to have its inheritance hierarchy changed, removing the loop is deemed safe. --- src/Mono.Android/Android.Graphics/Color.cs | 12 +++++++++++- .../Android.Runtime/IJavaObjectValueMarshaler.cs | 11 ++++++++++- src/Mono.Android/System.Drawing/PointConverter.cs | 2 ++ .../System.Drawing/RectangleConverter.cs | 2 ++ src/Mono.Android/System.Drawing/SizeConverter.cs | 2 ++ src/Mono.Android/System.Drawing/SizeFConverter.cs | 2 ++ .../Xamarin.Android.Net/AndroidClientHandler.cs | 11 +++-------- 7 files changed, 32 insertions(+), 10 deletions(-) diff --git a/src/Mono.Android/Android.Graphics/Color.cs b/src/Mono.Android/Android.Graphics/Color.cs index bf57968663c..97a46fd5b11 100644 --- a/src/Mono.Android/Android.Graphics/Color.cs +++ b/src/Mono.Android/Android.Graphics/Color.cs @@ -395,11 +395,18 @@ public static void RGBToHSV (int red, int green, int blue, float[] hsv) public class ColorValueMarshaler : JniValueMarshaler { + const DynamicallyAccessedMemberTypes ConstructorsAndInterfaces = DynamicallyAccessedMemberTypes.PublicConstructors | DynamicallyAccessedMemberTypes.NonPublicConstructors | DynamicallyAccessedMemberTypes.Interfaces; + const string ExpressionRequiresUnreferencedCode = "System.Linq.Expression usage may trim away required code."; + public override Type MarshalType { get { return typeof (int); } } - public override Color CreateGenericValue (ref JniObjectReference reference, JniObjectReferenceOptions options, Type targetType) + public override Color CreateGenericValue ( + ref JniObjectReference reference, + JniObjectReferenceOptions options, + [DynamicallyAccessedMembers (ConstructorsAndInterfaces)] + Type targetType) { throw new NotImplementedException (); } @@ -414,6 +421,7 @@ public override void DestroyGenericArgumentState (Color value, ref JniValueMarsh throw new NotImplementedException (); } + [RequiresUnreferencedCode (ExpressionRequiresUnreferencedCode)] public override Expression CreateParameterToManagedExpression (JniValueMarshalerContext context, ParameterExpression sourceValue, ParameterAttributes synchronize, Type targetType) { var c = typeof (Color).GetConstructor (new[]{typeof (int)})!; @@ -424,6 +432,7 @@ public override Expression CreateParameterToManagedExpression (JniValueMarshaler return v; } + [RequiresUnreferencedCode (ExpressionRequiresUnreferencedCode)] public override Expression CreateParameterFromManagedExpression (JniValueMarshalerContext context, ParameterExpression sourceValue, ParameterAttributes synchronize) { var r = Expression.Variable (MarshalType, sourceValue.Name + "_p"); @@ -433,6 +442,7 @@ public override Expression CreateParameterFromManagedExpression (JniValueMarshal return r; } + [RequiresUnreferencedCode (ExpressionRequiresUnreferencedCode)] public override Expression CreateReturnValueFromManagedExpression (JniValueMarshalerContext context, ParameterExpression sourceValue) { return CreateParameterFromManagedExpression (context, sourceValue, 0); diff --git a/src/Mono.Android/Android.Runtime/IJavaObjectValueMarshaler.cs b/src/Mono.Android/Android.Runtime/IJavaObjectValueMarshaler.cs index f5cab61ec1d..89436ab2b08 100644 --- a/src/Mono.Android/Android.Runtime/IJavaObjectValueMarshaler.cs +++ b/src/Mono.Android/Android.Runtime/IJavaObjectValueMarshaler.cs @@ -10,9 +10,16 @@ namespace Android.Runtime { sealed class IJavaObjectValueMarshaler : JniValueMarshaler { + const DynamicallyAccessedMemberTypes ConstructorsAndInterfaces = DynamicallyAccessedMemberTypes.PublicConstructors | DynamicallyAccessedMemberTypes.NonPublicConstructors | DynamicallyAccessedMemberTypes.Interfaces; + const string ExpressionRequiresUnreferencedCode = "System.Linq.Expression usage may trim away required code."; + internal static IJavaObjectValueMarshaler Instance = new IJavaObjectValueMarshaler (); - public override IJavaObject CreateGenericValue (ref JniObjectReference reference, JniObjectReferenceOptions options, Type? targetType) + public override IJavaObject CreateGenericValue ( + ref JniObjectReference reference, + JniObjectReferenceOptions options, + [DynamicallyAccessedMembers (ConstructorsAndInterfaces)] + Type? targetType) { throw new NotImplementedException (); } @@ -27,6 +34,7 @@ public override void DestroyGenericArgumentState ([AllowNull]IJavaObject value, throw new NotImplementedException (); } + [RequiresUnreferencedCode (ExpressionRequiresUnreferencedCode)] public override Expression CreateReturnValueFromManagedExpression (JniValueMarshalerContext context, ParameterExpression sourceValue) { return Expression.Call ( @@ -36,6 +44,7 @@ public override Expression CreateReturnValueFromManagedExpression (JniValueMarsh sourceValue); } + [RequiresUnreferencedCode (ExpressionRequiresUnreferencedCode)] public override Expression CreateParameterToManagedExpression (JniValueMarshalerContext context, ParameterExpression sourceValue, ParameterAttributes synchronize, Type? targetType) { var r = Expression.Variable (targetType, sourceValue.Name + "_val"); diff --git a/src/Mono.Android/System.Drawing/PointConverter.cs b/src/Mono.Android/System.Drawing/PointConverter.cs index 0255e55c87e..98a791cf951 100644 --- a/src/Mono.Android/System.Drawing/PointConverter.cs +++ b/src/Mono.Android/System.Drawing/PointConverter.cs @@ -30,6 +30,7 @@ using System.Collections; using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.ComponentModel.Design.Serialization; using System.Runtime.CompilerServices; @@ -133,6 +134,7 @@ public override bool GetCreateInstanceSupported (ITypeDescriptorContext context) return true; } + [RequiresUnreferencedCode ("The Type of value cannot be statically discovered.")] public override PropertyDescriptorCollection? GetProperties ( ITypeDescriptorContext context, object value, Attribute[] attributes) diff --git a/src/Mono.Android/System.Drawing/RectangleConverter.cs b/src/Mono.Android/System.Drawing/RectangleConverter.cs index 364784f495f..053f07882af 100644 --- a/src/Mono.Android/System.Drawing/RectangleConverter.cs +++ b/src/Mono.Android/System.Drawing/RectangleConverter.cs @@ -31,6 +31,7 @@ using System.ComponentModel; using System.Collections; +using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Text; using System.ComponentModel.Design.Serialization; @@ -147,6 +148,7 @@ public override bool GetCreateInstanceSupported (ITypeDescriptorContext context) return true; } + [RequiresUnreferencedCode ("The Type of value cannot be statically discovered.")] public override PropertyDescriptorCollection? GetProperties ( ITypeDescriptorContext context, object value, Attribute[] attributes) diff --git a/src/Mono.Android/System.Drawing/SizeConverter.cs b/src/Mono.Android/System.Drawing/SizeConverter.cs index 53fe055733c..f60b57442b1 100644 --- a/src/Mono.Android/System.Drawing/SizeConverter.cs +++ b/src/Mono.Android/System.Drawing/SizeConverter.cs @@ -31,6 +31,7 @@ using System.Collections; using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.ComponentModel.Design.Serialization; using System.Reflection; @@ -135,6 +136,7 @@ public override bool GetCreateInstanceSupported (ITypeDescriptorContext context) return true; } + [RequiresUnreferencedCode ("The Type of value cannot be statically discovered.")] public override PropertyDescriptorCollection? GetProperties ( ITypeDescriptorContext context, object value, Attribute[] attributes) diff --git a/src/Mono.Android/System.Drawing/SizeFConverter.cs b/src/Mono.Android/System.Drawing/SizeFConverter.cs index 8933839fba2..fb2e81fce0f 100644 --- a/src/Mono.Android/System.Drawing/SizeFConverter.cs +++ b/src/Mono.Android/System.Drawing/SizeFConverter.cs @@ -32,6 +32,7 @@ using System; using System.Collections; using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.ComponentModel.Design.Serialization; using System.Reflection; @@ -118,6 +119,7 @@ public override bool GetCreateInstanceSupported (ITypeDescriptorContext context) return true; } + [RequiresUnreferencedCode ("The Type of value cannot be statically discovered.")] public override PropertyDescriptorCollection? GetProperties (ITypeDescriptorContext context, object value, Attribute[] attributes) { if (value is SizeF) diff --git a/src/Mono.Android/Xamarin.Android.Net/AndroidClientHandler.cs b/src/Mono.Android/Xamarin.Android.Net/AndroidClientHandler.cs index 83e858e1a62..f12317c1ead 100644 --- a/src/Mono.Android/Xamarin.Android.Net/AndroidClientHandler.cs +++ b/src/Mono.Android/Xamarin.Android.Net/AndroidClientHandler.cs @@ -313,14 +313,9 @@ protected virtual Task SetupRequest (HttpRequestMessage request, HttpURLConnecti object? GetUnderlyingHandler () { var fieldName = "_nativeHandler"; - FieldInfo? field = null; - - for (var type = GetType (); type != null; type = type.BaseType) { - field = type.GetField (fieldName, BindingFlags.Instance | BindingFlags.NonPublic); - if (field != null) - break; - } - + const BindingFlags flags = BindingFlags.Instance | BindingFlags.NonPublic; + FieldInfo? field = typeof (HttpClientHandler).GetField (fieldName, flags) ?? + typeof (HttpMessageHandler).GetField (fieldName, flags); if (field == null) { throw new InvalidOperationException ($"Field '{fieldName}' is missing from type '{GetType ()}'."); } From 8c7cf91fc17dfac2e073ae464eddc137b9cd8390 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Feb 2024 08:52:53 -0600 Subject: [PATCH 13/18] Bump com.android.tools:r8 from 8.2.42 to 8.2.47 (#8761) Context: https://r8.googlesource.com/r8/+/refs/tags/8.2.44 Context: https://r8.googlesource.com/r8/+/62baf24ba2b71f4e5bc04c7f01930bc5d47b02be Context: https://maven.google.com/web/index.html?q=r8#com.android.tools:r8:8.2.47 Note that I was not able to find an *exact* matching tag for this release, but found the commit. --- src/r8/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/r8/build.gradle b/src/r8/build.gradle index 58ce928562d..e95e9ce1a52 100644 --- a/src/r8/build.gradle +++ b/src/r8/build.gradle @@ -15,7 +15,7 @@ repositories { } dependencies { - implementation 'com.android.tools:r8:8.2.42' + implementation 'com.android.tools:r8:8.2.47' } jar { From efcd007b0ce2e5a69f684dcf747f90d1a6531ef6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Rozs=C3=ADval?= Date: Tue, 27 Feb 2024 11:33:34 +0100 Subject: [PATCH 14/18] [Mono.Android] Do not dispose request content stream in AndroidMessageHandler (#8764) Fixes: https://github.com/xamarin/xamarin-android/issues/2901 Fixes: https://github.com/xamarin/xamarin-android/issues/4476 Fixes: https://github.com/xamarin/xamarin-android/issues/7086 Neither the `SocketsHttpHandler` nor the `iOS/macOS` `NSUrlSessionHandler` dispose the content stream, let's follow suit. --- .../AndroidMessageHandler.cs | 47 +++++++++---------- .../AndroidMessageHandlerTests.cs | 25 ++++++++++ 2 files changed, 48 insertions(+), 24 deletions(-) diff --git a/src/Mono.Android/Xamarin.Android.Net/AndroidMessageHandler.cs b/src/Mono.Android/Xamarin.Android.Net/AndroidMessageHandler.cs index 44883640487..26645e6fdcb 100644 --- a/src/Mono.Android/Xamarin.Android.Net/AndroidMessageHandler.cs +++ b/src/Mono.Android/Xamarin.Android.Net/AndroidMessageHandler.cs @@ -536,30 +536,29 @@ protected virtual async Task WriteRequestContentToOutput (HttpRequestMessage req if (request.Content is null) return; - using (var stream = await request.Content.ReadAsStreamAsync ().ConfigureAwait (false)) { - await stream.CopyToAsync(httpConnection.OutputStream!, 4096, cancellationToken).ConfigureAwait(false); - - // - // Rewind the stream to beginning in case the HttpContent implementation - // will be accessed again (e.g. after redirect) and it keeps its stream - // open behind the scenes instead of recreating it on the next call to - // ReadAsStreamAsync. If we don't rewind it, the ReadAsStreamAsync - // call above will throw an exception as we'd be attempting to read an - // already "closed" stream (that is one whose Position is set to its - // end). - // - // This is not a perfect solution since the HttpContent may do weird - // things in its implementation, but it's better than copying the - // content into a buffer since we have no way of knowing how the data is - // read or generated and also we don't want to keep potentially large - // amounts of data in memory (which would happen if we read the content - // into a byte[] buffer and kept it cached for re-use on redirect). - // - // See https://bugzilla.xamarin.com/show_bug.cgi?id=55477 - // - if (stream.CanSeek) - stream.Seek (0, SeekOrigin.Begin); - } + var stream = await request.Content.ReadAsStreamAsync ().ConfigureAwait (false); + await stream.CopyToAsync(httpConnection.OutputStream!, 4096, cancellationToken).ConfigureAwait(false); + + // + // Rewind the stream to beginning in case the HttpContent implementation + // will be accessed again (e.g. after redirect) and it keeps its stream + // open behind the scenes instead of recreating it on the next call to + // ReadAsStreamAsync. If we don't rewind it, the ReadAsStreamAsync + // call above will throw an exception as we'd be attempting to read an + // already "closed" stream (that is one whose Position is set to its + // end). + // + // This is not a perfect solution since the HttpContent may do weird + // things in its implementation, but it's better than copying the + // content into a buffer since we have no way of knowing how the data is + // read or generated and also we don't want to keep potentially large + // amounts of data in memory (which would happen if we read the content + // into a byte[] buffer and kept it cached for re-use on redirect). + // + // See https://bugzilla.xamarin.com/show_bug.cgi?id=55477 + // + if (stream.CanSeek) + stream.Seek (0, SeekOrigin.Begin); } internal Task WriteRequestContentToOutputInternal (HttpRequestMessage request, HttpURLConnection httpConnection, CancellationToken cancellationToken) diff --git a/tests/Mono.Android-Tests/Xamarin.Android.Net/AndroidMessageHandlerTests.cs b/tests/Mono.Android-Tests/Xamarin.Android.Net/AndroidMessageHandlerTests.cs index 96f66778ef3..b71ee2893da 100644 --- a/tests/Mono.Android-Tests/Xamarin.Android.Net/AndroidMessageHandlerTests.cs +++ b/tests/Mono.Android-Tests/Xamarin.Android.Net/AndroidMessageHandlerTests.cs @@ -98,6 +98,31 @@ async Task DoDecompression (string urlPath, string encoding, string jsonFi return true; } + [Test] + public async Task DoesNotDisposeContentStream() + { + using var listener = new HttpListener (); + listener.Prefixes.Add ("http://+:47663/"); + listener.Start (); + listener.BeginGetContext (ar => { + var ctx = listener.EndGetContext (ar); + ctx.Response.StatusCode = 204; + ctx.Response.ContentLength64 = 0; + ctx.Response.Close (); + }, null); + + var jsonContent = new StringContent ("hello"); + var request = new HttpRequestMessage (HttpMethod.Post, "http://localhost:47663/") { Content = jsonContent }; + + var response = await new HttpClient (new AndroidMessageHandler ()).SendAsync (request); + Assert.True (response.IsSuccessStatusCode); + + var contentValue = await jsonContent.ReadAsStringAsync (); + Assert.AreEqual ("hello", contentValue); + + listener.Close (); + } + [Test] public async Task ServerCertificateCustomValidationCallback_ApproveRequest () { From 37594cab8133d91ef25f2e3f9e35974ffd6717e4 Mon Sep 17 00:00:00 2001 From: Jonathan Peppers Date: Tue, 27 Feb 2024 08:39:47 -0600 Subject: [PATCH 15/18] $(AndroidPackVersionSuffix)=preview.3; net9 is 34.99.0.preview.3 (#8765) Context: https://github.com/xamarin/xamarin-android/tree/release/9.0.1xx-preview2 We branched for .NET 9 Preview 2 from 8c7cf91f into release/9.0.1xx-preview2; the main branch is now .NET 9 Preview 3. Update xamarin-android/main's version number to 34.99.0-preview.3. --- Directory.Build.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Directory.Build.props b/Directory.Build.props index eb9b8dfd3c9..3f855aae763 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -36,7 +36,7 @@ * Bump first digit of the patch version for feature releases (and reset the first two digits to 0) --> 34.99.0 - preview.2 + preview.3 From 43a397fa93b7d2ad4702c8a0b048844aefaa5fec Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Feb 2024 08:44:10 -0600 Subject: [PATCH 16/18] Bump to xamarin/java.interop/main@14a9470 (#8766) Changes: https://github.com/xamarin/java.interop/compare/c825dcad8e7fe2e1ba5846a592a02f6a578db991...14a9470176b314fe521479fc75f6d9dd919abb56 * xamarin/java.interop/main@14a9470 [Java.Interop.Tools.TypeNameMappings] introduce project for net8.0 * xamarin/java.interop/main@bd793f1 [Java.Interop.Tools.TypeNameMappings] fix ToJniName() * xamarin/java.interop/main@67c079c [Java.Interop.Tools.JavaCallableWrappers] fix net8.0 targeting in XA * xamarin/java.interop/main@2197579 [Hello-NativeAOTFromJNI] Add NativeAOT sample * xamarin/java.interop/main@c8fcf70 Bump to xamarin/xamarin-android-tools@37d79c9 * xamarin/java.interop/main@56b7eeb [Java.Interop.Tools.TypeNameMappings] fix trimmer warnings --- external/Java.Interop | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/Java.Interop b/external/Java.Interop index c825dcad8e7..14a9470176b 160000 --- a/external/Java.Interop +++ b/external/Java.Interop @@ -1 +1 @@ -Subproject commit c825dcad8e7fe2e1ba5846a592a02f6a578db991 +Subproject commit 14a9470176b314fe521479fc75f6d9dd919abb56 From 2f4e01ec15102dd9cd922cbd833f6482d69512b5 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Tue, 27 Feb 2024 13:43:58 -0600 Subject: [PATCH 17/18] Bump to dotnet/installer/main@d070660282 9.0.100-preview.3.24126.2 (#8763) Changes: https://github.com/dotnet/installer/compare/0a73f814e1...d070660282 Changes: https://github.com/dotnet/runtime/compare/dcc66a7ca2...99b76018b6 Changes: https://github.com/dotnet/emsdk/compare/258b51a8e5...2d3f1fe480 Updates: * Microsoft.Dotnet.Sdk.Internal: from 9.0.100-preview.2.24122.3 to 9.0.100-preview.3.24126.2 * Microsoft.NETCore.App.Ref: from 9.0.0-preview.2.24122.2 to 9.0.0-preview.2.24123.1 * Microsoft.NET.Workload.Emscripten.Current.Manifest-9.0.100.Transport: from 9.0.0-preview.2.24120.1 to 9.0.0-preview.2.24121.1 * Microsoft.NET.ILLink.Tasks: from 9.0.0-preview.2.24122.2 to 9.0.0-preview.2.24123.1 Other changes: * [tests] ignore `IncrementalFastDeployment` for now Context: https://github.com/NuGet/Home/issues/13269 Co-authored-by: Jonathan Peppers --- eng/Version.Details.xml | 16 ++++++++-------- eng/Versions.props | 8 ++++---- .../Tests/InstallTests.cs | 1 + 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index ace9f2b5381..566089db986 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,20 +1,20 @@ - + https://github.com/dotnet/installer - 0a73f814e19c9a239371cb732c9d1257e10fb8a2 + d070660282eb5f78497310f77093638744112e03 - + https://github.com/dotnet/runtime - dcc66a7ca25696a2326f296c3d8d3ac5a13f0524 + 99b76018b6e4edc4ce185dd5f3c5697c6941d88e - + https://github.com/dotnet/runtime - dcc66a7ca25696a2326f296c3d8d3ac5a13f0524 + 99b76018b6e4edc4ce185dd5f3c5697c6941d88e - + https://github.com/dotnet/emsdk - 258b51a8e5f5bea07766ab99d2aaa75582d1ceb9 + 2d3f1fe4807a21879cedba9d3fde8cd329fb17f2 https://github.com/dotnet/cecil diff --git a/eng/Versions.props b/eng/Versions.props index c096a719b25..6bf62dc7325 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -1,12 +1,12 @@ - 9.0.100-preview.2.24122.3 - 9.0.0-preview.2.24122.2 - 9.0.0-preview.2.24122.2 + 9.0.100-preview.3.24126.2 + 9.0.0-preview.2.24123.1 + 9.0.0-preview.2.24123.1 7.0.0-beta.22103.1 7.0.0-beta.22103.1 - 9.0.0-preview.2.24120.1 + 9.0.0-preview.2.24121.1 $(MicrosoftNETWorkloadEmscriptenCurrentManifest90100TransportVersion) 7.0.100-rc.1.22410.7 0.11.4-alpha.24119.1 diff --git a/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs b/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs index f25d948b44c..f1021c2d15d 100644 --- a/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs +++ b/tests/MSBuildDeviceIntegration/Tests/InstallTests.cs @@ -495,6 +495,7 @@ public void LocalizedAssemblies_ShouldBeFastDeployed () [Test] public void IncrementalFastDeployment () { + Assert.Ignore ("https://github.com/NuGet/Home/issues/13269"); AssertCommercialBuild (); var class1src = new BuildItem.Source ("Class1.cs") { From 4a2179ce8831183294aaa8d18cbe2af67686ce47 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 27 Feb 2024 17:22:52 -0500 Subject: [PATCH 18/18] Bump to xamarin/xamarin-android-tools/main@37d79c9 (#8752) Changes: https://github.com/xamarin/xamarin-android-tools/compare/a698a33aa4ffcaac90b54caf5e77236d57b0cf9e...37d79c9dcdf738a181084b0b5890877128d75f1e * xamarin/xamarin-android-tools@37d79c9: Bump LibZipSharp to 3.1.1 (xamarin/xamarin-android-tools#228) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- external/xamarin-android-tools | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/xamarin-android-tools b/external/xamarin-android-tools index a698a33aa4f..37d79c9dcdf 160000 --- a/external/xamarin-android-tools +++ b/external/xamarin-android-tools @@ -1 +1 @@ -Subproject commit a698a33aa4ffcaac90b54caf5e77236d57b0cf9e +Subproject commit 37d79c9dcdf738a181084b0b5890877128d75f1e