Skip to content

Commit

Permalink
feat: port "add support for MSVC cross-compilation" from node
Browse files Browse the repository at this point in the history 
Original commit message:

    tools,gyp: add support for MSVC cross-compilation

    This change means that GYP can now generate two sets of projects: one
    exclusively for a host x64 machine and one containing a mix of x64 and
    Arm targets. The names of host targets are fixed up to end with
    _host.exe, and any actions involving them are fixed up. This allows
    compilation of Node on an x64 server for a Windows on Arm target.

Closes: nodejs#40
  • Loading branch information
@targos
targos committed May 23, 2020
1 parent 44e3f3e commit 6a162fd
Showing 1 changed file with 79 additions and 40 deletions.
119 changes: 79 additions & 40 deletions pylib/gyp/generator/msvs.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
# letters.
VALID_MSVS_GUID_CHARS = re.compile(r"^[A-F0-9\-]+$")

generator_supports_multiple_toolsets = gyp.common.CrossCompileRequested()

generator_default_variables = {
"DRIVER_PREFIX": "",
Expand All @@ -50,7 +51,7 @@
"STATIC_LIB_SUFFIX": ".lib",
"SHARED_LIB_SUFFIX": ".dll",
"INTERMEDIATE_DIR": "$(IntDir)",
"SHARED_INTERMEDIATE_DIR": "$(OutDir)obj/global_intermediate",
"SHARED_INTERMEDIATE_DIR": "$(OutDir)/obj/global_intermediate",
"OS": "win",
"PRODUCT_DIR": "$(OutDir)",
"LIB_DIR": "$(OutDir)lib",
Expand Down Expand Up @@ -305,7 +306,7 @@ def _ConfigTargetVersion(config_data):
return config_data.get("msvs_target_version", "Windows7")


def _ConfigPlatform(config_data):
def _ConfigPlatform(config_data, spec):
return config_data.get("msvs_configuration_platform", "Win32")


Expand All @@ -316,8 +317,8 @@ def _ConfigBaseName(config_name, platform_name):
return config_name


def _ConfigFullName(config_name, config_data):
platform_name = _ConfigPlatform(config_data)
def _ConfigFullName(config_name, config_data, spec):
platform_name = _ConfigPlatform(config_data, spec)
return "%s|%s" % (_ConfigBaseName(config_name, platform_name), platform_name)


Expand Down Expand Up @@ -1005,7 +1006,7 @@ def _GetMsbuildToolsetOfProject(proj_path, spec, version):
return toolset


def _GenerateProject(project, options, version, generator_flags):
def _GenerateProject(project, options, version, generator_flags, spec):
"""Generates a vcproj file.
Arguments:
Expand All @@ -1023,7 +1024,7 @@ def _GenerateProject(project, options, version, generator_flags):
return []

if version.UsesVcxproj():
return _GenerateMSBuildProject(project, options, version, generator_flags)
return _GenerateMSBuildProject(project, options, version, generator_flags, spec)
else:
return _GenerateMSVSProject(project, options, version, generator_flags)

Expand Down Expand Up @@ -1145,7 +1146,7 @@ def _GetUniquePlatforms(spec):
# Gather list of unique platforms.
platforms = OrderedSet()
for configuration in spec["configurations"]:
platforms.add(_ConfigPlatform(spec["configurations"][configuration]))
platforms.add(_ConfigPlatform(spec["configurations"][configuration], spec))
platforms = list(platforms)
return platforms

Expand Down Expand Up @@ -1903,6 +1904,8 @@ def _GatherSolutionFolders(sln_projects, project_objects, flat):
# Convert into a tree of dicts on path.
for p in sln_projects:
gyp_file, target = gyp.common.ParseQualifiedTarget(p)[0:2]
if p.endswith("#host"):
target += "_host"
gyp_dir = os.path.dirname(gyp_file)
path_dict = _GetPathDict(root, gyp_dir)
path_dict[target + ".vcproj"] = project_objects[p]
Expand All @@ -1921,9 +1924,10 @@ def _GetPathOfProject(qualified_target, spec, options, msvs_version):
default_config = _GetDefaultConfiguration(spec)
proj_filename = default_config.get("msvs_existing_vcproj")
if not proj_filename:
proj_filename = (
spec["target_name"] + options.suffix + msvs_version.ProjectExtension()
)
proj_filename = spec["target_name"]
if spec["toolset"] == "host":
proj_filename += "_host"
proj_filename = proj_filename + options.suffix + msvs_version.ProjectExtension()

build_file = gyp.common.BuildFile(qualified_target)
proj_path = os.path.join(os.path.dirname(build_file), proj_filename)
Expand All @@ -1942,12 +1946,14 @@ def _GetPlatformOverridesOfProject(spec):
# solution configurations for this target.
config_platform_overrides = {}
for config_name, c in spec["configurations"].items():
config_fullname = _ConfigFullName(config_name, c)
platform = c.get("msvs_target_platform", _ConfigPlatform(c))
config_fullname = _ConfigFullName(config_name, c, spec)
platform = c.get("msvs_target_platform", _ConfigPlatform(c, spec))
fixed_config_fullname = "%s|%s" % (
_ConfigBaseName(config_name, _ConfigPlatform(c)),
_ConfigBaseName(config_name, _ConfigPlatform(c, spec)),
platform,
)
if spec["toolset"] == "host" and generator_supports_multiple_toolsets:
fixed_config_fullname = "%s|x64" % (config_name,)
config_platform_overrides[config_fullname] = fixed_config_fullname
return config_platform_overrides

Expand All @@ -1968,21 +1974,19 @@ def _CreateProjectObjects(target_list, target_dicts, options, msvs_version):
projects = {}
for qualified_target in target_list:
spec = target_dicts[qualified_target]
if spec["toolset"] != "target":
raise GypError(
"Multiple toolsets not supported in msvs build (target %s)"
% qualified_target
)
proj_path, fixpath_prefix = _GetPathOfProject(
qualified_target, spec, options, msvs_version
)
guid = _GetGuidOfProject(proj_path, spec)
overrides = _GetPlatformOverridesOfProject(spec)
build_file = gyp.common.BuildFile(qualified_target)
# Create object for this project.
target_name = spec["target_name"]
if spec["toolset"] == "host":
target_name += "_host"
obj = MSVSNew.MSVSProject(
proj_path,
name=spec["target_name"],
name=target_name,
guid=guid,
spec=spec,
build_file=build_file,
Expand Down Expand Up @@ -2161,7 +2165,10 @@ def GenerateOutput(target_list, target_dicts, data, params):
for qualified_target in target_list:
spec = target_dicts[qualified_target]
for config_name, config in spec["configurations"].items():
configs.add(_ConfigFullName(config_name, config))
config_name = _ConfigFullName(config_name, config, spec)
configs.add(config_name)
if config_name == "Release|arm64":
configs.add("Release|x64")
configs = list(configs)

# Figure out all the projects that will be generated and their guids
Expand All @@ -2174,12 +2181,15 @@ def GenerateOutput(target_list, target_dicts, data, params):
for project in project_objects.values():
fixpath_prefix = project.fixpath_prefix
missing_sources.extend(
_GenerateProject(project, options, msvs_version, generator_flags)
_GenerateProject(project, options, msvs_version, generator_flags, spec)
)
fixpath_prefix = None

for build_file in data:
# Validate build_file extension
target_only_configs = configs
if generator_supports_multiple_toolsets:
target_only_configs = [i for i in configs if i.endswith("arm64")]
if not build_file.endswith(".gyp"):
continue
sln_path = os.path.splitext(build_file)[0] + options.suffix + ".sln"
Expand All @@ -2196,7 +2206,7 @@ def GenerateOutput(target_list, target_dicts, data, params):
sln = MSVSNew.MSVSSolution(
sln_path,
entries=root_entries,
variants=configs,
variants=target_only_configs,
websiteProperties=False,
version=msvs_version,
)
Expand Down Expand Up @@ -2930,22 +2940,24 @@ def _GenerateMSBuildRuleXmlFile(xml_path, msbuild_rules):
easy_xml.WriteXmlIfChanged(content, xml_path, pretty=True, win32=True)


def _GetConfigurationAndPlatform(name, settings):
def _GetConfigurationAndPlatform(name, settings, spec):
configuration = name.rsplit("_", 1)[0]
platform = settings.get("msvs_configuration_platform", "Win32")
if spec["toolset"] == "host" and platform == "arm64":
platform = "x64" # Host-only tools are always built for x64
return (configuration, platform)


def _GetConfigurationCondition(name, settings):
def _GetConfigurationCondition(name, settings, spec):
return r"'$(Configuration)|$(Platform)'=='%s|%s'" % _GetConfigurationAndPlatform(
name, settings
name, settings, spec
)


def _GetMSBuildProjectConfigurations(configurations):
def _GetMSBuildProjectConfigurations(configurations, spec):
group = ["ItemGroup", {"Label": "ProjectConfigurations"}]
for (name, settings) in sorted(configurations.items()):
configuration, platform = _GetConfigurationAndPlatform(name, settings)
configuration, platform = _GetConfigurationAndPlatform(name, settings, spec)
designation = "%s|%s" % (configuration, platform)
group.append(
[
Expand Down Expand Up @@ -3006,7 +3018,7 @@ def _GetMSBuildGlobalProperties(spec, version, guid, gyp_file_name):
platform_name = None
msvs_windows_sdk_version = None
for configuration in spec["configurations"].values():
platform_name = platform_name or _ConfigPlatform(configuration)
platform_name = platform_name or _ConfigPlatform(configuration, spec)
msvs_windows_sdk_version = (
msvs_windows_sdk_version
or _ConfigWindowsTargetPlatformVersion(configuration, version)
Expand All @@ -3033,7 +3045,7 @@ def _GetMSBuildConfigurationDetails(spec, build_file):
properties = {}
for name, settings in spec["configurations"].items():
msbuild_attributes = _GetMSBuildAttributes(spec, settings, build_file)
condition = _GetConfigurationCondition(name, settings)
condition = _GetConfigurationCondition(name, settings, spec)
character_set = msbuild_attributes.get("CharacterSet")
config_type = msbuild_attributes.get("ConfigurationType")
_AddConditionalProperty(properties, condition, "ConfigurationType", config_type)
Expand Down Expand Up @@ -3064,12 +3076,12 @@ def _GetMSBuildLocalProperties(msbuild_toolset):
return properties


def _GetMSBuildPropertySheets(configurations):
def _GetMSBuildPropertySheets(configurations, spec):
user_props = r"$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props"
additional_props = {}
props_specified = False
for name, settings in sorted(configurations.items()):
configuration = _GetConfigurationCondition(name, settings)
configuration = _GetConfigurationCondition(name, settings, spec)
if "msbuild_props" in settings:
additional_props[configuration] = _FixPaths(settings["msbuild_props"])
props_specified = True
Expand Down Expand Up @@ -3222,7 +3234,7 @@ def _GetMSBuildConfigurationGlobalProperties(spec, configurations, build_file):

properties = {}
for (name, configuration) in sorted(configurations.items()):
condition = _GetConfigurationCondition(name, configuration)
condition = _GetConfigurationCondition(name, configuration, spec)
attributes = _GetMSBuildAttributes(spec, configuration, build_file)
msbuild_settings = configuration["finalized_msbuild_settings"]
_AddConditionalProperty(
Expand Down Expand Up @@ -3345,7 +3357,7 @@ def _GetMSBuildToolSettingsSections(spec, configurations):
msbuild_settings = configuration["finalized_msbuild_settings"]
group = [
"ItemDefinitionGroup",
{"Condition": _GetConfigurationCondition(name, configuration)},
{"Condition": _GetConfigurationCondition(name, configuration, spec)},
]
for tool_name, tool_settings in sorted(msbuild_settings.items()):
# Skip the tool named '' which is a holder of global settings handled
Expand Down Expand Up @@ -3625,7 +3637,7 @@ def _AddSources2(

if precompiled_source == source:
condition = _GetConfigurationCondition(
config_name, configuration
config_name, configuration, spec
)
detail.append(
["PrecompiledHeader", {"Condition": condition}, "Create"]
Expand All @@ -3648,11 +3660,26 @@ def _AddSources2(
grouped_sources[group].append([element, {"Include": source}] + detail)


def _GetMSBuildProjectReferences(project):
def _GetMSBuildProjectReferences(project, spec):
current_configuration = spec["default_configuration"]
references = []
if project.dependencies:
group = ["ItemGroup"]
added_dependency_set = set()
for dependency in project.dependencies:
dependency_spec = dependency.spec
should_skip_dep = False
if project.spec["toolset"] == "target":
if dependency_spec["toolset"] == "host":
if dependency_spec["type"] == "static_library":
should_skip_dep = True
if dependency.name.startswith("run_"):
should_skip_dep = False
if should_skip_dep:
continue

canonical_name = dependency.name.replace("_host", "")
added_dependency_set.add(canonical_name)
guid = dependency.guid
project_dir = os.path.split(project.path)[0]
relative_path = gyp.common.RelativePath(dependency.path, project_dir)
Expand All @@ -3675,7 +3702,7 @@ def _GetMSBuildProjectReferences(project):
return references


def _GenerateMSBuildProject(project, options, version, generator_flags):
def _GenerateMSBuildProject(project, options, version, generator_flags, spec):
spec = project.spec
configurations = spec["configurations"]
project_dir, project_file_name = os.path.split(project.path)
Expand Down Expand Up @@ -3774,7 +3801,7 @@ def _GenerateMSBuildProject(project, options, version, generator_flags):
},
]

content += _GetMSBuildProjectConfigurations(configurations)
content += _GetMSBuildProjectConfigurations(configurations, spec)
content += _GetMSBuildGlobalProperties(
spec, version, project.guid, project_file_name
)
Expand All @@ -3786,10 +3813,10 @@ def _GenerateMSBuildProject(project, options, version, generator_flags):
content += _GetMSBuildLocalProperties(project.msbuild_toolset)
content += import_cpp_props_section
content += import_masm_props_section
if spec.get("msvs_enable_marmasm"):
if spec.get("msvs_enable_marmasm") or True:
content += import_marmasm_props_section
content += _GetMSBuildExtensions(props_files_of_rules)
content += _GetMSBuildPropertySheets(configurations)
content += _GetMSBuildPropertySheets(configurations, spec)
content += macro_section
content += _GetMSBuildConfigurationGlobalProperties(
spec, configurations, project.build_file
Expand All @@ -3805,7 +3832,7 @@ def _GenerateMSBuildProject(project, options, version, generator_flags):
sources_handled_by_action,
list_excluded,
)
content += _GetMSBuildProjectReferences(project)
content += _GetMSBuildProjectReferences(project, spec)
content += import_cpp_targets_section
content += import_masm_targets_section
if spec.get("msvs_enable_marmasm"):
Expand Down Expand Up @@ -3893,15 +3920,27 @@ def _GenerateActionsForMSBuild(spec, actions_to_add):
sources_handled_by_action = OrderedSet()
actions_spec = []
for primary_input, actions in actions_to_add.items():
if generator_supports_multiple_toolsets:
primary_input = primary_input.replace(".exe", "_host.exe")
inputs = OrderedSet()
outputs = OrderedSet()
descriptions = []
commands = []
for action in actions:

def fixup_host_exe(i):
if "$(OutDir)" in i:
i = i.replace(".exe", "_host.exe")
return i

if generator_supports_multiple_toolsets:
action["inputs"] = [fixup_host_exe(i) for i in action["inputs"]]
inputs.update(OrderedSet(action["inputs"]))
outputs.update(OrderedSet(action["outputs"]))
descriptions.append(action["description"])
cmd = action["command"]
if generator_supports_multiple_toolsets:
cmd = cmd.replace(".exe", "_host.exe")
# For most actions, add 'call' so that actions that invoke batch files
# return and continue executing. msbuild_use_call provides a way to
# disable this but I have not seen any adverse effect from doing that
Expand Down

0 comments on commit 6a162fd

Please sign in to comment.