Skip to content

Remove bash dependency from init-compiler #77304

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Dec 21, 2022
Merged

Remove bash dependency from init-compiler #77304

Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
26f9fb1
Remove bash dependency from init-compiler
am11 Oct 21, 2022
0d8d146
Remove dependency on pipeline-logging-functions.sh
am11 Oct 21, 2022
30b703d
Remove hard-coded compiler versions from build
am11 Oct 25, 2022
3396707
Improve objcopy conditions
am11 Oct 25, 2022
78769a8
Merge dotnet/main into feature/shell-scripts
am11 Oct 25, 2022
210507b
Merge branch 'main' into feature/shell-scripts
am11 Dec 20, 2022
9049613
Update property name after merge-from-main
am11 Dec 20, 2022
68ab523
Use preferred property
am11 Dec 20, 2022
6a566f0
Feedback
am11 Dec 20, 2022
373ed2c
Update eng/testing/tests.singlefile.targets
am11 Dec 20, 2022
7163c3e
Fix condition
am11 Dec 20, 2022
ebaf445
Print stderr
am11 Dec 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
81 changes: 37 additions & 44 deletions eng/common/native/init-compiler.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,30 +1,25 @@
#!/usr/bin/env bash
#!/bin/sh
#
# This file detects the C/C++ compiler and exports it to the CC/CXX environment variables
#
# NOTE: some scripts source this file and rely on stdout being empty, make sure to not output anything here!

if [[ "$#" -lt 3 ]]; then
if [ -z "$build_arch" ] || [ -z "$compiler" ]; then
echo "Usage..."
echo "init-compiler.sh <script directory> <Architecture> <compiler>"
echo "Specify the script directory."
echo "build_arch=<ARCH> compiler=<NAME> init-compiler.sh"
echo "Specify the target architecture."
echo "Specify the name of compiler (clang or gcc)."
exit 1
fi

nativescriptroot="$1"
build_arch="$2"
compiler="$3"

case "$compiler" in
clang*|-clang*|--clang*)
# clangx.y or clang-x.y
version="$(echo "$compiler" | tr -d '[:alpha:]-=')"
parts=(${version//./ })
majorVersion="${parts[0]}"
minorVersion="${parts[1]}"
if [[ -z "$minorVersion" && "$majorVersion" -le 6 ]]; then
majorVersion="${version%%.*}"
[ -z "${version##*.*}" ] && minorVersion="${version#*.}"

if [ -z "$minorVersion" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -le 6 ]; then
minorVersion=0;
fi
compiler=clang
Expand All @@ -33,23 +28,20 @@ case "$compiler" in
gcc*|-gcc*|--gcc*)
# gccx.y or gcc-x.y
version="$(echo "$compiler" | tr -d '[:alpha:]-=')"
parts=(${version//./ })
majorVersion="${parts[0]}"
minorVersion="${parts[1]}"
majorVersion="${version%%.*}"
[ -z "${version##*.*}" ] && minorVersion="${version#*.}"
compiler=gcc
;;
esac

cxxCompiler="$compiler++"

. "$nativescriptroot"/../pipeline-logging-functions.sh

# clear the existing CC and CXX from environment
CC=
CXX=
LDFLAGS=

if [[ "$compiler" == "gcc" ]]; then cxxCompiler="g++"; fi
if [ "$compiler" = "gcc" ]; then cxxCompiler="g++"; fi

check_version_exists() {
desired_version=-1
Expand All @@ -66,74 +58,75 @@ check_version_exists() {
echo "$desired_version"
}

if [[ -z "$CLR_CC" ]]; then
if [ -z "$CLR_CC" ]; then

# Set default versions
if [[ -z "$majorVersion" ]]; then
if [ -z "$majorVersion" ]; then
# note: gcc (all versions) and clang versions higher than 6 do not have minor version in file name, if it is zero.
if [[ "$compiler" == "clang" ]]; then versions=( 15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5 )
elif [[ "$compiler" == "gcc" ]]; then versions=( 12 11 10 9 8 7 6 5 4.9 ); fi

for version in "${versions[@]}"; do
parts=(${version//./ })
desired_version="$(check_version_exists "${parts[0]}" "${parts[1]}")"
if [[ "$desired_version" != "-1" ]]; then majorVersion="${parts[0]}"; break; fi
if [ "$compiler" = "clang" ]; then versions="15 14 13 12 11 10 9 8 7 6.0 5.0 4.0 3.9 3.8 3.7 3.6 3.5"
elif [ "$compiler" = "gcc" ]; then versions="12 11 10 9 8 7 6 5 4.9"; fi

for version in $versions; do
_major="${version%%.*}"
[ -z "${version##*.*}" ] && _minor="${version#*.}"
desired_version="$(check_version_exists "$_major" "$_minor")"
if [ "$desired_version" != "-1" ]; then majorVersion="$_major"; break; fi
done

if [[ -z "$majorVersion" ]]; then
if [ -z "$majorVersion" ]; then
if command -v "$compiler" > /dev/null; then
if [[ "$(uname)" != "Darwin" ]]; then
Write-PipelineTelemetryError -category "Build" -type "warning" "Specific version of $compiler not found, falling back to use the one in PATH."
if [ "$(uname)" != "Darwin" ]; then
echo "Warning: Specific version of $compiler not found, falling back to use the one in PATH."
fi
CC="$(command -v "$compiler")"
CXX="$(command -v "$cxxCompiler")"
else
Write-PipelineTelemetryError -category "Build" "No usable version of $compiler found."
echo "No usable version of $compiler found."
exit 1
fi
else
if [[ "$compiler" == "clang" && "$majorVersion" -lt 5 ]]; then
if [[ "$build_arch" == "arm" || "$build_arch" == "armel" ]]; then
if [ "$compiler" = "clang" ] && [ "$majorVersion" -lt 5 ]; then
if [ "$build_arch" = "arm" ] || [ "$build_arch" = "armel" ]; then
if command -v "$compiler" > /dev/null; then
Write-PipelineTelemetryError -category "Build" -type "warning" "Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH."
echo "Warning: Found clang version $majorVersion which is not supported on arm/armel architectures, falling back to use clang from PATH."
CC="$(command -v "$compiler")"
CXX="$(command -v "$cxxCompiler")"
else
Write-PipelineTelemetryError -category "Build" "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH."
echo "Found clang version $majorVersion which is not supported on arm/armel architectures, and there is no clang in PATH."
exit 1
fi
fi
fi
fi
else
desired_version="$(check_version_exists "$majorVersion" "$minorVersion")"
if [[ "$desired_version" == "-1" ]]; then
Write-PipelineTelemetryError -category "Build" "Could not find specific version of $compiler: $majorVersion $minorVersion."
if [ "$desired_version" = "-1" ]; then
echo "Could not find specific version of $compiler: $majorVersion $minorVersion."
exit 1
fi
fi

if [[ -z "$CC" ]]; then
if [ -z "$CC" ]; then
CC="$(command -v "$compiler$desired_version")"
CXX="$(command -v "$cxxCompiler$desired_version")"
if [[ -z "$CXX" ]]; then CXX="$(command -v "$cxxCompiler")"; fi
if [ -z "$CXX" ]; then CXX="$(command -v "$cxxCompiler")"; fi
fi
else
if [[ ! -f "$CLR_CC" ]]; then
Write-PipelineTelemetryError -category "Build" "CLR_CC is set but path '$CLR_CC' does not exist"
if [ ! -f "$CLR_CC" ]; then
echo "CLR_CC is set but path '$CLR_CC' does not exist"
exit 1
fi
CC="$CLR_CC"
CXX="$CLR_CXX"
fi

if [[ -z "$CC" ]]; then
Write-PipelineTelemetryError -category "Build" "Unable to find $compiler."
if [ -z "$CC" ]; then
echo "Unable to find $compiler."
exit 1
fi

# Only lld version >= 9 can be considered stable. lld doesn't support s390x.
if [[ "$compiler" == "clang" && "$majorVersion" -ge 9 && "$build_arch" != "s390x" ]]; then
if [ "$compiler" = "clang" ] && [ -n "$majorVersion" ] && [ "$majorVersion" -ge 9 ] && [ "$build_arch" != "s390x" ]; then
if "$CC" -fuse-ld=lld -Wl,--version >/dev/null 2>&1; then
LDFLAGS="-fuse-ld=lld"
fi
Expand Down
2 changes: 1 addition & 1 deletion eng/native/gen-buildsys.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ compiler="$5"

if [[ "$compiler" != "default" ]]; then
nativescriptroot="$( cd -P "$scriptroot/../common/native" && pwd )"
source "$nativescriptroot/init-compiler.sh" "$nativescriptroot" "$host_arch" "$compiler"
build_arch="$host_arch" compiler="$compiler" . "$nativescriptroot/init-compiler.sh"

CCC_CC="$CC"
CCC_CXX="$CXX"
Expand Down
17 changes: 15 additions & 2 deletions eng/testing/tests.singlefile.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@
<PropertyGroup Condition="'$(TestNativeAot)' == 'true'">
<IlcToolsPath>$(CoreCLRILCompilerDir)</IlcToolsPath>
<IlcToolsPath Condition="'$(TargetArchitecture)' != '$(BuildArchitecture)'">$(CoreCLRCrossILCompilerDir)</IlcToolsPath>
<CppCompilerAndLinker Condition="'$(TargetArchitecture)' != '$(BuildArchitecture)' and '$(HostOS)' == 'linux' and '$(RuntimeIdentifier)' == 'linux-musl-arm64'">clang-15</CppCompilerAndLinker>
<CppCompilerAndLinker Condition="'$(TargetArchitecture)' != '$(BuildArchitecture)' and '$(HostOS)' == 'linux' and '$(RuntimeIdentifier)' != 'linux-musl-arm64'">clang-9</CppCompilerAndLinker>
<SysRoot Condition="'$(TargetArchitecture)' != '$(BuildArchitecture)' and '$(HostOS)' != 'windows'">$(ROOTFS_DIR)</SysRoot>
<IlcBuildTasksPath>$(CoreCLRILCompilerDir)netstandard/ILCompiler.Build.Tasks.dll</IlcBuildTasksPath>
<IlcSdkPath>$(CoreCLRAotSdkDir)</IlcSdkPath>
Expand Down Expand Up @@ -84,6 +82,21 @@
</ItemGroup>
</Target>

<Target Name="LocateNativeCompiler"
Condition="'$(TestNativeAot)' == 'true' and '$(TargetOS)' != 'windows'"
BeforeTargets="SetupOSSpecificProps">
<PropertyGroup>
<CppCompilerAndLinker Condition="'$(CppCompilerAndLinker)' == ''">clang</CppCompilerAndLinker>
Copy link
Member

@akoeplinger akoeplinger Dec 20, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

looking at the whole PR again, maybe it'd be better to not use this property name at all and go with what we use in the NativeExports.csproj:

    <PropertyGroup>
      <NativeCompiler>$(Compiler)</NativeCompiler>
      <NativeCompiler Condition="'$(NativeCompiler)' == ''">clang</NativeCompiler>
    </PropertyGroup>

that'd also support overriding via the $(Compiler) property.

Copy link
Member Author

@am11 am11 Dec 20, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Compiler == CppCompilerAndLinker here, see eng/build.sh

Copy link
Member

@akoeplinger akoeplinger Dec 20, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

omg, so many abstractions 😄

Copy link
Member Author

@am11 am11 Dec 20, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

:)

I am all for replacing CppCompilerAndLinker and CppLinker in favor of NativeCompiler as property name, but I think some consumers are relying on CppCompilerAndLinker.

</PropertyGroup>

<Exec Command="sh -c 'build_arch=&quot;$(TargetArchitecture)&quot; compiler=&quot;$(CppCompilerAndLinker)&quot; . &quot;$(RepositoryEngineeringDir)/common/native/init-compiler.sh&quot; &amp;&amp; echo $CC' 2>/dev/null"
EchoOff="true"
ConsoleToMsBuild="true"
StandardOutputImportance="Low">
<Output TaskParameter="ConsoleOutput" PropertyName="CppLinker" />
</Exec>
</Target>

<Target Name="PublishTestAsSingleFile"
Condition="'$(IsCrossTargetingBuild)' != 'true'"
AfterTargets="Build"
Expand Down
6 changes: 3 additions & 3 deletions src/coreclr/nativeaot/BuildIntegration/Microsoft.NETCore.Native.Unix.targets
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,10 +140,10 @@ The .NET Foundation licenses this file to you under the MIT license.
</PropertyGroup>

<PropertyGroup Condition="'$(ObjCopyName)' == '' and '$(TargetOS)' != 'OSX'">
<ObjCopyName Condition="'$(CppCompilerAndLinker)' != 'clang'">objcopy</ObjCopyName>
<ObjCopyName Condition="'$(CppCompilerAndLinker)' == 'clang'">llvm-objcopy</ObjCopyName>
<ObjCopyName Condition="!$(CppCompilerAndLinker.Contains('clang'))">objcopy</ObjCopyName>
<ObjCopyName Condition="$(CppCompilerAndLinker.Contains('clang'))">llvm-objcopy</ObjCopyName>
<ObjCopyNameAlternative />
<ObjCopyNameAlternative Condition="'$(CppCompilerAndLinker)' == 'clang'">objcopy</ObjCopyNameAlternative>
<ObjCopyNameAlternative Condition="$(CppCompilerAndLinker.Contains('clang'))">objcopy</ObjCopyNameAlternative>
</PropertyGroup>

<Error Condition="'$(_WhereLinker)' != '0' and '$(TargetOS)' == 'OSX'" Text="Platform linker ('$(CppLinker)') not found in PATH. Try installing Xcode to resolve the problem." />
Expand Down
17 changes: 15 additions & 2 deletions src/coreclr/tools/aot/crossgen2/crossgen2.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,6 @@
<PropertyGroup Condition="'$(NativeAotSupported)' == 'true'">
<IlcToolsPath>$(CoreCLRILCompilerDir)</IlcToolsPath>
<IlcToolsPath Condition="'$(TargetArchitecture)' != '$(BuildArchitecture)'">$(CoreCLRCrossILCompilerDir)</IlcToolsPath>
<CppCompilerAndLinker Condition="'$(TargetArchitecture)' != '$(BuildArchitecture)' and '$(HostOS)' == 'linux' and '$(RuntimeIdentifier)' == 'linux-musl-arm64'">clang-15</CppCompilerAndLinker>
<CppCompilerAndLinker Condition="'$(TargetArchitecture)' != '$(BuildArchitecture)' and '$(HostOS)' == 'linux' and '$(RuntimeIdentifier)' != 'linux-musl-arm64'">clang-9</CppCompilerAndLinker>
<SysRoot Condition="'$(TargetArchitecture)' != '$(BuildArchitecture)' and '$(HostOS)' != 'windows'">$(ROOTFS_DIR)</SysRoot>
<IlcBuildTasksPath>$(CoreCLRILCompilerDir)netstandard/ILCompiler.Build.Tasks.dll</IlcBuildTasksPath>
<IlcSdkPath>$(CoreCLRAotSdkDir)</IlcSdkPath>
Expand All @@ -59,4 +57,19 @@
</ItemGroup>
</Target>

<Target Name="LocateNativeCompiler"
Condition="'$(RunningPublish)' == 'true' and '$(HostOS)' != 'windows'"
BeforeTargets="SetupOSSpecificProps">
<PropertyGroup>
<CppCompilerAndLinker Condition="'$(CppCompilerAndLinker)' == ''">clang</CppCompilerAndLinker>
</PropertyGroup>

<Exec Command="sh -c 'build_arch=&quot;$(TargetArchitecture)&quot; compiler=&quot;$(CppCompilerAndLinker)&quot; . &quot;$(RepositoryEngineeringDir)/common/native/init-compiler.sh&quot; &amp;&amp; echo $CC' 2>/dev/null"
EchoOff="true"
ConsoleToMsBuild="true"
StandardOutputImportance="Low">
<Output TaskParameter="ConsoleOutput" PropertyName="CppLinker" />
</Exec>
</Target>

</Project>
4 changes: 2 additions & 2 deletions ...raries/System.Runtime.InteropServices/tests/TestAssets/NativeExports/NativeExports.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,14 +44,14 @@
<NativeCompiler Condition="'$(NativeCompiler)' == ''">clang</NativeCompiler>
</PropertyGroup>

<Exec Command="bash -c 'source &quot;$(RepositoryEngineeringDir)/common/native/init-compiler.sh&quot; &quot;$(RepositoryEngineeringDir)/common/native&quot; $(TargetArchitecture) $(NativeCompiler) &amp;&amp; echo $CC' 2>/dev/null"
<Exec Command="sh -c 'build_arch=&quot;$(TargetArchitecture)&quot; compiler=&quot;$(NativeCompiler)&quot; . &quot;$(RepositoryEngineeringDir)/common/native/init-compiler.sh&quot; &amp;&amp; echo $CC' 2>/dev/null"
EchoOff="true"
ConsoleToMsBuild="true"
StandardOutputImportance="Low">
<Output TaskParameter="ConsoleOutput" PropertyName="DnneCompilerCommand" />
</Exec>

<Exec Command="bash -c 'source &quot;$(RepositoryEngineeringDir)/common/native/init-compiler.sh&quot; &quot;$(RepositoryEngineeringDir)/common/native&quot; $(TargetArchitecture) $(NativeCompiler) &amp;&amp; echo $LDFLAGS' 2>/dev/null"
<Exec Command="sh -c 'build_arch=&quot;$(TargetArchitecture)&quot; compiler=&quot;$(NativeCompiler)&quot; . &quot;$(RepositoryEngineeringDir)/common/native/init-compiler.sh&quot; &amp;&amp; echo $LDFLAGS' 2>/dev/null"
EchoOff="true"
ConsoleToMsBuild="true"
StandardOutputImportance="Low">
Expand Down
2 changes: 1 addition & 1 deletion src/mono/mono.proj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -513,7 +513,7 @@
<PropertyGroup>
<EMSDK_PATH>$([MSBuild]::EnsureTrailingSlash('$(EMSDK_PATH)'))</EMSDK_PATH>
<_MonoCMakeConfigureCommand>cmake @(_MonoCMakeArgs, ' ') $(MonoCMakeExtraArgs) &quot;$(MonoProjectRoot.TrimEnd('\/'))&quot;</_MonoCMakeConfigureCommand>
<_MonoCMakeConfigureCommand Condition="'$(TargetsBrowser)' != 'true' and '$(_MonoRunInitCompiler)' != 'false' and '$(HostOS)' != 'windows'">bash -c 'source $(RepositoryEngineeringCommonDir)native/init-compiler.sh &quot;$(RepositoryEngineeringCommonDir)native&quot; &quot;$(_CompilerTargetArch)&quot; &quot;$(MonoCCompiler)&quot; &amp;&amp; @(_MonoBuildEnv, ' ') $(_MonoCMakeConfigureCommand)'</_MonoCMakeConfigureCommand>
<_MonoCMakeConfigureCommand Condition="'$(TargetsBrowser)' != 'true' and '$(_MonoRunInitCompiler)' != 'false' and '$(HostOS)' != 'windows'">sh -c 'build_arch=&quot;$(_CompilerTargetArch)&quot; compiler=&quot;$(MonoCCompiler)&quot; . &quot;$(RepositoryEngineeringCommonDir)native/init-compiler.sh&quot; &amp;&amp; @(_MonoBuildEnv, ' ') $(_MonoCMakeConfigureCommand)'</_MonoCMakeConfigureCommand>
<_MonoCMakeConfigureCommand Condition="'$(TargetsBrowser)' != 'true' and '$(_MonoRunInitCompiler)' != 'false' and '$(HostOS)' == 'windows'">call &quot;$(RepositoryEngineeringDir)native\init-vs-env.cmd&quot; $(_CompilerTargetArch) &amp;&amp; cd /D &quot;$(MonoObjDir)&quot; &amp;&amp; @(_MonoBuildEnv, ' ') $(_MonoCMakeConfigureCommand)</_MonoCMakeConfigureCommand>
<_MonoCMakeConfigureCommand Condition="'$(TargetsBrowser)' != 'true' and '$(_MonoRunInitCompiler)' == 'false'">$(_MonoCCOption) $(_MonoCXXOption) @(_MonoBuildEnv, ' ') $(_MonoCMakeConfigureCommand)</_MonoCMakeConfigureCommand>
<_MonoCMakeConfigureCommand Condition="'$(TargetsBrowser)' == 'true' and '$(HostOS)' != 'windows'">bash -c 'source $(EMSDK_PATH)/emsdk_env.sh 2>&amp;1 &amp;&amp; emcmake $(_MonoCMakeConfigureCommand)'</_MonoCMakeConfigureCommand>
Expand Down
27 changes: 17 additions & 10 deletions src/native/libs/System.Globalization.Native/local_build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,24 @@

# Currently, only Linux is supported

SHIM_SOURCE_DIR=$1/native/src
INTERMEDIATE_OUTPUT_PATH=$2
SHIM_SOURCE_DIR="$1"/native/src
INTERMEDIATE_OUTPUT_PATH="$2"

if [ -d "$SHIM_SOURCE_DIR" ]; then
LOCAL_SHIM_DIR="$INTERMEDIATE_OUTPUT_PATH"/libs/System.Globalization.Native/build
mkdir -p "$LOCAL_SHIM_DIR" && cd "$LOCAL_SHIM_DIR"
if [ $? -ne 0 ]; then echo "local_build.sh::ERROR: Cannot use local build directory"; exit 1; fi
cmake -S "$SHIM_SOURCE_DIR/libs/System.Globalization.Native/" -DLOCAL_BUILD:STRING=1 -DCLR_CMAKE_TARGET_UNIX:STRING=1
if [ $? -ne 0 ]; then echo "local_build.sh::ERROR: cmake failed"; exit 1; fi
make -j
if [ $? -ne 0 ]; then echo "local_build.sh::ERROR: Build failed"; exit 1; fi
fi

exit 0
if ! mkdir -p "$LOCAL_SHIM_DIR" && cd "$LOCAL_SHIM_DIR"; then
echo "local_build.sh::ERROR: Cannot use local build directory"
exit 1
fi

if ! cmake -S "$SHIM_SOURCE_DIR/libs/System.Globalization.Native/" -DLOCAL_BUILD:STRING=1 -DCLR_CMAKE_TARGET_UNIX:STRING=1; then
echo "local_build.sh::ERROR: cmake failed"
exit 1
fi

if ! make -j; then
echo "local_build.sh::ERROR: Build failed"
exit 1
fi
fi