diff --git a/.gitignore b/.gitignore index e6d289dd19b..6ca107a5779 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,13 @@ nCrunchTemp_* tests/FSharp.Compiler.Service.Tests/FSharp.CompilerService.SurfaceArea.netstandard.actual *.vsp /tests/AheadOfTime/Trimming/output.txt +*.svclog +micro.exe +positive.exe +/tests/FSharp.Compiler.ComponentTests/FSharpChecker/StandardError.txt +/tests/FSharp.Compiler.ComponentTests/FSharpChecker/StandardOutput.txt + +# ilverify baseline result files +*.bsl.actual +/src/FSharp.DependencyManager.Nuget/StandardError.txt +/src/FSharp.DependencyManager.Nuget/StandardOutput.txt diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 301279bb18a..aaedc450567 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -100,6 +100,8 @@ extends: enablePublishBuildAssets: true enablePublishUsingPipelines: $(_PublishUsingPipelines) enableSourceBuild: true + sourceBuildParameters: + enableInternalSources: true enableTelemetry: true helixRepo: dotnet/fsharp jobs: diff --git a/eng/SourceBuildPrebuiltBaseline.xml b/eng/SourceBuildPrebuiltBaseline.xml index 18c94ed53e3..285bf7b9a82 100644 --- a/eng/SourceBuildPrebuiltBaseline.xml +++ b/eng/SourceBuildPrebuiltBaseline.xml @@ -19,4 +19,18 @@ + + + + + + + + + + + + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 6994b087c4f..ffb393a83f6 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -30,17 +30,17 @@ - + https://github.com/dotnet/arcade - 188340e12c0a372b1681ad6a5e72c608021efdba + 802042c6e779b73b4edb012ee1d5bae02ec8d41c - + https://github.com/dotnet/xliff-tasks 73f0850939d96131c28cf6ea6ee5aacb4da0083a - + https://github.com/dotnet/xliff-tasks 73f0850939d96131c28cf6ea6ee5aacb4da0083a diff --git a/eng/Versions.props b/eng/Versions.props index 8ac9507e013..1f1b980eff2 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -14,7 +14,7 @@ 8 0 - 301 + 302 0 @@ -79,12 +79,12 @@ 4.5.1 - 7.0.0 + 8.0.0 1.6.0 7.0.2 4.5.5 4.7.0 - 7.0.0 + 8.0.0 7.0.0 6.0.0 4.5.0 @@ -126,10 +126,10 @@ 11.0.50727 15.0.25123-Dev15Preview - 17.7.0-preview-23217-02 - 17.7.0-preview-23217-02 - 17.7.0-preview-23217-02 - 17.7.0-preview-23217-02 + 17.10.29 + $(MicrosoftBuildVersion) + $(MicrosoftBuildVersion) + $(MicrosoftBuildVersion) $(VisualStudioEditorPackagesVersion) $(VisualStudioEditorPackagesVersion) diff --git a/eng/build.sh b/eng/build.sh index f8db4462ff7..f90768c7a4c 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -161,6 +161,13 @@ while [[ $# > 0 ]]; do tfm=$2 shift ;; + # nop implementations of runtime* args sourcebuild might pass in on internal builds + --runtimesourcefeed) + shift + ;; + --runtimesourcefeedkey) + shift + ;; /p:*) properties="$properties $1" ;; diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 0998e875e5f..f93dc440df0 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -40,7 +40,7 @@ if(TARGET_ARCH_NAME STREQUAL "arm") set(TOOLCHAIN "arm-linux-gnueabihf") endif() if(TIZEN) - set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf/9.2.0") + set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf") endif() elseif(TARGET_ARCH_NAME STREQUAL "arm64") set(CMAKE_SYSTEM_PROCESSOR aarch64) @@ -49,7 +49,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64") elseif(LINUX) set(TOOLCHAIN "aarch64-linux-gnu") if(TIZEN) - set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0") + set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu") endif() elseif(FREEBSD) set(triple "aarch64-unknown-freebsd12") @@ -58,7 +58,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "armel") set(CMAKE_SYSTEM_PROCESSOR armv7l) set(TOOLCHAIN "arm-linux-gnueabi") if(TIZEN) - set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/9.2.0") + set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi") endif() elseif(TARGET_ARCH_NAME STREQUAL "armv6") set(CMAKE_SYSTEM_PROCESSOR armv6l) @@ -95,7 +95,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x64") elseif(LINUX) set(TOOLCHAIN "x86_64-linux-gnu") if(TIZEN) - set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu/9.2.0") + set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu") endif() elseif(FREEBSD) set(triple "x86_64-unknown-freebsd12") @@ -112,7 +112,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86") set(TOOLCHAIN "i686-linux-gnu") endif() if(TIZEN) - set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu/9.2.0") + set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu") endif() else() message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm, arm64, armel, armv6, ppc64le, riscv64, s390x, x64 and x86 are supported!") @@ -124,26 +124,25 @@ endif() # Specify include paths if(TIZEN) - if(TARGET_ARCH_NAME STREQUAL "arm") - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7hl-tizen-linux-gnueabihf) - endif() - if(TARGET_ARCH_NAME STREQUAL "armel") - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7l-tizen-linux-gnueabi) - endif() - if(TARGET_ARCH_NAME STREQUAL "arm64") - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/) - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/aarch64-tizen-linux-gnu) - endif() - if(TARGET_ARCH_NAME STREQUAL "x86") - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/i586-tizen-linux-gnu) - endif() - if(TARGET_ARCH_NAME STREQUAL "x64") - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/) - include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/x86_64-tizen-linux-gnu) + function(find_toolchain_dir prefix) + # Dynamically find the version subdirectory + file(GLOB DIRECTORIES "${prefix}/*") + list(GET DIRECTORIES 0 FIRST_MATCH) + get_filename_component(TOOLCHAIN_VERSION ${FIRST_MATCH} NAME) + + set(TIZEN_TOOLCHAIN_PATH "${prefix}/${TOOLCHAIN_VERSION}" PARENT_SCOPE) + endfunction() + + if(TARGET_ARCH_NAME MATCHES "^(arm|armel|x86)$") + find_toolchain_dir("${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + else() + find_toolchain_dir("${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") endif() + + message(STATUS "TIZEN_TOOLCHAIN_PATH set to: ${TIZEN_TOOLCHAIN_PATH}") + + include_directories(SYSTEM ${TIZEN_TOOLCHAIN_PATH}/include/c++) + include_directories(SYSTEM ${TIZEN_TOOLCHAIN_PATH}/include/c++/${TIZEN_TOOLCHAIN}) endif() if(ANDROID) @@ -265,22 +264,24 @@ endif() if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") if(TIZEN) - add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-B${TIZEN_TOOLCHAIN_PATH}") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib") - add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-L${TIZEN_TOOLCHAIN_PATH}") endif() elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64)$") if(TIZEN) - add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-B${TIZEN_TOOLCHAIN_PATH}") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib64") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64") - add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-L${TIZEN_TOOLCHAIN_PATH}") add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib64") add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64") - add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-Wl,--rpath-link=${TIZEN_TOOLCHAIN_PATH}") endif() +elseif(TARGET_ARCH_NAME STREQUAL "s390x") + add_toolchain_linker_flag("--target=${TOOLCHAIN}") elseif(TARGET_ARCH_NAME STREQUAL "x86") if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl) add_toolchain_linker_flag("--target=${TOOLCHAIN}") @@ -288,10 +289,10 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86") endif() add_toolchain_linker_flag(-m32) if(TIZEN) - add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-B${TIZEN_TOOLCHAIN_PATH}") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib") - add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-L${TIZEN_TOOLCHAIN_PATH}") endif() elseif(ILLUMOS) add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib/amd64") @@ -328,6 +329,8 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") if(TARGET_ARCH_NAME STREQUAL "armel") add_compile_options(-mfloat-abi=softfp) endif() +elseif(TARGET_ARCH_NAME STREQUAL "s390x") + add_compile_options("--target=${TOOLCHAIN}") elseif(TARGET_ARCH_NAME STREQUAL "x86") if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/i586-alpine-linux-musl) add_compile_options(--target=${TOOLCHAIN}) diff --git a/eng/common/generate-sbom-prep.ps1 b/eng/common/generate-sbom-prep.ps1 index 3e5c1c74a1c..a0c7d792a76 100644 --- a/eng/common/generate-sbom-prep.ps1 +++ b/eng/common/generate-sbom-prep.ps1 @@ -4,18 +4,26 @@ Param( . $PSScriptRoot\pipeline-logging-functions.ps1 +# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly +# with their own overwriting ours. So we create it as a sub directory of the requested manifest path. +$ArtifactName = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" +$SafeArtifactName = $ArtifactName -replace '["/:<>\\|?@*"() ]', '_' +$SbomGenerationDir = Join-Path $ManifestDirPath $SafeArtifactName + +Write-Host "Artifact name before : $ArtifactName" +Write-Host "Artifact name after : $SafeArtifactName" + Write-Host "Creating dir $ManifestDirPath" + # create directory for sbom manifest to be placed -if (!(Test-Path -path $ManifestDirPath)) +if (!(Test-Path -path $SbomGenerationDir)) { - New-Item -ItemType Directory -path $ManifestDirPath - Write-Host "Successfully created directory $ManifestDirPath" + New-Item -ItemType Directory -path $SbomGenerationDir + Write-Host "Successfully created directory $SbomGenerationDir" } else{ Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder." } Write-Host "Updating artifact name" -$artifact_name = "${env:SYSTEM_STAGENAME}_${env:AGENT_JOBNAME}_SBOM" -replace '["/:<>\\|?@*"() ]', '_' -Write-Host "Artifact name $artifact_name" -Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$artifact_name" +Write-Host "##vso[task.setvariable variable=ARTIFACT_NAME]$SafeArtifactName" diff --git a/eng/common/generate-sbom-prep.sh b/eng/common/generate-sbom-prep.sh index d5c76dc827b..bbb4922151e 100644 --- a/eng/common/generate-sbom-prep.sh +++ b/eng/common/generate-sbom-prep.sh @@ -14,19 +14,24 @@ done scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" . $scriptroot/pipeline-logging-functions.sh +# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts. +artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM" +safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}" + manifest_dir=$1 -if [ ! -d "$manifest_dir" ] ; then - mkdir -p "$manifest_dir" - echo "Sbom directory created." $manifest_dir +# Normally - we'd listen to the manifest path given, but 1ES templates will overwrite if this level gets uploaded directly +# with their own overwriting ours. So we create it as a sub directory of the requested manifest path. +sbom_generation_dir="$manifest_dir/$safe_artifact_name" + +if [ ! -d "$sbom_generation_dir" ] ; then + mkdir -p "$sbom_generation_dir" + echo "Sbom directory created." $sbom_generation_dir else Write-PipelineTelemetryError -category 'Build' "Unable to create sbom folder." fi -artifact_name=$SYSTEM_STAGENAME"_"$AGENT_JOBNAME"_SBOM" echo "Artifact name before : "$artifact_name -# replace all special characters with _, some builds use special characters like : in Agent.Jobname, that is not a permissible name while uploading artifacts. -safe_artifact_name="${artifact_name//["/:<>\\|?@*$" ]/_}" echo "Artifact name after : "$safe_artifact_name export ARTIFACT_NAME=$safe_artifact_name echo "##vso[task.setvariable variable=ARTIFACT_NAME]$safe_artifact_name" diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index 5a3a32ea8d7..238945cb5ab 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -2,7 +2,6 @@ param( [Parameter(Mandatory=$true)][int] $BuildId, [Parameter(Mandatory=$true)][int] $PublishingInfraVersion, [Parameter(Mandatory=$true)][string] $AzdoToken, - [Parameter(Mandatory=$true)][string] $MaestroToken, [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net', [Parameter(Mandatory=$true)][string] $WaitPublishingFinish, [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters, @@ -31,13 +30,13 @@ try { } & $darc add-build-to-channel ` - --id $buildId ` - --publishing-infra-version $PublishingInfraVersion ` - --default-channels ` - --source-branch main ` - --azdev-pat $AzdoToken ` - --bar-uri $MaestroApiEndPoint ` - --password $MaestroToken ` + --id $buildId ` + --publishing-infra-version $PublishingInfraVersion ` + --default-channels ` + --source-branch main ` + --azdev-pat "$AzdoToken" ` + --bar-uri "$MaestroApiEndPoint" ` + --ci ` @optionalParams if ($LastExitCode -ne 0) { diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 73828dd30d3..4f0546dce12 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -64,7 +64,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.8.1-2" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.12.0" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true diff --git a/eng/common/sdl/NuGet.config b/eng/common/sdl/NuGet.config index 3849bdb3cf5..5bfbb02ef04 100644 --- a/eng/common/sdl/NuGet.config +++ b/eng/common/sdl/NuGet.config @@ -5,11 +5,11 @@ - + - + diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index 4715d75e974..81ded5b7f47 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -6,7 +6,6 @@ Param( [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ('artifacts')), # Required: the directory where build artifacts are located - [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault # Optional: list of SDL tools to run on source code. See 'configure-sdl-tool.ps1' for tools list # format. @@ -75,7 +74,7 @@ try { } Exec-BlockVerbosely { - & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel + & $(Join-Path $PSScriptRoot 'init-sdl.ps1') -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -GuardianLoggerLevel $GuardianLoggerLevel } $gdnFolder = Join-Path $workingDirectory '.gdn' @@ -104,7 +103,6 @@ try { -TargetDirectory $targetDirectory ` -GdnFolder $gdnFolder ` -ToolsList $tools ` - -AzureDevOpsAccessToken $AzureDevOpsAccessToken ` -GuardianLoggerLevel $GuardianLoggerLevel ` -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams ` -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams ` diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index 3ac1d92b370..588ff8e22fb 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -3,7 +3,6 @@ Param( [string] $Repository, [string] $BranchName='master', [string] $WorkingDirectory, - [string] $AzureDevOpsAccessToken, [string] $GuardianLoggerLevel='Standard' ) @@ -21,14 +20,7 @@ $ci = $true # Don't display the console progress UI - it's a huge perf hit $ProgressPreference = 'SilentlyContinue' -# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file -$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken")) -$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") -$uri = "https://dev.azure.com/dnceng/internal/_apis/git/repositories/sdl-tool-cfg/Items?path=$escapedRepository&versionDescriptor[versionOptions]=0&`$format=zip&api-version=5.0" -$zipFile = "$WorkingDirectory/gdn.zip" - Add-Type -AssemblyName System.IO.Compression.FileSystem -$gdnFolder = (Join-Path $WorkingDirectory '.gdn') try { # if the folder does not exist, we'll do a guardian init and push it to the remote repository diff --git a/eng/common/sdl/sdl.ps1 b/eng/common/sdl/sdl.ps1 index 648c5068d7d..7fe603fe995 100644 --- a/eng/common/sdl/sdl.ps1 +++ b/eng/common/sdl/sdl.ps1 @@ -4,6 +4,8 @@ function Install-Gdn { [Parameter(Mandatory=$true)] [string]$Path, + [string]$Source = "https://pkgs.dev.azure.com/dnceng/_packaging/Guardian1ESPTUpstreamOrgFeed/nuget/v3/index.json", + # If omitted, install the latest version of Guardian, otherwise install that specific version. [string]$Version ) @@ -19,7 +21,7 @@ function Install-Gdn { $ci = $true . $PSScriptRoot\..\tools.ps1 - $argumentList = @("install", "Microsoft.Guardian.Cli", "-Source https://securitytools.pkgs.visualstudio.com/_packaging/Guardian/nuget/v3/index.json", "-OutputDirectory $Path", "-NonInteractive", "-NoCache") + $argumentList = @("install", "Microsoft.Guardian.Cli.win-x64", "-Source $Source", "-OutputDirectory $Path", "-NonInteractive", "-NoCache") if ($Version) { $argumentList += "-Version $Version" diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml index 1f035fee73f..98ccbd7a9c1 100644 --- a/eng/common/templates-official/job/job.yml +++ b/eng/common/templates-official/job/job.yml @@ -38,6 +38,7 @@ parameters: enableSbom: true PackageVersion: 7.0.0 BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom jobs: - job: ${{ parameters.name }} @@ -261,4 +262,4 @@ jobs: targetPath: '$(Build.SourcesDirectory)\eng\common\BuildConfiguration' artifactName: 'BuildConfiguration' displayName: 'Publish build retry configuration' - continueOnError: true \ No newline at end of file + continueOnError: true diff --git a/eng/common/templates-official/job/publish-build-assets.yml b/eng/common/templates-official/job/publish-build-assets.yml index 589ac80a18b..b2ccd9df680 100644 --- a/eng/common/templates-official/job/publish-build-assets.yml +++ b/eng/common/templates-official/job/publish-build-assets.yml @@ -76,14 +76,17 @@ jobs: - task: NuGetAuthenticate@1 - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Build Assets inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1 + arguments: > + -task PublishBuildAssets -restore -msbuildEngine dotnet /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) - /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com + /p:MaestroApiEndpoint=https://maestro.dot.net /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:OfficialBuildId=$(Build.BuildNumber) condition: ${{ parameters.condition }} @@ -137,14 +140,16 @@ jobs: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Using Darc inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' + -AzdoToken '$(System.AccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml index f193dfbe236..4217d6d8b14 100644 --- a/eng/common/templates-official/job/source-build.yml +++ b/eng/common/templates-official/job/source-build.yml @@ -31,6 +31,12 @@ parameters: # container and pool. platform: {} + # If set to true and running on a non-public project, + # Internal blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} displayName: Source-Build (${{ parameters.platform.name }}) @@ -48,7 +54,7 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] @@ -62,6 +68,8 @@ jobs: clean: all steps: + - ${{ if eq(parameters.enableInternalSources, true) }}: + - template: /eng/common/templates-official/steps/enable-internal-runtimes.yml - template: /eng/common/templates-official/steps/source-build.yml parameters: platform: ${{ parameters.platform }} diff --git a/eng/common/templates-official/job/source-index-stage1.yml b/eng/common/templates-official/job/source-index-stage1.yml index f0513aee5b0..60dfb6b2d1c 100644 --- a/eng/common/templates-official/job/source-index-stage1.yml +++ b/eng/common/templates-official/job/source-index-stage1.yml @@ -1,6 +1,7 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20230228.2 + sourceIndexUploadPackageVersion: 2.0.0-20240502.12 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] @@ -14,14 +15,14 @@ jobs: dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} variables: - - name: SourceIndexPackageVersion - value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexUploadPackageVersion + value: ${{ parameters.sourceIndexUploadPackageVersion }} + - name: SourceIndexProcessBinlogPackageVersion + value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} - name: SourceIndexPackageSource value: ${{ parameters.sourceIndexPackageSource }} - name: BinlogPath value: ${{ parameters.binlogPath }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: source-dot-net stage1 variables - template: /eng/common/templates-official/variables/pool-providers.yml ${{ if ne(parameters.pool, '') }}: @@ -41,16 +42,16 @@ jobs: - ${{ preStep }} - task: UseDotNet@2 - displayName: Use .NET Core SDK 6 + displayName: Use .NET 8 SDK inputs: packageType: sdk - version: 6.0.x + version: 8.0.x installationPath: $(Agent.TempDirectory)/dotnet workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: Download Tools # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) @@ -62,7 +63,21 @@ jobs: displayName: Process Binlog into indexable sln - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + - task: AzureCLI@2 + displayName: Get stage 1 auth token + inputs: + azureSubscription: 'SourceDotNet Stage1 Publish' + addSpnToEnvironment: true + scriptType: 'ps' + scriptLocation: 'inlineScript' + inlineScript: | + echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId" + + - script: | + az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) + displayName: "Login to Azure" + + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 displayName: Upload stage1 artifacts to source index - env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) diff --git a/eng/common/templates-official/jobs/source-build.yml b/eng/common/templates-official/jobs/source-build.yml index 08e5db9bb11..5cf6a269c0b 100644 --- a/eng/common/templates-official/jobs/source-build.yml +++ b/eng/common/templates-official/jobs/source-build.yml @@ -21,6 +21,12 @@ parameters: # one job runs on 'defaultManagedPlatform'. platforms: [] + # If set to true and running on a non-public project, + # Internal nuget and blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - ${{ if ne(parameters.allCompletedJobId, '') }}: @@ -38,9 +44,11 @@ jobs: parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ platform }} + enableInternalSources: ${{ parameters.enableInternalSources }} - ${{ if eq(length(parameters.platforms), 0) }}: - template: /eng/common/templates-official/job/source-build.yml parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ parameters.defaultManagedPlatform }} + enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/eng/common/templates-official/post-build/common-variables.yml b/eng/common/templates-official/post-build/common-variables.yml index c24193acfc9..173914f2364 100644 --- a/eng/common/templates-official/post-build/common-variables.yml +++ b/eng/common/templates-official/post-build/common-variables.yml @@ -7,7 +7,7 @@ variables: # Default Maestro++ API Endpoint and API Version - name: MaestroApiEndPoint - value: "https://maestro-prod.westus2.cloudapp.azure.com" + value: "https://maestro.dot.net" - name: MaestroApiAccessToken value: $(MaestroAccessToken) - name: MaestroApiVersion diff --git a/eng/common/templates-official/post-build/post-build.yml b/eng/common/templates-official/post-build/post-build.yml index da1f40958b4..b81b8770b34 100644 --- a/eng/common/templates-official/post-build/post-build.yml +++ b/eng/common/templates-official/post-build/post-build.yml @@ -272,14 +272,16 @@ stages: - task: NuGetAuthenticate@1 - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Using Darc inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' + -AzdoToken '$(System.AccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates-official/steps/enable-internal-runtimes.yml b/eng/common/templates-official/steps/enable-internal-runtimes.yml new file mode 100644 index 00000000000..93a8394a666 --- /dev/null +++ b/eng/common/templates-official/steps/enable-internal-runtimes.yml @@ -0,0 +1,28 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default + +parameters: +- name: federatedServiceConnection + type: string + default: 'dotnetbuilds-internal-read' +- name: outputVariableName + type: string + default: 'dotnetbuilds-internal-container-read-token-base64' +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: true + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/templates-official/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: ${{ parameters.federatedServiceConnection }} + outputVariableName: ${{ parameters.outputVariableName }} + expiryInHours: ${{ parameters.expiryInHours }} + base64Encode: ${{ parameters.base64Encode }} + storageAccount: dotnetbuilds + container: internal + permissions: rl diff --git a/eng/common/templates-official/steps/execute-sdl.yml b/eng/common/templates-official/steps/execute-sdl.yml index 07426fde05d..301d5c591eb 100644 --- a/eng/common/templates-official/steps/execute-sdl.yml +++ b/eng/common/templates-official/steps/execute-sdl.yml @@ -9,8 +9,6 @@ parameters: steps: - task: NuGetAuthenticate@1 - inputs: - nuGetServiceConnections: GuardianConnect - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' diff --git a/eng/common/templates-official/steps/generate-sbom.yml b/eng/common/templates-official/steps/generate-sbom.yml index 1bf43bf807a..daf0957b68d 100644 --- a/eng/common/templates-official/steps/generate-sbom.yml +++ b/eng/common/templates-official/steps/generate-sbom.yml @@ -35,7 +35,7 @@ steps: PackageName: ${{ parameters.packageName }} BuildDropPath: ${{ parameters.buildDropPath }} PackageVersion: ${{ parameters.packageVersion }} - ManifestDirPath: ${{ parameters.manifestDirPath }} + ManifestDirPath: ${{ parameters.manifestDirPath }}/$(ARTIFACT_NAME) ${{ if ne(parameters.IgnoreDirectories, '') }}: AdditionalComponentDetectorArgs: '--IgnoreDirectories ${{ parameters.IgnoreDirectories }}' diff --git a/eng/common/templates-official/steps/get-delegation-sas.yml b/eng/common/templates-official/steps/get-delegation-sas.yml new file mode 100644 index 00000000000..c690cc0a070 --- /dev/null +++ b/eng/common/templates-official/steps/get-delegation-sas.yml @@ -0,0 +1,52 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: false +- name: storageAccount + type: string +- name: container + type: string +- name: permissions + type: string + default: 'rl' + +steps: +- task: AzureCLI@2 + displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + # Calculate the expiration of the SAS token and convert to UTC + $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + + # Temporarily work around a helix issue where SAS tokens with / in them will cause incorrect downloads + # of correlation payloads. https://github.com/dotnet/dnceng/issues/3484 + $sas = "" + do { + $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + } while($sas.IndexOf('/') -ne -1) + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + + if ('${{ parameters.base64Encode }}' -eq 'true') { + $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) + } + + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" diff --git a/eng/common/templates-official/steps/get-federated-access-token.yml b/eng/common/templates-official/steps/get-federated-access-token.yml new file mode 100644 index 00000000000..55e33bd38f7 --- /dev/null +++ b/eng/common/templates-official/steps/get-federated-access-token.yml @@ -0,0 +1,40 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: stepName + type: string + default: 'getFederatedAccessToken' +- name: condition + type: string + default: '' +# Resource to get a token for. Common values include: +# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps +# - 'https://storage.azure.com/' for storage +# Defaults to Azure DevOps +- name: resource + type: string + default: '499b84ac-1321-427f-aa17-267ca6975798' +- name: isStepOutputVariable + type: boolean + default: false + +steps: +- task: AzureCLI@2 + displayName: 'Getting federated access token for feeds' + name: ${{ parameters.stepName }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" + exit 1 + } + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken" \ No newline at end of file diff --git a/eng/common/templates-official/steps/send-to-helix.yml b/eng/common/templates-official/steps/send-to-helix.yml index 3eb7e2d5f84..22f2501307d 100644 --- a/eng/common/templates-official/steps/send-to-helix.yml +++ b/eng/common/templates-official/steps/send-to-helix.yml @@ -8,6 +8,7 @@ parameters: HelixConfiguration: '' # optional -- additional property attached to a job HelixPreCommands: '' # optional -- commands to run before Helix work item execution HelixPostCommands: '' # optional -- commands to run after Helix work item execution + HelixProjectArguments: '' # optional -- arguments passed to the build command for helixpublish.proj WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects @@ -24,12 +25,12 @@ parameters: IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) Creator: '' # optional -- if the build is external, use this to specify who is sending the job - DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO + DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj ${{ parameters.HelixProjectArguments }} /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' displayName: ${{ parameters.DisplayNamePrefix }} (Windows) env: BuildConfig: $(_BuildConfig) @@ -59,7 +60,7 @@ steps: SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj ${{ parameters.HelixProjectArguments }} /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog displayName: ${{ parameters.DisplayNamePrefix }} (Unix) env: BuildConfig: $(_BuildConfig) diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 8ec5c4f2d9f..e295031c098 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -128,7 +128,7 @@ jobs: - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - task: MicroBuildSigningPlugin@3 + - task: MicroBuildSigningPlugin@4 displayName: Install MicroBuild plugin inputs: signType: $(_SignType) diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 8ec0151def2..cc2b346ba8b 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -74,13 +74,16 @@ jobs: - task: NuGetAuthenticate@1 - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Build Assets inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1 + arguments: > + -task PublishBuildAssets -restore -msbuildEngine dotnet /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests' - /p:BuildAssetRegistryToken=$(MaestroAccessToken) /p:MaestroApiEndpoint=https://maestro.dot.net /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }} /p:OfficialBuildId=$(Build.BuildNumber) @@ -133,14 +136,16 @@ jobs: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Using Darc inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: -BuildId $(BARBuildId) + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: -BuildId $(BARBuildId) -PublishingInfraVersion 3 - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' + -AzdoToken '$(System.AccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index 8a3deef2b72..c48f95d93d9 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -31,6 +31,12 @@ parameters: # container and pool. platform: {} + # If set to true and running on a non-public project, + # Internal blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }} displayName: Source-Build (${{ parameters.platform.name }}) @@ -48,11 +54,11 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + demands: ImageOverride -equals Build.Ubuntu.2204.Amd64.Open ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + demands: ImageOverride -equals Build.Ubuntu.2204.Amd64 ${{ if ne(parameters.platform.pool, '') }}: pool: ${{ parameters.platform.pool }} @@ -61,6 +67,8 @@ jobs: clean: all steps: + - ${{ if eq(parameters.enableInternalSources, true) }}: + - template: /eng/common/templates/steps/enable-internal-runtimes.yml - template: /eng/common/templates/steps/source-build.yml parameters: platform: ${{ parameters.platform }} diff --git a/eng/common/templates/job/source-index-stage1.yml b/eng/common/templates/job/source-index-stage1.yml index b98202aa02d..0b6bb89dc78 100644 --- a/eng/common/templates/job/source-index-stage1.yml +++ b/eng/common/templates/job/source-index-stage1.yml @@ -1,6 +1,7 @@ parameters: runAsPublic: false - sourceIndexPackageVersion: 1.0.1-20230228.2 + sourceIndexUploadPackageVersion: 2.0.0-20240502.12 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20240129.2 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci" preSteps: [] @@ -14,14 +15,14 @@ jobs: dependsOn: ${{ parameters.dependsOn }} condition: ${{ parameters.condition }} variables: - - name: SourceIndexPackageVersion - value: ${{ parameters.sourceIndexPackageVersion }} + - name: SourceIndexUploadPackageVersion + value: ${{ parameters.sourceIndexUploadPackageVersion }} + - name: SourceIndexProcessBinlogPackageVersion + value: ${{ parameters.sourceIndexProcessBinlogPackageVersion }} - name: SourceIndexPackageSource value: ${{ parameters.sourceIndexPackageSource }} - name: BinlogPath value: ${{ parameters.binlogPath }} - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - group: source-dot-net stage1 variables - template: /eng/common/templates/variables/pool-providers.yml ${{ if ne(parameters.pool, '') }}: @@ -40,16 +41,16 @@ jobs: - ${{ preStep }} - task: UseDotNet@2 - displayName: Use .NET Core SDK 6 + displayName: Use .NET 8 SDK inputs: packageType: sdk - version: 6.0.x + version: 8.0.x installationPath: $(Agent.TempDirectory)/dotnet workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(SourceIndexPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version $(sourceIndexProcessBinlogPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version $(sourceIndexUploadPackageVersion) --add-source $(SourceIndexPackageSource) --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: Download Tools # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) @@ -61,7 +62,21 @@ jobs: displayName: Process Binlog into indexable sln - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) + - task: AzureCLI@2 + displayName: Get stage 1 auth token + inputs: + azureSubscription: 'SourceDotNet Stage1 Publish' + addSpnToEnvironment: true + scriptType: 'ps' + scriptLocation: 'inlineScript' + inlineScript: | + echo "##vso[task.setvariable variable=ARM_CLIENT_ID;issecret=true]$env:servicePrincipalId" + echo "##vso[task.setvariable variable=ARM_ID_TOKEN;issecret=true]$env:idToken" + echo "##vso[task.setvariable variable=ARM_TENANT_ID;issecret=true]$env:tenantId" + + - script: | + az login --service-principal -u $(ARM_CLIENT_ID) --tenant $(ARM_TENANT_ID) --allow-no-subscriptions --federated-token $(ARM_ID_TOKEN) + displayName: "Login to Azure" + + - script: $(Agent.TempDirectory)/.source-index/tools/UploadIndexStage1 -i .source-index/stage1output -n $(Build.Repository.Name) -s netsourceindexstage1 -b stage1 displayName: Upload stage1 artifacts to source index - env: - BLOB_CONTAINER_URL: $(source-dot-net-stage1-blob-container-url) diff --git a/eng/common/templates/jobs/source-build.yml b/eng/common/templates/jobs/source-build.yml index a15b07eb51d..5f46bfa895c 100644 --- a/eng/common/templates/jobs/source-build.yml +++ b/eng/common/templates/jobs/source-build.yml @@ -21,6 +21,12 @@ parameters: # one job runs on 'defaultManagedPlatform'. platforms: [] + # If set to true and running on a non-public project, + # Internal nuget and blob storage locations will be enabled. + # This is not enabled by default because many repositories do not need internal sources + # and do not need to have the required service connections approved in the pipeline. + enableInternalSources: false + jobs: - ${{ if ne(parameters.allCompletedJobId, '') }}: @@ -38,9 +44,11 @@ jobs: parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ platform }} + enableInternalSources: ${{ parameters.enableInternalSources }} - ${{ if eq(length(parameters.platforms), 0) }}: - template: /eng/common/templates/job/source-build.yml parameters: jobNamePrefix: ${{ parameters.jobNamePrefix }} platform: ${{ parameters.defaultManagedPlatform }} + enableInternalSources: ${{ parameters.enableInternalSources }} diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index aba44a25a33..c3b6a3012fe 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -268,14 +268,16 @@ stages: - task: NuGetAuthenticate@1 - - task: PowerShell@2 + - task: AzureCLI@2 displayName: Publish Using Darc inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} - -AzdoToken '$(publishing-dnceng-devdiv-code-r-build-re)' - -MaestroToken '$(MaestroApiAccessToken)' + -AzdoToken '$(System.AccessToken)' -WaitPublishingFinish true -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index 0c87f149a4a..64b9abc6850 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -11,13 +11,14 @@ steps: artifactName: ReleaseConfigs checkDownloadedFiles: true - - task: PowerShell@2 + - task: AzureCLI@2 name: setReleaseVars displayName: Set Release Configs Vars inputs: - targetType: inline - pwsh: true - script: | + azureSubscription: "Darc: Maestro Production" + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | try { if (!$Env:PromoteToMaestroChannels -or $Env:PromoteToMaestroChannels.Trim() -eq '') { $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt @@ -31,15 +32,16 @@ steps: $AzureDevOpsBuildId = $Env:Build_BuildId } else { - $buildApiEndpoint = "${Env:MaestroApiEndPoint}/api/builds/${Env:BARBuildId}?api-version=${Env:MaestroApiVersion}" + . $(Build.SourcesDirectory)\eng\common\tools.ps1 + $darc = Get-Darc + $buildInfo = & $darc get-build ` + --id ${{ parameters.BARBuildId }} ` + --extended ` + --output-format json ` + --ci ` + | convertFrom-Json - $apiHeaders = New-Object 'System.Collections.Generic.Dictionary[[String],[String]]' - $apiHeaders.Add('Accept', 'application/json') - $apiHeaders.Add('Authorization',"Bearer ${Env:MAESTRO_API_TOKEN}") - - $buildInfo = try { Invoke-WebRequest -Method Get -Uri $buildApiEndpoint -Headers $apiHeaders | ConvertFrom-Json } catch { Write-Host "Error: $_" } - - $BarId = $Env:BARBuildId + $BarId = ${{ parameters.BARBuildId }} $Channels = $Env:PromoteToMaestroChannels -split "," $Channels = $Channels -join "][" $Channels = "[$Channels]" @@ -65,6 +67,4 @@ steps: exit 1 } env: - MAESTRO_API_TOKEN: $(MaestroApiAccessToken) - BARBuildId: ${{ parameters.BARBuildId }} PromoteToMaestroChannels: ${{ parameters.PromoteToChannelIds }} diff --git a/eng/common/templates/steps/enable-internal-runtimes.yml b/eng/common/templates/steps/enable-internal-runtimes.yml new file mode 100644 index 00000000000..54dc9416c51 --- /dev/null +++ b/eng/common/templates/steps/enable-internal-runtimes.yml @@ -0,0 +1,28 @@ +# Obtains internal runtime download credentials and populates the 'dotnetbuilds-internal-container-read-token-base64' +# variable with the base64-encoded SAS token, by default + +parameters: +- name: federatedServiceConnection + type: string + default: 'dotnetbuilds-internal-read' +- name: outputVariableName + type: string + default: 'dotnetbuilds-internal-container-read-token-base64' +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: true + +steps: +- ${{ if ne(variables['System.TeamProject'], 'public') }}: + - template: /eng/common/templates/steps/get-delegation-sas.yml + parameters: + federatedServiceConnection: ${{ parameters.federatedServiceConnection }} + outputVariableName: ${{ parameters.outputVariableName }} + expiryInHours: ${{ parameters.expiryInHours }} + base64Encode: ${{ parameters.base64Encode }} + storageAccount: dotnetbuilds + container: internal + permissions: rl diff --git a/eng/common/templates/steps/execute-sdl.yml b/eng/common/templates/steps/execute-sdl.yml index 07426fde05d..fe0ebf8c904 100644 --- a/eng/common/templates/steps/execute-sdl.yml +++ b/eng/common/templates/steps/execute-sdl.yml @@ -9,8 +9,6 @@ parameters: steps: - task: NuGetAuthenticate@1 - inputs: - nuGetServiceConnections: GuardianConnect - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' @@ -36,16 +34,19 @@ steps: displayName: Execute SDL (Overridden) continueOnError: ${{ parameters.sdlContinueOnError }} condition: ${{ parameters.condition }} + env: + GUARDIAN_DEFAULT_PACKAGE_SOURCE_SECRET: $(System.AccessToken) - ${{ if eq(parameters.overrideParameters, '') }}: - powershell: ${{ parameters.executeAllSdlToolsScript }} -GuardianCliLocation $(GuardianCliLocation) -NugetPackageDirectory $(Build.SourcesDirectory)\.packages - -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} displayName: Execute SDL continueOnError: ${{ parameters.sdlContinueOnError }} condition: ${{ parameters.condition }} + env: + GUARDIAN_DEFAULT_PACKAGE_SOURCE_SECRET: $(System.AccessToken) - ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: # We want to publish the Guardian results and configuration for easy diagnosis. However, the diff --git a/eng/common/templates/steps/get-delegation-sas.yml b/eng/common/templates/steps/get-delegation-sas.yml new file mode 100644 index 00000000000..c690cc0a070 --- /dev/null +++ b/eng/common/templates/steps/get-delegation-sas.yml @@ -0,0 +1,52 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: expiryInHours + type: number + default: 1 +- name: base64Encode + type: boolean + default: false +- name: storageAccount + type: string +- name: container + type: string +- name: permissions + type: string + default: 'rl' + +steps: +- task: AzureCLI@2 + displayName: 'Generate delegation SAS Token for ${{ parameters.storageAccount }}/${{ parameters.container }}' + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + # Calculate the expiration of the SAS token and convert to UTC + $expiry = (Get-Date).AddHours(${{ parameters.expiryInHours }}).ToUniversalTime().ToString("yyyy-MM-ddTHH:mm:ssZ") + + # Temporarily work around a helix issue where SAS tokens with / in them will cause incorrect downloads + # of correlation payloads. https://github.com/dotnet/dnceng/issues/3484 + $sas = "" + do { + $sas = az storage container generate-sas --account-name ${{ parameters.storageAccount }} --name ${{ parameters.container }} --permissions ${{ parameters.permissions }} --expiry $expiry --auth-mode login --as-user -o tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + } while($sas.IndexOf('/') -ne -1) + + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to generate SAS token." + exit 1 + } + + if ('${{ parameters.base64Encode }}' -eq 'true') { + $sas = [Convert]::ToBase64String([System.Text.Encoding]::UTF8.GetBytes($sas)) + } + + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true]$sas" diff --git a/eng/common/templates/steps/get-federated-access-token.yml b/eng/common/templates/steps/get-federated-access-token.yml new file mode 100644 index 00000000000..55e33bd38f7 --- /dev/null +++ b/eng/common/templates/steps/get-federated-access-token.yml @@ -0,0 +1,40 @@ +parameters: +- name: federatedServiceConnection + type: string +- name: outputVariableName + type: string +- name: stepName + type: string + default: 'getFederatedAccessToken' +- name: condition + type: string + default: '' +# Resource to get a token for. Common values include: +# - '499b84ac-1321-427f-aa17-267ca6975798' for Azure DevOps +# - 'https://storage.azure.com/' for storage +# Defaults to Azure DevOps +- name: resource + type: string + default: '499b84ac-1321-427f-aa17-267ca6975798' +- name: isStepOutputVariable + type: boolean + default: false + +steps: +- task: AzureCLI@2 + displayName: 'Getting federated access token for feeds' + name: ${{ parameters.stepName }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + inputs: + azureSubscription: ${{ parameters.federatedServiceConnection }} + scriptType: 'pscore' + scriptLocation: 'inlineScript' + inlineScript: | + $accessToken = az account get-access-token --query accessToken --resource ${{ parameters.resource }} --output tsv + if ($LASTEXITCODE -ne 0) { + Write-Error "Failed to get access token for resource '${{ parameters.resource }}'" + exit 1 + } + Write-Host "Setting '${{ parameters.outputVariableName }}' with the access token value" + Write-Host "##vso[task.setvariable variable=${{ parameters.outputVariableName }};issecret=true;isOutput=${{ parameters.isStepOutputVariable }}]$accessToken" \ No newline at end of file diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index 3eb7e2d5f84..22f2501307d 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -8,6 +8,7 @@ parameters: HelixConfiguration: '' # optional -- additional property attached to a job HelixPreCommands: '' # optional -- commands to run before Helix work item execution HelixPostCommands: '' # optional -- commands to run after Helix work item execution + HelixProjectArguments: '' # optional -- arguments passed to the build command for helixpublish.proj WorkItemDirectory: '' # optional -- a payload directory to zip up and send to Helix; requires WorkItemCommand; incompatible with XUnitProjects WorkItemCommand: '' # optional -- a command to execute on the payload; requires WorkItemDirectory; incompatible with XUnitProjects WorkItemTimeout: '' # optional -- a timeout in TimeSpan.Parse-ready value (e.g. 00:02:00) for the work item command; requires WorkItemDirectory; incompatible with XUnitProjects @@ -24,12 +25,12 @@ parameters: IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting https://helix.int-dot.net ) Creator: '' # optional -- if the build is external, use this to specify who is sending the job - DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO + DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false steps: - - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' + - powershell: 'powershell "$env:BUILD_SOURCESDIRECTORY\eng\common\msbuild.ps1 $env:BUILD_SOURCESDIRECTORY\eng\common\helixpublish.proj ${{ parameters.HelixProjectArguments }} /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$env:BUILD_SOURCESDIRECTORY\artifacts\log\$env:BuildConfig\SendToHelix.binlog"' displayName: ${{ parameters.DisplayNamePrefix }} (Windows) env: BuildConfig: $(_BuildConfig) @@ -59,7 +60,7 @@ steps: SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) continueOnError: ${{ parameters.continueOnError }} - - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog + - script: $BUILD_SOURCESDIRECTORY/eng/common/msbuild.sh $BUILD_SOURCESDIRECTORY/eng/common/helixpublish.proj ${{ parameters.HelixProjectArguments }} /restore /p:TreatWarningsAsErrors=false /t:Test /bl:$BUILD_SOURCESDIRECTORY/artifacts/log/$BuildConfig/SendToHelix.binlog displayName: ${{ parameters.DisplayNamePrefix }} (Unix) env: BuildConfig: $(_BuildConfig) diff --git a/eng/common/templates/steps/telemetry-start.yml b/eng/common/templates/steps/telemetry-start.yml index 32c01ef0b55..6abbcb33a67 100644 --- a/eng/common/templates/steps/telemetry-start.yml +++ b/eng/common/templates/steps/telemetry-start.yml @@ -8,7 +8,7 @@ parameters: steps: - ${{ if and(eq(parameters.runAsPublic, 'false'), not(eq(variables['System.TeamProject'], 'public'))) }}: - - task: AzureKeyVault@1 + - task: AzureKeyVault@2 inputs: azureSubscription: 'HelixProd_KeyVault' KeyVaultName: HelixProdKV diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index eb188cfda41..82b2798ba30 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -42,7 +42,7 @@ [bool]$useInstalledDotNetCli = if (Test-Path variable:useInstalledDotNetCli) { $useInstalledDotNetCli } else { $true } # Enable repos to use a particular version of the on-line dotnet-install scripts. -# default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.ps1 +# default URL: https://builds.dotnet.microsoft.com/dotnet/scripts/v1/dotnet-install.ps1 [string]$dotnetInstallScriptVersion = if (Test-Path variable:dotnetInstallScriptVersion) { $dotnetInstallScriptVersion } else { 'v1' } # True to use global NuGet cache instead of restoring packages to repository-local directory. @@ -263,7 +263,7 @@ function GetDotNetInstallScript([string] $dotnetRoot) { if (!(Test-Path $installScript)) { Create-Directory $dotnetRoot $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit - $uri = "https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.ps1" + $uri = "https://builds.dotnet.microsoft.com/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.ps1" Retry({ Write-Host "GET $uri" @@ -321,7 +321,7 @@ function InstallDotNet([string] $dotnetRoot, $variations += @($installParameters) $dotnetBuilds = $installParameters.Clone() - $dotnetbuilds.AzureFeed = "https://dotnetbuilds.azureedge.net/public" + $dotnetbuilds.AzureFeed = "https://ci.dot.net/public" $variations += @($dotnetBuilds) if ($runtimeSourceFeed) { @@ -384,8 +384,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # If the version of msbuild is going to be xcopied, # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.8.1-2 - $defaultXCopyMSBuildVersion = '17.8.1-2' + # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/RoslynTools.MSBuild/versions/17.12.0 + $defaultXCopyMSBuildVersion = '17.12.0' if (!$vsRequirements) { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { @@ -892,7 +892,7 @@ function IsWindowsPlatform() { } function Get-Darc($version) { - $darcPath = "$TempDir\darc\$(New-Guid)" + $darcPath = "$TempDir\darc\$([guid]::NewGuid())" if ($version -ne $null) { & $PSScriptRoot\darc-init.ps1 -toolpath $darcPath -darcVersion $version | Out-Host } else { diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 3392e3a9992..68db1543023 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -54,7 +54,7 @@ warn_as_error=${warn_as_error:-true} use_installed_dotnet_cli=${use_installed_dotnet_cli:-true} # Enable repos to use a particular version of the on-line dotnet-install scripts. -# default URL: https://dotnet.microsoft.com/download/dotnet/scripts/v1/dotnet-install.sh +# default URL: https://builds.dotnet.microsoft.com/dotnet/scripts/v1/dotnet-install.sh dotnetInstallScriptVersion=${dotnetInstallScriptVersion:-'v1'} # True to use global NuGet cache instead of restoring packages to repository-local directory. @@ -234,7 +234,7 @@ function InstallDotNet { local public_location=("${installParameters[@]}") variations+=(public_location) - local dotnetbuilds=("${installParameters[@]}" --azure-feed "https://dotnetbuilds.azureedge.net/public") + local dotnetbuilds=("${installParameters[@]}" --azure-feed "https://ci.dot.net/public") variations+=(dotnetbuilds) if [[ -n "${6:-}" ]]; then @@ -297,7 +297,7 @@ function with_retries { function GetDotNetInstallScript { local root=$1 local install_script="$root/dotnet-install.sh" - local install_script_url="https://dotnet.microsoft.com/download/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.sh" + local install_script_url="https://builds.dotnet.microsoft.com/dotnet/scripts/$dotnetInstallScriptVersion/dotnet-install.sh" if [[ ! -a "$install_script" ]]; then mkdir -p "$root" diff --git a/global.json b/global.json index b1a203d2466..1fc2fa83c21 100644 --- a/global.json +++ b/global.json @@ -1,10 +1,10 @@ { "sdk": { - "version": "8.0.101", + "version": "8.0.114", "allowPrerelease": true }, "tools": { - "dotnet": "8.0.101", + "dotnet": "8.0.114", "vs": { "version": "17.8", "components": [ @@ -17,7 +17,7 @@ "perl": "5.38.2.2" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.24204.3", + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.25164.5", "Microsoft.DotNet.Helix.Sdk": "8.0.0-beta.23255.2" } } diff --git a/src/Compiler/Driver/ScriptClosure.fs b/src/Compiler/Driver/ScriptClosure.fs index 2374fd6793b..6ba00c6ae54 100644 --- a/src/Compiler/Driver/ScriptClosure.fs +++ b/src/Compiler/Driver/ScriptClosure.fs @@ -45,6 +45,9 @@ type LoadClosure = /// The resolved pacakge references along with the ranges of the #r positions in each file. PackageReferences: (range * string list)[] + /// The raw package manager lines in the script + PackageManagerLines: Map + /// Whether we're decided to use .NET Framework analysis for this script UseDesktopFramework: bool @@ -82,7 +85,8 @@ type CodeContext = module ScriptPreprocessClosure = /// Represents an input to the closure finding process - type ClosureSource = ClosureSource of fileName: string * referenceRange: range * sourceText: ISourceText * parseRequired: bool + type ClosureSource = + | ClosureSource of fileName: string * referenceRange: range * sourceText: ISourceText * Position option * parseRequired: bool /// Represents an output of the closure finding process type ClosureFile = @@ -253,7 +257,7 @@ module ScriptPreprocessClosure = | Some(n: int) -> new StreamReader(stream, Encoding.GetEncoding n) let source = reader.ReadToEnd() - [ ClosureSource(fileName, m, SourceText.ofString source, parseRequired) ] + [ ClosureSource(fileName, m, SourceText.ofString source, None, parseRequired) ] with RecoverableException exn -> errorRecovery exn m [] @@ -313,12 +317,23 @@ module ScriptPreprocessClosure = let packageReferences = Dictionary(HashIdentity.Structural) // Resolve the packages - let rec resolveDependencyManagerSources scriptName = + let rec resolveDependencyManagerSources scriptName (caret: Position option) = + let caretLine = + match caret with + | None -> Int32.MinValue + | Some pos -> pos.Line + + let isEditorCursorInPackageLines (line: PackageManagerLine) = + caretLine >= line.Range.StartLine && caretLine <= line.Range.EndLine + [ if not (loadScripts.Contains scriptName) then for kv in tcConfig.packageManagerLines do let packageManagerKey, packageManagerLines = kv.Key, kv.Value + let packageManagerLines = + packageManagerLines |> List.filter (not << isEditorCursorInPackageLines) + match packageManagerLines with | [] -> () | packageManagerLine :: _ -> @@ -426,7 +441,7 @@ module ScriptPreprocessClosure = let scriptText = stream.ReadAllText() loadScripts.Add script |> ignore let iSourceText = SourceText.ofString scriptText - yield! processClosureSource (ClosureSource(script, m, iSourceText, true)) + yield! processClosureSource (ClosureSource(script, m, iSourceText, None, true)) else // Send outputs via diagnostics @@ -443,7 +458,7 @@ module ScriptPreprocessClosure = tcConfig <- TcConfig.Create(tcConfigB, validate = false) ] - and processClosureSource (ClosureSource(fileName, m, sourceText, parseRequired)) = + and processClosureSource (ClosureSource(fileName, m, sourceText, caret, parseRequired)) = [ if not (observedSources.HaveSeen(fileName)) then observedSources.SetSeen(fileName) @@ -473,7 +488,7 @@ module ScriptPreprocessClosure = tcConfig <- tcConfigResult // We accumulate the tcConfig in order to collect assembly references - yield! resolveDependencyManagerSources fileName + yield! resolveDependencyManagerSources fileName caret let postSources = tcConfig.GetAvailableLoadedSources() @@ -483,7 +498,7 @@ module ScriptPreprocessClosure = else [] - yield! resolveDependencyManagerSources fileName + yield! resolveDependencyManagerSources fileName caret for m, subFile in sources do if IsScript subFile then @@ -540,7 +555,7 @@ module ScriptPreprocessClosure = | _ -> lastClosureFile /// Reduce the full directive closure into LoadClosure - let GetLoadClosure (rootFilename, closureFiles, tcConfig: TcConfig, codeContext, packageReferences, earlierDiagnostics) = + let GetLoadClosure (rootFilename, closureFiles, tcConfig: TcConfig, codeContext, packageReferences, earlierDiagnostics) : LoadClosure = // Mark the last file as isLastCompiland. let closureFiles = @@ -612,23 +627,21 @@ module ScriptPreprocessClosure = // Filter out non-root errors and warnings let allRootDiagnostics = allRootDiagnostics |> List.filter (fst >> isRootRange) - let result: LoadClosure = - { - SourceFiles = List.groupBy fst sourceFiles |> List.map (map2Of2 (List.map snd)) - References = List.groupBy fst references |> List.map (map2Of2 (List.map snd)) - PackageReferences = packageReferences - UseDesktopFramework = (tcConfig.primaryAssembly = PrimaryAssembly.Mscorlib) - SdkDirOverride = tcConfig.sdkDirOverride - UnresolvedReferences = unresolvedReferences - Inputs = sourceInputs - NoWarns = List.groupBy fst globalNoWarns |> List.map (map2Of2 (List.map snd)) - OriginalLoadReferences = tcConfig.loadedSources - ResolutionDiagnostics = resolutionDiagnostics - AllRootFileDiagnostics = allRootDiagnostics - LoadClosureRootFileDiagnostics = loadClosureRootDiagnostics - } - - result + { + SourceFiles = List.groupBy fst sourceFiles |> List.map (map2Of2 (List.map snd)) + References = List.groupBy fst references |> List.map (map2Of2 (List.map snd)) + PackageReferences = packageReferences + PackageManagerLines = tcConfig.packageManagerLines + UseDesktopFramework = (tcConfig.primaryAssembly = PrimaryAssembly.Mscorlib) + SdkDirOverride = tcConfig.sdkDirOverride + UnresolvedReferences = unresolvedReferences + Inputs = sourceInputs + NoWarns = List.groupBy fst globalNoWarns |> List.map (map2Of2 (List.map snd)) + OriginalLoadReferences = tcConfig.loadedSources + ResolutionDiagnostics = resolutionDiagnostics + AllRootFileDiagnostics = allRootDiagnostics + LoadClosureRootFileDiagnostics = loadClosureRootDiagnostics + } /// Given source text, find the full load closure. Used from service.fs, when editing a script file let GetFullClosureOfScriptText @@ -637,6 +650,7 @@ module ScriptPreprocessClosure = defaultFSharpBinariesDir, fileName, sourceText, + caret, codeContext, useSimpleResolution, useFsiAuxLib, @@ -649,7 +663,6 @@ module ScriptPreprocessClosure = reduceMemoryUsage, dependencyProvider ) = - // Resolve the basic references such as FSharp.Core.dll first, before processing any #I directives in the script // // This is tries to mimic the action of running the script in F# Interactive - the initial context for scripting is created @@ -700,7 +713,7 @@ module ScriptPreprocessClosure = reduceMemoryUsage ) - let closureSources = [ ClosureSource(fileName, range0, sourceText, true) ] + let closureSources = [ ClosureSource(fileName, range0, sourceText, caret, true) ] let closureFiles, tcConfig, packageReferences = FindClosureFiles(fileName, closureSources, tcConfig, codeContext, lexResourceManager, dependencyProvider) @@ -742,6 +755,7 @@ type LoadClosure with defaultFSharpBinariesDir, fileName: string, sourceText: ISourceText, + caret: Position option, implicitDefines, useSimpleResolution: bool, useFsiAuxLib, @@ -762,6 +776,7 @@ type LoadClosure with defaultFSharpBinariesDir, fileName, sourceText, + caret, implicitDefines, useSimpleResolution, useFsiAuxLib, diff --git a/src/Compiler/Driver/ScriptClosure.fsi b/src/Compiler/Driver/ScriptClosure.fsi index c5deec56b64..4d34efc66a7 100644 --- a/src/Compiler/Driver/ScriptClosure.fsi +++ b/src/Compiler/Driver/ScriptClosure.fsi @@ -42,6 +42,9 @@ type LoadClosure = /// The resolved pacakge references along with the ranges of the #r positions in each file. PackageReferences: (range * string list)[] + /// The raw package manager lines in the script + PackageManagerLines: Map + /// Whether we're decided to use .NET Framework analysis for this script UseDesktopFramework: bool @@ -80,6 +83,7 @@ type LoadClosure = defaultFSharpBinariesDir: string * fileName: string * sourceText: ISourceText * + caret: Position option * implicitDefines: CodeContext * useSimpleResolution: bool * useFsiAuxLib: bool * diff --git a/src/Compiler/Service/BackgroundCompiler.fs b/src/Compiler/Service/BackgroundCompiler.fs index 3a9b805b676..8b3b7aea9fe 100644 --- a/src/Compiler/Service/BackgroundCompiler.fs +++ b/src/Compiler/Service/BackgroundCompiler.fs @@ -112,6 +112,7 @@ type internal IBackgroundCompiler = abstract member GetProjectOptionsFromScript: fileName: string * sourceText: ISourceText * + caret: Position option * previewEnabled: bool option * loadedTimeStamp: System.DateTime option * otherFlags: string array option * @@ -126,6 +127,7 @@ type internal IBackgroundCompiler = abstract GetProjectSnapshotFromScript: fileName: string * sourceText: ISourceTextNew * + caret: Position option * documentSource: DocumentSource * previewEnabled: bool option * loadedTimeStamp: System.DateTime option * @@ -1307,6 +1309,7 @@ type internal BackgroundCompiler ( fileName, sourceText, + caret, previewEnabled, loadedTimeStamp, otherFlags, @@ -1360,6 +1363,7 @@ type internal BackgroundCompiler FSharpCheckerResultsSettings.defaultFSharpBinariesDir, fileName, sourceText, + caret, CodeContext.Editing, useSimpleResolution, useFsiAuxLib, @@ -1600,6 +1604,7 @@ type internal BackgroundCompiler ( fileName: string, sourceText: ISourceText, + caret: Position option, previewEnabled: bool option, loadedTimeStamp: DateTime option, otherFlags: string array option, @@ -1613,6 +1618,7 @@ type internal BackgroundCompiler self.GetProjectOptionsFromScript( fileName, sourceText, + caret, previewEnabled, loadedTimeStamp, otherFlags, @@ -1628,6 +1634,7 @@ type internal BackgroundCompiler ( fileName: string, sourceText: ISourceTextNew, + caret: Position option, documentSource: DocumentSource, previewEnabled: bool option, loadedTimeStamp: DateTime option, @@ -1644,6 +1651,7 @@ type internal BackgroundCompiler self.GetProjectOptionsFromScript( fileName, sourceText, + caret, previewEnabled, loadedTimeStamp, otherFlags, diff --git a/src/Compiler/Service/BackgroundCompiler.fsi b/src/Compiler/Service/BackgroundCompiler.fsi index 498e2a5102e..37d141d7bf8 100644 --- a/src/Compiler/Service/BackgroundCompiler.fsi +++ b/src/Compiler/Service/BackgroundCompiler.fsi @@ -91,6 +91,7 @@ type internal IBackgroundCompiler = abstract GetProjectOptionsFromScript: fileName: string * sourceText: ISourceText * + caret: Position option * previewEnabled: bool option * loadedTimeStamp: System.DateTime option * otherFlags: string array option * @@ -105,6 +106,7 @@ type internal IBackgroundCompiler = abstract GetProjectSnapshotFromScript: fileName: string * sourceText: ISourceTextNew * + caret: Position option * documentSource: DocumentSource * previewEnabled: bool option * loadedTimeStamp: System.DateTime option * diff --git a/src/Compiler/Service/FSharpCheckerResults.fs b/src/Compiler/Service/FSharpCheckerResults.fs index 46ea4bce4f5..72b4722a4c0 100644 --- a/src/Compiler/Service/FSharpCheckerResults.fs +++ b/src/Compiler/Service/FSharpCheckerResults.fs @@ -3630,6 +3630,7 @@ type FsiInteractiveChecker(legacyReferenceResolver, tcConfig: TcConfig, tcGlobal defaultFSharpBinariesDir, fileName, sourceText, + None, CodeContext.Editing, tcConfig.useSimpleResolution, tcConfig.useFsiAuxLib, diff --git a/src/Compiler/Service/TransparentCompiler.fs b/src/Compiler/Service/TransparentCompiler.fs index 679c0d46d3f..5bca9b8984d 100644 --- a/src/Compiler/Service/TransparentCompiler.fs +++ b/src/Compiler/Service/TransparentCompiler.fs @@ -402,6 +402,7 @@ type internal TransparentCompiler defaultFSharpBinariesDir, fileName, source, + None, CodeContext.Editing, useSimpleResolution, useFsiAuxLib, @@ -2258,6 +2259,7 @@ type internal TransparentCompiler ( fileName: string, sourceText: ISourceText, + caret: Position option, previewEnabled: bool option, loadedTimeStamp: DateTime option, otherFlags: string array option, @@ -2275,6 +2277,7 @@ type internal TransparentCompiler bc.GetProjectSnapshotFromScript( fileName, SourceTextNew.ofISourceText sourceText, + caret, DocumentSource.FileSystem, previewEnabled, loadedTimeStamp, @@ -2295,6 +2298,7 @@ type internal TransparentCompiler ( fileName: string, sourceText: ISourceTextNew, + _caret: Position option, documentSource: DocumentSource, previewEnabled: bool option, loadedTimeStamp: DateTime option, diff --git a/src/Compiler/Service/service.fs b/src/Compiler/Service/service.fs index 093a692f1ec..47d5f1fd811 100644 --- a/src/Compiler/Service/service.fs +++ b/src/Compiler/Service/service.fs @@ -535,6 +535,7 @@ type FSharpChecker ( fileName, source, + ?caret, ?previewEnabled, ?loadedTimeStamp, ?otherFlags, @@ -550,6 +551,7 @@ type FSharpChecker backgroundCompiler.GetProjectOptionsFromScript( fileName, source, + caret, previewEnabled, loadedTimeStamp, otherFlags, @@ -566,6 +568,7 @@ type FSharpChecker ( fileName, source, + ?caret, ?documentSource, ?previewEnabled, ?loadedTimeStamp, @@ -583,6 +586,7 @@ type FSharpChecker backgroundCompiler.GetProjectSnapshotFromScript( fileName, source, + caret, documentSource, previewEnabled, loadedTimeStamp, diff --git a/src/Compiler/Service/service.fsi b/src/Compiler/Service/service.fsi index 5e78154d77e..13390454474 100644 --- a/src/Compiler/Service/service.fsi +++ b/src/Compiler/Service/service.fsi @@ -226,6 +226,7 @@ type public FSharpChecker = /// /// Used to differentiate between scripts, to consider each script a separate project. Also used in formatted error messages. /// The source for the file. + /// The editor location for the cursor if available. /// Is the preview compiler enabled. /// Indicates when the script was loaded into the editing environment, /// so that an 'unload' and 'reload' action will cause the script to be considered as a new project, @@ -240,6 +241,7 @@ type public FSharpChecker = member GetProjectOptionsFromScript: fileName: string * source: ISourceText * + ?caret: Position * ?previewEnabled: bool * ?loadedTimeStamp: DateTime * ?otherFlags: string[] * @@ -253,6 +255,7 @@ type public FSharpChecker = /// Used to differentiate between scripts, to consider each script a separate project. Also used in formatted error messages. /// The source for the file. + /// The editor location for the cursor if available. /// DocumentSource to load any additional files. /// Is the preview compiler enabled. /// Indicates when the script was loaded into the editing environment, @@ -269,6 +272,7 @@ type public FSharpChecker = member GetProjectSnapshotFromScript: fileName: string * source: ISourceTextNew * + ?caret: Position * ?documentSource: DocumentSource * ?previewEnabled: bool * ?loadedTimeStamp: DateTime * diff --git a/src/FSharp.DependencyManager.Nuget/FSharp.DependencyManager.ProjectFile.fs b/src/FSharp.DependencyManager.Nuget/FSharp.DependencyManager.ProjectFile.fs index c8acf99d7e7..f70c795710e 100644 --- a/src/FSharp.DependencyManager.Nuget/FSharp.DependencyManager.ProjectFile.fs +++ b/src/FSharp.DependencyManager.Nuget/FSharp.DependencyManager.ProjectFile.fs @@ -11,6 +11,7 @@ type PackageReference = Version: string RestoreSources: string Script: string + UsePackageTargets: bool } module internal ProjectFile = diff --git a/src/FSharp.DependencyManager.Nuget/FSharp.DependencyManager.fs b/src/FSharp.DependencyManager.Nuget/FSharp.DependencyManager.fs index c6c7247b0df..1fa8a06df05 100644 --- a/src/FSharp.DependencyManager.Nuget/FSharp.DependencyManager.fs +++ b/src/FSharp.DependencyManager.Nuget/FSharp.DependencyManager.fs @@ -55,16 +55,30 @@ module FSharpDependencyManager = Version = ver RestoreSources = src Script = script + UsePackageTargets = usePackageTargets } = p + let usePackageTargets = + match usePackageTargets with + | false -> "ExcludeAssets='build;buildTransitive;buildMultitargeting'" + | true -> "" + seq { match not (String.IsNullOrEmpty(inc)), not (String.IsNullOrEmpty(ver)), not (String.IsNullOrEmpty(script)) with - | true, true, false -> yield sprintf @" " inc ver + | true, true, false -> + yield sprintf @" " inc ver usePackageTargets | true, true, true -> - yield sprintf @" " inc ver script - | true, false, false -> yield sprintf @" " inc - | true, false, true -> yield sprintf @" " inc script + yield + sprintf + @" " + inc + ver + script + usePackageTargets + | true, false, false -> yield sprintf @" " inc usePackageTargets + | true, false, true -> + yield sprintf @" " inc script usePackageTargets | _ -> () match not (String.IsNullOrEmpty(src)) with @@ -96,6 +110,7 @@ module FSharpDependencyManager = Version = "*" RestoreSources = "" Script = "" + UsePackageTargets = false } match options with @@ -114,11 +129,20 @@ module FSharpDependencyManager = let setVersion v = Some { current with Version = v } + let setUsePackageTargets v = + Some { current with UsePackageTargets = v } + match opt with | Some "include", Some v -> addInclude v |> parsePackageReferenceOption' rest implicitArgumentCount | Some "include", None -> raise (ArgumentException(SR.requiresAValue ("Include"))) | Some "version", Some v -> setVersion v |> parsePackageReferenceOption' rest implicitArgumentCount | Some "version", None -> setVersion "*" |> parsePackageReferenceOption' rest implicitArgumentCount + | Some "usepackagetargets", v -> + match v with + | Some v when v.ToLowerInvariant() = "true" -> setUsePackageTargets true + | Some v when v.ToLowerInvariant() = "false" -> setUsePackageTargets false + | _ -> raise (ArgumentException(ArgumentOutOfRangeException("usepackagetargets").Message)) + |> parsePackageReferenceOption' rest implicitArgumentCount | Some "restoresources", Some v -> Some { current with @@ -212,12 +236,12 @@ module FSharpDependencyManager = let referencesHaveWildCardVersion = // Verify to see if the developer specified a wildcard version. If they did then caching is not possible let hasWildCardVersion p = - // Todo: named record please let { Include = package Version = ver RestoreSources = _ Script = _ + UsePackageTargets = _ } = p diff --git a/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharpScriptTests.fs b/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharpScriptTests.fs index 78a535c39b9..bdbbdcdb861 100644 --- a/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharpScriptTests.fs +++ b/tests/FSharp.Compiler.Private.Scripting.UnitTests/FSharpScriptTests.fs @@ -482,3 +482,43 @@ test pfloat "1.234" let opt = script.Eval(code) |> getValue let value = opt.Value Assert.True(true = downcast value.ReflectionValue) + + [] + member _.``Nuget package with method duplicates differing only in generic arity``() = + // regression test for: https://github.com/dotnet/fsharp/issues/17796 + // Was an internal error + let code = """ +#r "nuget: Microsoft.Extensions.DependencyInjection.Abstractions" +open Microsoft.Extensions.DependencyInjection +let add (col:IServiceCollection) = + col.AddSingleton() +""" + use script = new FSharpScript(additionalArgs=[| |]) + let _value,diag = script.Eval(code) + Assert.Empty(diag) + + [] + [] + [] + [] + [] + member _.``Eval script with usepackagetargets options``(code, shouldSucceed, error) = + use script = new FSharpScript() + let result, errors = script.Eval(code) + match shouldSucceed with + | true -> + Assert.Empty(errors) + match result with + | Ok(_) -> () + | Error(ex) -> Assert.True(false, "expected no failures") + | false -> + Assert.NotEmpty(errors) + Assert.Equal(1, errors.Length) + // coreclr emits the value with "name: usepackagetargets", desktop framework just emits it "usepackagetargets" + Assert.Equal(error, errors[0].ToString() + .Replace("\r\n", "\r") + .Replace("Parameter name: ", "Parameter ") + .Replace("\rParameter ", " Parameter ") + .Replace("(Parameter ", "Parameter ") + .Replace("'usepackagetargets'", "usepackagetargets") + .Replace("usepackagetargets)", "usepackagetargets")) diff --git a/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.debug.bsl b/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.debug.bsl index 6c8423483a0..a34afb7bbdd 100644 --- a/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.debug.bsl +++ b/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.debug.bsl @@ -1996,7 +1996,7 @@ FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults: FSharp.Compiler.CodeAnalysi FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults: FSharp.Compiler.CodeAnalysis.FSharpSymbolUse[] GetUsesOfSymbolInFile(FSharp.Compiler.Symbols.FSharpSymbol, Microsoft.FSharp.Core.FSharpOption`1[System.Threading.CancellationToken]) FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults: FSharp.Compiler.Diagnostics.FSharpDiagnostic[] Diagnostics FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults: FSharp.Compiler.Diagnostics.FSharpDiagnostic[] get_Diagnostics() -FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults: FSharp.Compiler.EditorServices.DeclarationListInfo GetDeclarationListInfo(Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.CodeAnalysis.FSharpParseFileResults], Int32, System.String, FSharp.Compiler.EditorServices.PartialLongName, Microsoft.FSharp.Core.FSharpOption`1[Microsoft.FSharp.Core.FSharpFunc`2[Microsoft.FSharp.Core.Unit,Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.EditorServices.AssemblySymbol]]], Microsoft.FSharp.Core.FSharpOption`1[System.Tuple`2[FSharp.Compiler.Text.Position,Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.EditorServices.CompletionContext]]]) +FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults: FSharp.Compiler.EditorServices.DeclarationListInfo GetDeclarationListInfo(Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.CodeAnalysis.FSharpParseFileResults], Int32, System.String, FSharp.Compiler.EditorServices.PartialLongName, Microsoft.FSharp.Core.FSharpOption`1[Microsoft.FSharp.Core.FSharpFunc`2[Microsoft.FSharp.Core.Unit,Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.EditorServices.AssemblySymbol]]], Microsoft.FSharp.Core.FSharpOption`1[System.Tuple`2[FSharp.Compiler.Text.Position,Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.EditorServices.CompletionContext]]], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean]) FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults: FSharp.Compiler.EditorServices.FindDeclResult GetDeclarationLocation(Int32, Int32, System.String, Microsoft.FSharp.Collections.FSharpList`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean]) FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults: FSharp.Compiler.EditorServices.MethodGroup GetMethods(Int32, Int32, System.String, Microsoft.FSharp.Core.FSharpOption`1[Microsoft.FSharp.Collections.FSharpList`1[System.String]]) FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults: FSharp.Compiler.EditorServices.SemanticClassificationItem[] GetSemanticClassification(Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.Text.Range]) @@ -2009,7 +2009,7 @@ FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults: FSharp.Compiler.Symbols.FSh FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults: FSharp.Compiler.Symbols.FSharpOpenDeclaration[] get_OpenDeclarations() FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults: FSharp.Compiler.Text.Range[] GetFormatSpecifierLocations() FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults: Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.CodeAnalysis.FSharpSymbolUse] GetSymbolUsesAtLocation(Int32, Int32, System.String, Microsoft.FSharp.Collections.FSharpList`1[System.String]) -FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults: Microsoft.FSharp.Collections.FSharpList`1[Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.CodeAnalysis.FSharpSymbolUse]] GetDeclarationListSymbols(Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.CodeAnalysis.FSharpParseFileResults], Int32, System.String, FSharp.Compiler.EditorServices.PartialLongName, Microsoft.FSharp.Core.FSharpOption`1[Microsoft.FSharp.Core.FSharpFunc`2[Microsoft.FSharp.Core.Unit,Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.EditorServices.AssemblySymbol]]]) +FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults: Microsoft.FSharp.Collections.FSharpList`1[Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.CodeAnalysis.FSharpSymbolUse]] GetDeclarationListSymbols(Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.CodeAnalysis.FSharpParseFileResults], Int32, System.String, FSharp.Compiler.EditorServices.PartialLongName, Microsoft.FSharp.Core.FSharpOption`1[Microsoft.FSharp.Core.FSharpFunc`2[Microsoft.FSharp.Core.Unit,Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.EditorServices.AssemblySymbol]]], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean]) FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults: Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.CodeAnalysis.FSharpSymbolUse] GetSymbolUseAtLocation(Int32, Int32, System.String, Microsoft.FSharp.Collections.FSharpList`1[System.String]) FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults: Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.Symbols.FSharpDisplayContext] GetDisplayContextForPos(FSharp.Compiler.Text.Position) FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults: Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.Symbols.FSharpImplementationFileContents] ImplementationFile @@ -2066,9 +2066,9 @@ FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.FSharpParseFileResults,FSharp.Compiler.CodeAnalysis.FSharpCheckFileAnswer]] ParseAndCheckFileInProject(System.String, FSharpProjectSnapshot, Microsoft.FSharp.Core.FSharpOption`1[System.String]) FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.FSharpParseFileResults,FSharp.Compiler.CodeAnalysis.FSharpCheckFileAnswer]] ParseAndCheckFileInProject(System.String, Int32, FSharp.Compiler.Text.ISourceText, FSharp.Compiler.CodeAnalysis.FSharpProjectOptions, Microsoft.FSharp.Core.FSharpOption`1[System.String]) FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.FSharpParseFileResults,FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults]] GetBackgroundCheckResultsForFileInProject(System.String, FSharp.Compiler.CodeAnalysis.FSharpProjectOptions, Microsoft.FSharp.Core.FSharpOption`1[System.String]) -FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.FSharpProjectOptions,Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.Diagnostics.FSharpDiagnostic]]] GetProjectOptionsFromScript(System.String, FSharp.Compiler.Text.ISourceText, Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.DateTime], Microsoft.FSharp.Core.FSharpOption`1[System.String[]], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.Int64], Microsoft.FSharp.Core.FSharpOption`1[System.String]) -FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.ProjectSnapshot+FSharpProjectSnapshot,Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.Diagnostics.FSharpDiagnostic]]] GetProjectSnapshotFromScript(System.String, FSharp.Compiler.Text.ISourceTextNew, Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.CodeAnalysis.DocumentSource], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.DateTime], Microsoft.FSharp.Core.FSharpOption`1[System.String[]], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.Int64], Microsoft.FSharp.Core.FSharpOption`1[System.String]) -FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.Diagnostics.FSharpDiagnostic[],System.Int32]] Compile(System.String[], Microsoft.FSharp.Core.FSharpOption`1[System.String]) +FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.FSharpProjectOptions,Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.Diagnostics.FSharpDiagnostic]]] GetProjectOptionsFromScript(System.String, FSharp.Compiler.Text.ISourceText, Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.Text.Position], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.DateTime], Microsoft.FSharp.Core.FSharpOption`1[System.String[]], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.Int64], Microsoft.FSharp.Core.FSharpOption`1[System.String]) +FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.ProjectSnapshot+FSharpProjectSnapshot,Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.Diagnostics.FSharpDiagnostic]]] GetProjectSnapshotFromScript(System.String, FSharp.Compiler.Text.ISourceTextNew, Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.Text.Position], Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.CodeAnalysis.DocumentSource], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.DateTime], Microsoft.FSharp.Core.FSharpOption`1[System.String[]], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.Int64], Microsoft.FSharp.Core.FSharpOption`1[System.String]) +FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.Diagnostics.FSharpDiagnostic[],Microsoft.FSharp.Core.FSharpOption`1[System.Exception]]] Compile(System.String[], Microsoft.FSharp.Core.FSharpOption`1[System.String]) FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.Text.Range,FSharp.Compiler.Text.Range][]] MatchBraces(System.String, FSharp.Compiler.Text.ISourceText, FSharp.Compiler.CodeAnalysis.FSharpParsingOptions, Microsoft.FSharp.Core.FSharpOption`1[System.String]) FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.Text.Range,FSharp.Compiler.Text.Range][]] MatchBraces(System.String, System.String, FSharp.Compiler.CodeAnalysis.FSharpProjectOptions, Microsoft.FSharp.Core.FSharpOption`1[System.String]) FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.IEvent`2[Microsoft.FSharp.Control.FSharpHandler`1[FSharp.Compiler.CodeAnalysis.FSharpProjectOptions],FSharp.Compiler.CodeAnalysis.FSharpProjectOptions] ProjectChecked @@ -2600,12 +2600,20 @@ FSharp.Compiler.Diagnostics.ExtendedData+DiagnosticContextInfo: Int32 GetHashCod FSharp.Compiler.Diagnostics.ExtendedData+DiagnosticContextInfo: Int32 Tag FSharp.Compiler.Diagnostics.ExtendedData+DiagnosticContextInfo: Int32 get_Tag() FSharp.Compiler.Diagnostics.ExtendedData+DiagnosticContextInfo: System.String ToString() +FSharp.Compiler.Diagnostics.ExtendedData+ExperimentalExtendedData: Microsoft.FSharp.Core.FSharpOption`1[System.String] DiagnosticId +FSharp.Compiler.Diagnostics.ExtendedData+ExperimentalExtendedData: Microsoft.FSharp.Core.FSharpOption`1[System.String] UrlFormat +FSharp.Compiler.Diagnostics.ExtendedData+ExperimentalExtendedData: Microsoft.FSharp.Core.FSharpOption`1[System.String] get_DiagnosticId() +FSharp.Compiler.Diagnostics.ExtendedData+ExperimentalExtendedData: Microsoft.FSharp.Core.FSharpOption`1[System.String] get_UrlFormat() FSharp.Compiler.Diagnostics.ExtendedData+ExpressionIsAFunctionExtendedData: FSharp.Compiler.Symbols.FSharpType ActualType FSharp.Compiler.Diagnostics.ExtendedData+ExpressionIsAFunctionExtendedData: FSharp.Compiler.Symbols.FSharpType get_ActualType() FSharp.Compiler.Diagnostics.ExtendedData+FieldNotContainedDiagnosticExtendedData: FSharp.Compiler.Symbols.FSharpField ImplementationField FSharp.Compiler.Diagnostics.ExtendedData+FieldNotContainedDiagnosticExtendedData: FSharp.Compiler.Symbols.FSharpField SignatureField FSharp.Compiler.Diagnostics.ExtendedData+FieldNotContainedDiagnosticExtendedData: FSharp.Compiler.Symbols.FSharpField get_ImplementationField() FSharp.Compiler.Diagnostics.ExtendedData+FieldNotContainedDiagnosticExtendedData: FSharp.Compiler.Symbols.FSharpField get_SignatureField() +FSharp.Compiler.Diagnostics.ExtendedData+ObsoleteDiagnosticExtendedData: Microsoft.FSharp.Core.FSharpOption`1[System.String] DiagnosticId +FSharp.Compiler.Diagnostics.ExtendedData+ObsoleteDiagnosticExtendedData: Microsoft.FSharp.Core.FSharpOption`1[System.String] UrlFormat +FSharp.Compiler.Diagnostics.ExtendedData+ObsoleteDiagnosticExtendedData: Microsoft.FSharp.Core.FSharpOption`1[System.String] get_DiagnosticId() +FSharp.Compiler.Diagnostics.ExtendedData+ObsoleteDiagnosticExtendedData: Microsoft.FSharp.Core.FSharpOption`1[System.String] get_UrlFormat() FSharp.Compiler.Diagnostics.ExtendedData+TypeMismatchDiagnosticExtendedData: DiagnosticContextInfo ContextInfo FSharp.Compiler.Diagnostics.ExtendedData+TypeMismatchDiagnosticExtendedData: DiagnosticContextInfo get_ContextInfo() FSharp.Compiler.Diagnostics.ExtendedData+TypeMismatchDiagnosticExtendedData: FSharp.Compiler.Symbols.FSharpDisplayContext DisplayContext @@ -2621,6 +2629,7 @@ FSharp.Compiler.Diagnostics.ExtendedData+ValueNotContainedDiagnosticExtendedData FSharp.Compiler.Diagnostics.ExtendedData: FSharp.Compiler.Diagnostics.ExtendedData+ArgumentsInSigAndImplMismatchExtendedData FSharp.Compiler.Diagnostics.ExtendedData: FSharp.Compiler.Diagnostics.ExtendedData+DefinitionsInSigAndImplNotCompatibleAbbreviationsDifferExtendedData FSharp.Compiler.Diagnostics.ExtendedData: FSharp.Compiler.Diagnostics.ExtendedData+DiagnosticContextInfo +FSharp.Compiler.Diagnostics.ExtendedData: FSharp.Compiler.Diagnostics.ExtendedData+ExperimentalExtendedData FSharp.Compiler.Diagnostics.ExtendedData: FSharp.Compiler.Diagnostics.ExtendedData+ExpressionIsAFunctionExtendedData FSharp.Compiler.Diagnostics.ExtendedData: FSharp.Compiler.Diagnostics.ExtendedData+FieldNotContainedDiagnosticExtendedData FSharp.Compiler.Diagnostics.ExtendedData: FSharp.Compiler.Diagnostics.ExtendedData+IFSharpDiagnosticExtendedData diff --git a/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.release.bsl b/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.release.bsl index 6c8423483a0..49cb9d141ef 100644 --- a/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.release.bsl +++ b/tests/FSharp.Compiler.Service.Tests/FSharp.Compiler.Service.SurfaceArea.netstandard20.release.bsl @@ -2066,8 +2066,8 @@ FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.FSharpParseFileResults,FSharp.Compiler.CodeAnalysis.FSharpCheckFileAnswer]] ParseAndCheckFileInProject(System.String, FSharpProjectSnapshot, Microsoft.FSharp.Core.FSharpOption`1[System.String]) FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.FSharpParseFileResults,FSharp.Compiler.CodeAnalysis.FSharpCheckFileAnswer]] ParseAndCheckFileInProject(System.String, Int32, FSharp.Compiler.Text.ISourceText, FSharp.Compiler.CodeAnalysis.FSharpProjectOptions, Microsoft.FSharp.Core.FSharpOption`1[System.String]) FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.FSharpParseFileResults,FSharp.Compiler.CodeAnalysis.FSharpCheckFileResults]] GetBackgroundCheckResultsForFileInProject(System.String, FSharp.Compiler.CodeAnalysis.FSharpProjectOptions, Microsoft.FSharp.Core.FSharpOption`1[System.String]) -FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.FSharpProjectOptions,Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.Diagnostics.FSharpDiagnostic]]] GetProjectOptionsFromScript(System.String, FSharp.Compiler.Text.ISourceText, Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.DateTime], Microsoft.FSharp.Core.FSharpOption`1[System.String[]], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.Int64], Microsoft.FSharp.Core.FSharpOption`1[System.String]) -FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.ProjectSnapshot+FSharpProjectSnapshot,Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.Diagnostics.FSharpDiagnostic]]] GetProjectSnapshotFromScript(System.String, FSharp.Compiler.Text.ISourceTextNew, Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.CodeAnalysis.DocumentSource], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.DateTime], Microsoft.FSharp.Core.FSharpOption`1[System.String[]], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.Int64], Microsoft.FSharp.Core.FSharpOption`1[System.String]) +FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.FSharpProjectOptions,Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.Diagnostics.FSharpDiagnostic]]] GetProjectOptionsFromScript(System.String, FSharp.Compiler.Text.ISourceText, Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.Text.Position], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.DateTime], Microsoft.FSharp.Core.FSharpOption`1[System.String[]], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.Int64], Microsoft.FSharp.Core.FSharpOption`1[System.String]) +FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.CodeAnalysis.ProjectSnapshot+FSharpProjectSnapshot,Microsoft.FSharp.Collections.FSharpList`1[FSharp.Compiler.Diagnostics.FSharpDiagnostic]]] GetProjectSnapshotFromScript(System.String, FSharp.Compiler.Text.ISourceTextNew, Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.Text.Position], Microsoft.FSharp.Core.FSharpOption`1[FSharp.Compiler.CodeAnalysis.DocumentSource], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.DateTime], Microsoft.FSharp.Core.FSharpOption`1[System.String[]], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.Boolean], Microsoft.FSharp.Core.FSharpOption`1[System.String], Microsoft.FSharp.Core.FSharpOption`1[System.Int64], Microsoft.FSharp.Core.FSharpOption`1[System.String]) FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.Diagnostics.FSharpDiagnostic[],System.Int32]] Compile(System.String[], Microsoft.FSharp.Core.FSharpOption`1[System.String]) FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.Text.Range,FSharp.Compiler.Text.Range][]] MatchBraces(System.String, FSharp.Compiler.Text.ISourceText, FSharp.Compiler.CodeAnalysis.FSharpParsingOptions, Microsoft.FSharp.Core.FSharpOption`1[System.String]) FSharp.Compiler.CodeAnalysis.FSharpChecker: Microsoft.FSharp.Control.FSharpAsync`1[System.Tuple`2[FSharp.Compiler.Text.Range,FSharp.Compiler.Text.Range][]] MatchBraces(System.String, System.String, FSharp.Compiler.CodeAnalysis.FSharpProjectOptions, Microsoft.FSharp.Core.FSharpOption`1[System.String]) diff --git a/tests/fsharp/core/printing/output.1000.stdout.bsl b/tests/fsharp/core/printing/output.1000.stdout.bsl index ddde159fd6b..f2c87c488b9 100644 --- a/tests/fsharp/core/printing/output.1000.stdout.bsl +++ b/tests/fsharp/core/printing/output.1000.stdout.bsl @@ -2765,7 +2765,7 @@ val ShortName: string = "hi" > val list2: int list = [1] module FSI_0317. - A8a951db8294f99e95ae1d276a7ddaefd93d1548e6bf749bdeae55d2649682b3 + Ee54a783c1b4e3fe8e3c42c5a10c384669fed24c7ffbdacfc9698816f925f337 {"ImmutableField0":6} type R1 = diff --git a/tests/fsharp/core/printing/output.200.stdout.bsl b/tests/fsharp/core/printing/output.200.stdout.bsl index 89f13b2b05f..1f0db74ff6e 100644 --- a/tests/fsharp/core/printing/output.200.stdout.bsl +++ b/tests/fsharp/core/printing/output.200.stdout.bsl @@ -2010,7 +2010,7 @@ val ShortName: string = "hi" > val list2: int list = [1] module FSI_0317. - A8a951db8294f99e95ae1d276a7ddaefd93d1548e6bf749bdeae55d2649682b3 + Ee54a783c1b4e3fe8e3c42c5a10c384669fed24c7ffbdacfc9698816f925f337 {"ImmutableField0":6} type R1 = diff --git a/tests/fsharp/core/printing/output.multiemit.stdout.bsl b/tests/fsharp/core/printing/output.multiemit.stdout.bsl index ceb7d3d69a3..9894be5ced6 100644 --- a/tests/fsharp/core/printing/output.multiemit.stdout.bsl +++ b/tests/fsharp/core/printing/output.multiemit.stdout.bsl @@ -6312,7 +6312,7 @@ val ShortName: string = "hi" > val list2: int list = [1] module FSI_0316. - A8a951db8294f99e95ae1d276a7ddaefd93d1548e6bf749bdeae55d2649682b3 + Ee54a783c1b4e3fe8e3c42c5a10c384669fed24c7ffbdacfc9698816f925f337 {"ImmutableField0":6} type R1 = diff --git a/tests/fsharp/core/printing/output.off.stdout.bsl b/tests/fsharp/core/printing/output.off.stdout.bsl index 64444ac405b..5c641e28b3d 100644 --- a/tests/fsharp/core/printing/output.off.stdout.bsl +++ b/tests/fsharp/core/printing/output.off.stdout.bsl @@ -1779,7 +1779,7 @@ val ShortName: string = "hi" > val list2: int list module FSI_0317. - A8a951db8294f99e95ae1d276a7ddaefd93d1548e6bf749bdeae55d2649682b3 + Ee54a783c1b4e3fe8e3c42c5a10c384669fed24c7ffbdacfc9698816f925f337 {"ImmutableField0":6} type R1 = diff --git a/tests/fsharp/core/printing/output.stdout.bsl b/tests/fsharp/core/printing/output.stdout.bsl index ceb7d3d69a3..9894be5ced6 100644 --- a/tests/fsharp/core/printing/output.stdout.bsl +++ b/tests/fsharp/core/printing/output.stdout.bsl @@ -6312,7 +6312,7 @@ val ShortName: string = "hi" > val list2: int list = [1] module FSI_0316. - A8a951db8294f99e95ae1d276a7ddaefd93d1548e6bf749bdeae55d2649682b3 + Ee54a783c1b4e3fe8e3c42c5a10c384669fed24c7ffbdacfc9698816f925f337 {"ImmutableField0":6} type R1 = diff --git a/vsintegration/src/FSharp.Editor/Common/Extensions.fs b/vsintegration/src/FSharp.Editor/Common/Extensions.fs index b0eb7305713..eb530bfffaa 100644 --- a/vsintegration/src/FSharp.Editor/Common/Extensions.fs +++ b/vsintegration/src/FSharp.Editor/Common/Extensions.fs @@ -7,9 +7,15 @@ open System open System.IO open System.Collections.Immutable open System.Collections.Generic +open System.Runtime.InteropServices open System.Threading open System.Threading.Tasks +open Microsoft.VisualStudio +open Microsoft.VisualStudio.Shell +open Microsoft.VisualStudio.Shell.Interop +open Microsoft.VisualStudio.TextManager.Interop + open Microsoft.CodeAnalysis open Microsoft.CodeAnalysis.Text open Microsoft.CodeAnalysis.Host @@ -19,6 +25,10 @@ open FSharp.Compiler.Syntax open FSharp.Compiler.Text open Microsoft.VisualStudio.FSharp.Editor +open Microsoft.VisualStudio.Editor +open Microsoft.VisualStudio.Text.Editor +open Microsoft.VisualStudio +open Microsoft.VisualStudio.OLE.Interop type private FSharpGlyph = FSharp.Compiler.EditorServices.FSharpGlyph type private FSharpRoslynGlyph = Microsoft.CodeAnalysis.ExternalAccess.FSharp.FSharpGlyph @@ -59,6 +69,62 @@ type Project with member this.IsFSharp = this.Language = LanguageNames.FSharp +type TextViewEventsHandler + ( + onChangeCaretHandler: (IVsTextView * int * int -> unit) option, + onKillFocus: (IVsTextView -> unit) option, + onSetFocus: (IVsTextView -> unit) option + ) = + interface IVsTextViewEvents with + member this.OnChangeCaretLine(view: IVsTextView, newline: int, oldline: int) = + onChangeCaretHandler + |> Option.iter (fun handler -> handler (view, newline, oldline)) + + member this.OnChangeScrollInfo + ( + _view: IVsTextView, + _iBar: int, + _iMinUnit: int, + _iMaxUnits: int, + _iVisibleUnits: int, + _iFirstVisibleUnit: int + ) = + () + + member this.OnKillFocus(view: IVsTextView) = + onKillFocus |> Option.iter (fun handler -> handler (view)) + + member this.OnSetBuffer(_view: IVsTextView, _buffer: IVsTextLines) = () + + member this.OnSetFocus(view: IVsTextView) = + onSetFocus |> Option.iter (fun handler -> handler (view)) + +type ConnectionPointSubscription = System.IDisposable option + +// Usage example: +// If a handler is None, to not handle that event +// let subscription = subscribeToTextViewEvents (textView, onChangeCaretHandler, onKillFocus, onSetFocus) +// Unsubscribe using subscription.Dispose() +let subscribeToTextViewEvents (textView: IVsTextView, onChangeCaretHandler, onKillFocus, onSetFocus) : ConnectionPointSubscription = + let handler = TextViewEventsHandler(onChangeCaretHandler, onKillFocus, onSetFocus) + + match textView with + | :? IConnectionPointContainer as cpContainer -> + let riid = typeof.GUID + let mutable cookie = 0u + + match cpContainer.FindConnectionPoint(ref riid) with + | null -> None + | cp -> + Some( + cp.Advise(handler, &cookie) + + { new IDisposable with + member _.Dispose() = cp.Unadvise(cookie) + } + ) + | _ -> None + type Document with member this.TryGetLanguageService<'T when 'T :> ILanguageService>() = @@ -69,6 +135,32 @@ type Document with | null -> None | languageServices -> languageServices.GetService<'T>() |> Some + member this.TryGetIVsTextView() : IVsTextView option = + match ServiceProvider.GlobalProvider.GetService(typeof) with + | :? IVsTextManager as textManager -> + // Grab IVsRunningDocumentTable + match ServiceProvider.GlobalProvider.GetService(typeof) with + | :? IVsRunningDocumentTable as rdt -> + match rdt.FindAndLockDocument(uint32 _VSRDTFLAGS.RDT_NoLock, this.FilePath) with + | hr, _, _, docData, _ when ErrorHandler.Succeeded(hr) && docData <> IntPtr.Zero -> + match Marshal.GetObjectForIUnknown docData with + | :? IVsTextBuffer as ivsTextBuffer -> + match textManager.GetActiveView(0, ivsTextBuffer) with + | hr, vsTextView when ErrorHandler.Succeeded(hr) -> Some vsTextView + | _ -> None + | _ -> None + | _ -> None + | _ -> None + | _ -> None + + member this.TryGetTextViewAndCaretPos() : (IVsTextView * Position) option = + match this.TryGetIVsTextView() with + | Some textView -> + match textView.GetCaretPos() with + | hr, line, column when ErrorHandler.Succeeded(hr) -> Some(textView, Position.fromZ line column) + | _ -> None + | None -> None + member this.IsFSharpScript = isScriptFile this.FilePath member this.IsFSharpSignatureFile = isSignatureFile this.FilePath diff --git a/vsintegration/src/FSharp.Editor/LanguageService/FSharpProjectOptionsManager.fs b/vsintegration/src/FSharp.Editor/LanguageService/FSharpProjectOptionsManager.fs index c51db3021cf..d61ba717b4d 100644 --- a/vsintegration/src/FSharp.Editor/LanguageService/FSharpProjectOptionsManager.fs +++ b/vsintegration/src/FSharp.Editor/LanguageService/FSharpProjectOptionsManager.fs @@ -16,6 +16,13 @@ open System.Threading open Microsoft.VisualStudio.FSharp.Interactive.Session open System.Runtime.CompilerServices open CancellableTasks +open Microsoft.VisualStudio.FSharp.Editor.Extensions +open System.Windows +open Microsoft.VisualStudio +open FSharp.Compiler.Text +open Microsoft.VisualStudio.TextManager.Interop + +#nowarn "57" [] module private FSharpProjectOptionsHelpers = @@ -118,7 +125,7 @@ type private FSharpProjectOptionsReactor(checker: FSharpChecker) = ConcurrentDictionary() let singleFileCache = - ConcurrentDictionary() + ConcurrentDictionary() // This is used to not constantly emit the same compilation. let weakPEReferences = ConditionalWeakTable() @@ -193,20 +200,36 @@ type private FSharpProjectOptionsReactor(checker: FSharpChecker) = cancellableTask { let! ct = CancellableTask.getCancellationToken () let! fileStamp = document.GetTextVersionAsync(ct) + let textViewAndCaret () : (IVsTextView * Position) option = document.TryGetTextViewAndCaretPos() match singleFileCache.TryGetValue(document.Id) with | false, _ -> let! sourceText = document.GetTextAsync(ct) - let! scriptProjectOptions, _ = - checker.GetProjectOptionsFromScript( - document.FilePath, - sourceText.ToFSharpSourceText(), - previewEnabled = SessionsProperties.fsiPreview, - assumeDotNetFramework = not SessionsProperties.fsiUseNetCore, - userOpName = userOpName - ) + let getProjectOptionsFromScript textViewAndCaret = + let caret = textViewAndCaret () + + match caret with + | None -> + checker.GetProjectOptionsFromScript( + document.FilePath, + sourceText.ToFSharpSourceText(), + previewEnabled = SessionsProperties.fsiPreview, + assumeDotNetFramework = not SessionsProperties.fsiUseNetCore, + userOpName = userOpName + ) + + | Some(_, caret) -> + checker.GetProjectOptionsFromScript( + document.FilePath, + sourceText.ToFSharpSourceText(), + caret, + previewEnabled = SessionsProperties.fsiPreview, + assumeDotNetFramework = not SessionsProperties.fsiUseNetCore, + userOpName = userOpName + ) + let! scriptProjectOptions, _ = getProjectOptionsFromScript textViewAndCaret let project = document.Project let otherOptions = @@ -243,13 +266,46 @@ type private FSharpProjectOptionsReactor(checker: FSharpChecker) = let parsingOptions, _ = checker.GetParsingOptionsFromProjectOptions(projectOptions) - singleFileCache.[document.Id] <- (document.Project, fileStamp, parsingOptions, projectOptions) + let updateProjectOptions () = + async { + let! scriptProjectOptions, _ = getProjectOptionsFromScript textViewAndCaret + + checker.NotifyFileChanged(document.FilePath, scriptProjectOptions) + |> Async.Start + } + |> Async.Start + + let onChangeCaretHandler (_, _newline: int, _oldline: int) = updateProjectOptions () + let onKillFocus (_) = updateProjectOptions () + let onSetFocus (_) = updateProjectOptions () + + let addToCacheAndSubscribe value = + match value with + | projectId, fileStamp, parsingOptions, projectOptions, _ -> + let subscription = + match textViewAndCaret () with + | Some(textView, _) -> + subscribeToTextViewEvents (textView, (Some onChangeCaretHandler), (Some onKillFocus), (Some onSetFocus)) + | None -> None + + (projectId, fileStamp, parsingOptions, projectOptions, subscription) + + singleFileCache.AddOrUpdate( + document.Id, // The key to the cache + (fun _ value -> addToCacheAndSubscribe value), // Function to add the cached value if the key does not exist + (fun _ _ value -> value), // Function to update the value if the key exists + (document.Project, fileStamp, parsingOptions, projectOptions, None) // The value to add or update + ) + |> ignore return ValueSome(parsingOptions, projectOptions) - | true, (oldProject, oldFileStamp, parsingOptions, projectOptions) -> + | true, (oldProject, oldFileStamp, parsingOptions, projectOptions, _) -> if fileStamp <> oldFileStamp || isProjectInvalidated document.Project oldProject ct then - singleFileCache.TryRemove(document.Id) |> ignore + match singleFileCache.TryRemove(document.Id) with + | true, (_, _, _, _, Some subscription) -> subscription.Dispose() + | _ -> () + return! tryComputeOptionsBySingleScriptOrFile document userOpName else return ValueSome(parsingOptions, projectOptions) @@ -460,9 +516,10 @@ type private FSharpProjectOptionsReactor(checker: FSharpChecker) = legacyProjectSites.TryRemove(projectId) |> ignore | FSharpProjectOptionsMessage.ClearSingleFileOptionsCache(documentId) -> match singleFileCache.TryRemove(documentId) with - | true, (_, _, _, projectOptions) -> + | true, (_, _, _, projectOptions, subscription) -> lastSuccessfulCompilations.TryRemove(documentId.ProjectId) |> ignore checker.ClearCache([ projectOptions ]) + subscription |> Option.iter (fun handler -> handler.Dispose()) | _ -> () } diff --git a/vsintegration/src/FSharp.LanguageService/BackgroundRequests.fs b/vsintegration/src/FSharp.LanguageService/BackgroundRequests.fs index e22e2b5011c..0bf6d7354a3 100644 --- a/vsintegration/src/FSharp.LanguageService/BackgroundRequests.fs +++ b/vsintegration/src/FSharp.LanguageService/BackgroundRequests.fs @@ -98,7 +98,7 @@ type internal FSharpLanguageServiceBackgroundRequests_DEPRECATED lazy // This portion is executed on the language service thread let timestamp = if source=null then System.DateTime(2000,1,1) else source.OpenedTime // source is null in unit tests let checker = getInteractiveChecker() - let checkOptions, _diagnostics = checker.GetProjectOptionsFromScript(fileName, FSharp.Compiler.Text.SourceText.ofString sourceText, SessionsProperties.fsiPreview, timestamp, [| |]) |> Async.RunImmediate + let checkOptions, _diagnostics = checker.GetProjectOptionsFromScript(fileName, FSharp.Compiler.Text.SourceText.ofString sourceText, previewEnabled=SessionsProperties.fsiPreview, loadedTimeStamp=timestamp, otherFlags=[| |]) |> Async.RunImmediate let referencedProjectFileNames = [| |] let projectSite = ProjectSitesAndFiles.CreateProjectSiteForScript(fileName, referencedProjectFileNames, checkOptions) { ProjectSite = projectSite diff --git a/vsintegration/tests/Salsa/salsa.fs b/vsintegration/tests/Salsa/salsa.fs index 48fa3b431f3..f7540ec0bef 100644 --- a/vsintegration/tests/Salsa/salsa.fs +++ b/vsintegration/tests/Salsa/salsa.fs @@ -636,16 +636,10 @@ module internal Salsa = sb.Append(text+"\r\n") |> ignore Append "" Append " " -// The salsa layer does Configuration/Platform in a kind of hacky way -// Append " Debug" -// Append " AnyCPU" + Append " abcdefg" Append " bin\Debug\" if versionFile<>null then Append (sprintf " %s" versionFile) if otherFlags<>null then Append (sprintf " %s --resolutions" otherFlags) -// if targetFrameworkVersion<>null then -// Append(sprintf " true") -// Append(sprintf " %s" targetFrameworkVersion) -// else Append(sprintf " %s" "4.7.2") Append " " for disabledWarning in disabledWarnings do @@ -655,15 +649,7 @@ module internal Salsa = for define in defines do Append (sprintf " %s;" define) Append " " - Append " " -// Append " " -// Append " bin\Debug\" -// Append " " -// Append " " -// Append " bin\Release\" -// Append " " - Append " " for (reference,specificVersion) in references do @@ -1110,7 +1096,7 @@ module internal Salsa = member file.GetFileName() = fileName member file.GetProjectOptionsOfScript() = - project.Solution.Vs.LanguageService.FSharpChecker.GetProjectOptionsFromScript(fileName, FSharp.Compiler.Text.SourceText.ofString file.CombinedLines, false, System.DateTime(2000,1,1), [| |]) + project.Solution.Vs.LanguageService.FSharpChecker.GetProjectOptionsFromScript(fileName, FSharp.Compiler.Text.SourceText.ofString file.CombinedLines, previewEnabled=false, loadedTimeStamp=System.DateTime(2000,1,1), otherFlags=[| |]) |> Async.RunImmediate |> fst // drop diagnostics diff --git a/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj b/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj index 5510d3dcb9c..75e3007ac97 100644 --- a/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj +++ b/vsintegration/tests/UnitTests/VisualFSharp.UnitTests.fsproj @@ -13,6 +13,14 @@ false nunit true + + + 17.13.9 + 17.13.9 + 17.13.9 + 17.13.9 + 8.0.0 + @@ -144,4 +152,8 @@ + + + +