Refactor/build scripts (#3636) (#3639)

* Move to BullsEye

(cherry picked from commit 90f82a3)

* scripts now a netcoreapp project and seems to compile, used hacks to get typeproviders happy

(cherry picked from commit bfbb6e2)

* created combined tasks

(cherry picked from commit 3022fef)

* played around with optional targets

(cherry picked from commit ec6dc72)

* Versioning refactor and args are typed and not passed around with getBuildParam

(cherry picked from commit a815714)

* Remove FAKE4, selectively depend on handy FAKE5 modules

(cherry picked from commit 0dfc3d4)

* update proc and prefer exec over start

(cherry picked from commit c5eff44)

* Differ and Cluster to separate files

(cherry picked from commit 30c88fd)

* Differ and Cluster separate, move remaining paket items over

(cherry picked from commit c379a4b)

* remove paket, now relying on the tools as DotNetCliTools

(cherry picked from commit 4f46084)

* update cluster command, handover from yaml to environment config now more explicit

(cherry picked from commit d3ad1c8)

* Pull ILRepack in and make sure its avabilable in ouput path thanks to new GeneratePathProperty

(cherry picked from commit 189b256)

* nuget.exe now pulled in though NuGet

(cherry picked from commit bcc3148)

* try to remove sn.exe and resolve it from SDK

(cherry picked from commit 1254c8a)

* DocGenerator run by referencing dll

(cherry picked from commit 78f50b9)

* make sure canary target works locally again

(cherry picked from commit f8c110d)

* make sure release command works

(cherry picked from commit ffaa06b)

* remove sn.exe from source control

(cherry picked from commit 9c909d0)

* fix IDE build on linux

* update build.sh to use our build scripts location

* update latest mono and sdk on build server, local build works so wondering if updating CI does the trick

* update to latest mono and sdk fixed typeproviders yay, however took 16 minutes to pull down on azure pipelines. Rewrote global json handling without type providers

* update path to global.json, json.net does not like private record types

* Tests.Benchmarking needs conditional net461 TFM

* update casing issues in the build

* Working diff target

(cherry picked from commit 5e858c5)

NuGet.config

@@ -3,5 +3,7 @@
 	<packageSources>
 		<add key="nuget.org" value="https://www.nuget.org/api/v2/" />
 		<add key="Elastic Abstractions CI" value="https://ci.appveyor.com/nuget/elasticsearch-net-abstractions" />
+		<add key="AssemblyRewriter" value="https://ci.appveyor.com/nuget/assemblyrewriter" />
+		<add key="AssemblyDiffer" value="https://ci.appveyor.com/nuget/assemblydiffer" />
 	</packageSources>
 </configuration>

build.bat

@@ -1,6 +1,2 @@
 @echo off
-
-.paket\paket.bootstrapper.exe 
-IF EXIST paket.lock (.paket\paket.exe restore)
-IF NOT EXIST paket.lock (.paket\paket.exe install)
-"packages\build\FAKE\tools\Fake.exe" "build\\scripts\\Targets.fsx" "cmdline=%*"
+dotnet run --project build/scripts -- %*

build.sh

@@ -1,8 +1,3 @@
 #!/usr/bin/env bash
-FAKE="packages/build/FAKE/tools/FAKE.exe"
-BUILDSCRIPT="build/scripts/Targets.fsx"
-
-mono .paket/paket.bootstrapper.exe
-if [[ -f paket.lock ]]; then mono .paket/paket.exe restore; fi
-if [[ ! -f paket.lock ]]; then mono .paket/paket.exe install; fi
-mono $FAKE $BUILDSCRIPT "cmdline=$*" --fsiargs -d:MONO 
+set -euo pipefail
+dotnet run --project build/scripts -- "$@"

build/scripts/Benchmarking.fs

@@ -0,0 +1,25 @@
+namespace Scripts
+
+open System.IO
+open Commandline
+
+module Benchmarker =
+
+    let private testsProjectDirectory = Path.GetFullPath(Paths.TestsSource("Tests.Benchmarking"))
+
+    let Run args =
+
+        let url = match args.CommandArguments with | Benchmark b -> Some b.Endpoint | _ -> None
+        let username = match args.CommandArguments with | Benchmark b -> b.Username | _ -> None
+        let password = match args.CommandArguments with | Benchmark b -> b.Password | _ -> None
+        let runInteractive = not args.NonInteractive
+        let credentials  = (username, password)
+        let runCommandPrefix = "run -f netcoreapp2.1 -c Release"
+        let runCommand =
+            match (runInteractive, url, credentials) with
+            | (false, Some url, (Some username, Some password)) -> sprintf "%s -- --all \"%s\" \"%s\" \"%s\"" runCommandPrefix url username password
+            | (false, Some url, _) -> sprintf "%s -- --all \"%s\"" runCommandPrefix url
+            | (false, _, _) -> sprintf "%s -- --all" runCommandPrefix 
+            | (true, _, _) -> runCommandPrefix
+
+        Tooling.DotNet.ExecIn testsProjectDirectory [runCommand] |> ignore

build/scripts/Building.fs

@@ -0,0 +1,44 @@
+namespace Scripts
+
+open System 
+open System.IO
+
+open FSharp.Data
+
+open Paths
+open Projects
+open Tooling
+open Versioning
+open Fake.Core
+open Fake.IO
+open Commandline
+
+module Build =
+
+    let Restore() = DotNet.Exec ["restore"; Solution; ] |> ignore
+
+    let Compile args (ArtifactsVersion(version)) = 
+        let sourceLink = if args.DoSourceLink then "1" else ""
+        let props = 
+            [ 
+                "CurrentVersion", (version.Full.ToString());
+                "CurrentAssemblyVersion", (version.Assembly.ToString());
+                "CurrentAssemblyFileVersion", (version.AssemblyFile.ToString());
+                "DoSourceLink", sourceLink;
+                "FakeBuild", "1";
+                "OutputPathBaseDir", Path.GetFullPath Paths.BuildOutput;
+            ] 
+            |> List.map (fun (p,v) -> sprintf "%s=%s" p v)
+            |> String.concat ";"
+            |> sprintf "/property:%s"
+
+        DotNet.Exec ["build"; Solution; "-c"; "Release"; props] |> ignore
+
+
+    let Clean () =
+        printfn "Cleaning known output folders"
+        Shell.cleanDir Paths.BuildOutput
+        DotNet.Exec ["clean"; Solution; "-c"; "Release"] |> ignore 
+        DotNetProject.All |> Seq.iter(fun p -> Shell.cleanDir (Paths.BinFolder p.Name))
+
+

build/scripts/Cluster.fs

@@ -0,0 +1,37 @@
+namespace Scripts
+
+open System
+open System.IO
+open Fake.Core
+open Fake.IO
+open Commandline
+
+module Cluster =
+
+    let Run args =
+        let clusterName = Option.defaultValue "" <| match args.CommandArguments with | Cluster c -> Some c.Name | _ -> None
+        let clusterVersion = Option.defaultValue "" <|match args.CommandArguments with | Cluster c -> c.Version | _ -> None
+
+        let testsProjectDirectory = Path.Combine(Path.GetFullPath(Paths.Output("Tests.ClusterLauncher")), "netcoreapp2.1")
+        let tempDir = Path.Combine(Path.GetTempPath(), Path.GetRandomFileName());
+
+        let sourceDir = Paths.Source("Tests/Tests.Configuration");
+        let defaultYaml = Path.Combine(sourceDir, "tests.default.yaml");
+        let userYaml = Path.Combine(sourceDir, "tests.yaml");
+        let e f = File.Exists f;
+        match ((e userYaml), (e defaultYaml)) with
+        | (true, _) -> Environment.setEnvironVar "NEST_YAML_FILE" (Path.GetFullPath(userYaml))
+        | (_, true) -> Environment.setEnvironVar "NEST_YAML_FILE" (Path.GetFullPath(defaultYaml))
+        | _ -> failwithf "Expected to find a tests.default.yaml or tests.yaml in %s" sourceDir
+
+        printfn "%s" testsProjectDirectory
+
+        Shell.copyDir tempDir testsProjectDirectory (fun s -> true)
+
+        let command = sprintf "%s %s" clusterName clusterVersion
+        let timeout = TimeSpan.FromMinutes(120.)
+        let dll = Path.Combine(tempDir, "Tests.ClusterLauncher.dll");
+        Tooling.DotNet.ExecInWithTimeout tempDir [dll; command] timeout  |> ignore
+
+        Shell.deleteDir tempDir
+