diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml new file mode 100644 index 00000000..79686a38 --- /dev/null +++ b/.github/workflows/pr-validation.yml @@ -0,0 +1,25 @@ +name: Pull Request Validation for feature/k8se + +on: + push: + branches: [ feature/k8se ] + pull_request: + branches: [ feature/k8se ] + +jobs: + build: + + runs-on: windows-latest + + steps: + - uses: actions/checkout@v2 + - name: Setup .NET + uses: actions/setup-dotnet@v1 + with: + dotnet-version: 3.1.x + - name: Restore dependencies + run: dotnet restore + - name: Build + run: dotnet build --no-restore + - name: Test + run: dotnet test --no-build --verbosity normal \ No newline at end of file diff --git a/Common/Constants.cs b/Common/Constants.cs index 662e2010..67ccabe6 100644 --- a/Common/Constants.cs +++ b/Common/Constants.cs @@ -5,6 +5,20 @@ namespace Kudu { public static class Constants { + //Scan functionality files + public const string ScanLockFile = "scan.lock"; + public static string ScanStatusFile = "status.json"; + public static string ScanLogFile = "scan_log.log"; + public static string ScanFolderName = "Scan_"; + public static string MaxScans = "2"; + public static string ScanDir = "/home/site/wwwroot"; + public static string ScriptPath = "/custom_scripts/daily_scan_script.sh"; + public static string ScanCommand = ScriptPath+" "+ScanDir; + public static string ScanTimeOutMillSec = "1200000"; // 20 mins + public static string ScanManifest = "modified_times.json"; + public static string AggregrateScanResults = "aggregrate_scans.log"; + public static string TempScanFile = "temp_scan_monitor"; + public const string WebRoot = "wwwroot"; public const string MappedSite = "/_app"; public const string RepositoryPath = "repository"; @@ -84,6 +98,8 @@ public static TimeSpan MaxAllowedExecutionTime public const string LogicAppJson = "logicapp.json"; public const string LogicAppUrlKey = "LOGICAPP_URL"; + public const string AppSettingsRegex = "%.*?%"; + public const string SiteExtensionProvisioningStateCreated = "Created"; public const string SiteExtensionProvisioningStateAccepted = "Accepted"; public const string SiteExtensionProvisioningStateSucceeded = "Succeeded"; @@ -105,6 +121,8 @@ public static TimeSpan MaxAllowedExecutionTime public const string SiteAuthEncryptionKey = "WEBSITE_AUTH_ENCRYPTION_KEY"; public const string HttpHost = "HTTP_HOST"; public const string WebSiteSwapSlotName = "WEBSITE_SWAP_SLOTNAME"; + public const string AzureWebsiteInstanceId = "WEBSITE_INSTANCE_ID"; + public const string ContainerName = "CONTAINER_NAME"; public const string Function = "function"; public const string Functions = "functions"; @@ -116,6 +134,7 @@ public static TimeSpan MaxAllowedExecutionTime public const string FunctionsPortal = "FunctionsPortal"; public const string FunctionKeyNewFormat = "~0.7"; public const string FunctionRunTimeVersion = "FUNCTIONS_EXTENSION_VERSION"; + public const string ScmRunFromPackage = "SCM_RUN_FROM_PACKAGE"; public const string WebSiteSku = "WEBSITE_SKU"; public const string WebSiteElasticScaleEnabled = "WEBSITE_ELASTIC_SCALING_ENABLED"; public const string DynamicSku = "Dynamic"; @@ -124,11 +143,31 @@ public static TimeSpan MaxAllowedExecutionTime public const string HubName = "HubName"; public const string DurableTaskStorageConnection = "connection"; public const string DurableTaskStorageConnectionName = "azureStorageConnectionStringName"; + public const string DurableTaskSqlConnectionName = "connectionStringName"; + public const string DurableTaskStorageProvider = "storageProvider"; + public const string DurableTaskMicrosoftSqlProviderType = "mssql"; + public const string MicrosoftSqlScaler = "mssql"; + public const string AzureQueueScaler = "azure-queue"; public const string DurableTask = "durableTask"; + public const string WorkflowAppKind = "workflowApp"; + public const string WorkflowExtensionName = "workflow"; + public const string WorkflowSettingsName = "Settings"; public const string Extensions = "extensions"; public const string SitePackages = "SitePackages"; - public const string SiteVersionTxt = "siteversion.txt"; public const string PackageNameTxt = "packagename.txt"; - public const string KuduBuild = "1.0.0.5"; + public const string KuduBuild = "1.0.0.7"; + + public const string WebSSHReverseProxyPortEnvVar = "KUDU_WEBSSH_PORT"; + public const string WebSSHReverseProxyDefaultPort = "3000"; + + public const string LinuxLogEventStreamName = "MS_KUDU_LOGS"; + public const string WebSiteHomeStampName = "WEBSITE_HOME_STAMPNAME"; + public const string WebSiteStampDeploymentId = "WEBSITE_STAMP_DEPLOYMENT_ID"; + public const string IsK8SEEnvironment = "K8SE_BUILD_SERVICE"; + + public const string OneDeploy = "OneDeploy"; + public const string ArtifactStagingDirectoryName = "extracted"; + + public const string K8SEAppTypeDefault = "functionapp,kubernetes,linux"; } -} \ No newline at end of file +} diff --git a/Kudu.Console/JavaScriptSerializer.cs b/Kudu.Console/JavaScriptSerializer.cs new file mode 100644 index 00000000..548f9397 --- /dev/null +++ b/Kudu.Console/JavaScriptSerializer.cs @@ -0,0 +1,9 @@ +namespace Kudu.Console +{ + internal class JavaScriptSerializer + { + public JavaScriptSerializer() + { + } + } +} \ No newline at end of file diff --git a/Kudu.Console/Kudu.Console.csproj b/Kudu.Console/Kudu.Console.csproj index 3cfc0393..87788eac 100644 --- a/Kudu.Console/Kudu.Console.csproj +++ b/Kudu.Console/Kudu.Console.csproj @@ -1,12 +1,13 @@  - netcoreapp2.2 + netcoreapp3.1 true kudu + diff --git a/Kudu.Console/Program.cs b/Kudu.Console/Program.cs index c758592e..669edc2e 100644 --- a/Kudu.Console/Program.cs +++ b/Kudu.Console/Program.cs @@ -22,14 +22,23 @@ using Kudu.Core.Tracing; using System.Reflection; using XmlSettings; +using k8s; +using IRepository = Kudu.Core.SourceControl.IRepository; +using log4net; +using log4net.Config; namespace Kudu.Console { internal class Program { - + private static IEnvironment env; + private static IDeploymentSettingsManager settingsManager; + private static string appRoot; + private static int Main(string[] args) { + var logRepository = LogManager.GetRepository(Assembly.GetEntryAssembly()); + XmlConfigurator.Configure(logRepository, new FileInfo("log4net.config")); // Turn flag on in app.config to wait for debugger on launch if (ConfigurationManager.AppSettings["WaitForDebuggerOnStart"] == "true") { @@ -55,14 +64,14 @@ private static int Main(string[] args) System.Console.Error.NewLine = "\n"; System.Console.Out.NewLine = "\n"; - string appRoot = args[0]; + appRoot = args[0]; string wapTargets = args[1]; string deployer = args.Length == 2 ? null : args[2]; string requestId = System.Environment.GetEnvironmentVariable(Constants.RequestIdHeader); - IEnvironment env = GetEnvironment(appRoot, requestId); + env = GetEnvironment(appRoot, requestId); ISettings settings = new XmlSettings.Settings(GetSettingsPath(env)); - IDeploymentSettingsManager settingsManager = new DeploymentSettingsManager(settings); + settingsManager = new DeploymentSettingsManager(settings); // Setup the trace TraceLevel level = settingsManager.GetTraceLevel(); @@ -74,7 +83,7 @@ private static int Main(string[] args) string deploymentLockPath = Path.Combine(lockPath, Constants.DeploymentLockFile); IOperationLock deploymentLock = DeploymentLockFile.GetInstance(deploymentLockPath, traceFactory); - + if (deploymentLock.IsHeld) { return PerformDeploy(appRoot, wapTargets, deployer, lockPath, env, settingsManager, level, tracer, traceFactory, deploymentLock); @@ -117,7 +126,6 @@ private static int PerformDeploy( // Adjust repo path env.RepositoryPath = Path.Combine(env.SiteRootPath, settingsManager.GetRepositoryPath()); - string statusLockPath = Path.Combine(lockPath, Constants.StatusLockFile); string hooksLockPath = Path.Combine(lockPath, Constants.HooksLockFile); @@ -126,7 +134,7 @@ private static int PerformDeploy( IOperationLock hooksLock = new LockFile(hooksLockPath, traceFactory); IBuildPropertyProvider buildPropertyProvider = new BuildPropertyProvider(); - ISiteBuilderFactory builderFactory = new SiteBuilderFactory(buildPropertyProvider, env); + ISiteBuilderFactory builderFactory = new SiteBuilderFactory(buildPropertyProvider, env, null); var logger = new ConsoleLogger(); IRepository gitRepository; @@ -139,11 +147,16 @@ private static int PerformDeploy( gitRepository = new GitExeRepository(env, settingsManager, traceFactory); } + env.CurrId = gitRepository.GetChangeSet(settingsManager.GetBranch()).Id; + IServerConfiguration serverConfiguration = new ServerConfiguration(); + IAnalytics analytics = new Analytics(settingsManager, serverConfiguration, traceFactory); IWebHooksManager hooksManager = new WebHooksManager(tracer, env, hooksLock); + IDeploymentStatusManager deploymentStatusManager = new DeploymentStatusManager(env, analytics, statusLock); + IDeploymentManager deploymentManager = new DeploymentManager(builderFactory, env, traceFactory, @@ -152,7 +165,8 @@ private static int PerformDeploy( deploymentStatusManager, deploymentLock, GetLogger(env, level, logger), - hooksManager); + hooksManager, + null); // K8 todo var step = tracer.Step(XmlTracer.ExecutingExternalProcessTrace, new Dictionary { @@ -180,7 +194,7 @@ private static int PerformDeploy( { string branch = settingsManager.GetBranch(); ChangeSet changeSet = gitRepository.GetChangeSet(branch); - IDeploymentStatusFile statusFile = deploymentStatusManager.Open(changeSet.Id); + IDeploymentStatusFile statusFile = deploymentStatusManager.Open(changeSet.Id, env); if (statusFile != null && statusFile.Status == DeployStatus.Success) { PostDeploymentHelper.PerformAutoSwap(env.RequestId, @@ -191,11 +205,18 @@ private static int PerformDeploy( } catch (Exception e) { + System.Console.WriteLine(e.InnerException); tracer.TraceError(e); System.Console.Error.WriteLine(e.GetBaseException().Message); System.Console.Error.WriteLine(Resources.Log_DeploymentError); return 1; } + finally + { + System.Console.WriteLine("Deployment Logs : '"+ + env.AppBaseUrlPrefix+ "/newui/jsonviewer?view_url=/api/deployments/" + + gitRepository.GetChangeSet(settingsManager.GetBranch()).Id+"/log'"); + } } if (logger.HasErrors) @@ -250,6 +271,7 @@ private static string GetSettingsPath(IEnvironment environment) private static IEnvironment GetEnvironment(string siteRoot, string requestId) { string root = Path.GetFullPath(Path.Combine(siteRoot, "..")); + string appName = root.Replace("/home/apps/",""); // CORE TODO : test by setting SCM_REPOSITORY_PATH // REVIEW: this looks wrong because it ignores SCM_REPOSITORY_PATH @@ -267,12 +289,14 @@ private static IEnvironment GetEnvironment(string siteRoot, string requestId) } // CORE TODO Handing in a null IHttpContextAccessor (and KuduConsoleFullPath) again - return new Kudu.Core.Environment(root, + var env= new Kudu.Core.Environment(root, EnvironmentHelper.NormalizeBinPath(binPath), repositoryPath, requestId, Path.Combine(AppContext.BaseDirectory, "KuduConsole", "kudu.dll"), - null); + null, + appName); + return env; } } } \ No newline at end of file diff --git a/Kudu.Console/Properties/launchSettings.json b/Kudu.Console/Properties/launchSettings.json new file mode 100644 index 00000000..704a661f --- /dev/null +++ b/Kudu.Console/Properties/launchSettings.json @@ -0,0 +1,27 @@ +{ + "iisSettings": { + "windowsAuthentication": false, + "anonymousAuthentication": true, + "iisExpress": { + "applicationUrl": "http://localhost:7668/", + "sslPort": 0 + } + }, + "profiles": { + "IIS Express": { + "commandName": "IISExpress", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + }, + "Kudu.Console": { + "commandName": "Project", + "launchBrowser": true, + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + }, + "applicationUrl": "http://localhost:7669/" + } + } +} \ No newline at end of file diff --git a/Kudu.Contracts/Deployment/ArtifactType.cs b/Kudu.Contracts/Deployment/ArtifactType.cs new file mode 100644 index 00000000..a4050c5a --- /dev/null +++ b/Kudu.Contracts/Deployment/ArtifactType.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Kudu.Contracts.Deployment +{ + public enum ArtifactType + { + Unknown, + War, + Jar, + Ear, + Lib, + Static, + Startup, + Script, + Zip, + } +} diff --git a/Kudu.Contracts/Deployment/BuildMetadata.cs b/Kudu.Contracts/Deployment/BuildMetadata.cs new file mode 100644 index 00000000..08f52351 --- /dev/null +++ b/Kudu.Contracts/Deployment/BuildMetadata.cs @@ -0,0 +1,16 @@ +using Newtonsoft.Json; + +namespace Kudu.Contracts.Deployment +{ + public class BuildMetadata + { + [JsonProperty(PropertyName = "appName")] + public string AppName; + + [JsonProperty(PropertyName = "buildVersion")] + public string BuildVersion; + + [JsonProperty(PropertyName = "appSubPath")] + public string AppSubPath; + } +} diff --git a/Kudu.Contracts/Deployment/DeploymentInfoBase.cs b/Kudu.Contracts/Deployment/DeploymentInfoBase.cs index 012b5639..94ff1110 100644 --- a/Kudu.Contracts/Deployment/DeploymentInfoBase.cs +++ b/Kudu.Contracts/Deployment/DeploymentInfoBase.cs @@ -2,6 +2,9 @@ using Kudu.Core.SourceControl; using Kudu.Contracts.Tracing; using System.Threading.Tasks; +using System.Collections; +using System.Collections.Generic; +using Kudu.Contracts.Deployment; namespace Kudu.Core.Deployment { @@ -28,6 +31,8 @@ protected DeploymentInfoBase() public FetchDelegate Fetch { get; set; } public bool AllowDeploymentWhileScmDisabled { get; set; } + public IDictionary repositorySymlinks { get; set; } + // Optional. // Path of the directory to be deployed to. The path should be relative to the wwwroot directory. // Example: "webapps/ROOT" @@ -68,5 +73,37 @@ public bool IsValid() // won't update until after a process restart. Therefore, we copy the needed // files into a separate folders and run sync triggers from there. public string SyncFunctionsTriggersPath { get; set; } = null; + + // Used to set Publish Endpoint context + public bool ShouldBuildArtifact { get; set; } + + // Optional. + // Type of artifact being deployed. + public ArtifactType ArtifactType { get; set; } + + // Optional. + // By default, TargetSubDirectoryRelativePath specifies the directory to deploy to relative to /home/site/wwwroot. + // This property can be used to change the root from wwwroot to something else. + public string TargetRootPath { get; set; } + + // Allows the use of a deployment Id, to be tracked + public string DeploymentTrackingId { get; set; } = null; + + // Optional. + // Path of the directory to be deployed to. The path should be relative to the wwwroot directory. + // Example: "webapps/ROOT" + public string TargetSubDirectoryRelativePath { get; set; } + + // Optional. + // Specifies the name of the deployed artifact. + // Example: When deploying startup files, OneDeploy will set this to startup.cmd (or startup.sh) + public string TargetFileName { get; set; } + + // Specifies whether to touch the watched file (example web.config, web.xml, etc) after the deployment + public bool WatchedFileEnabled { get; set; } + + // Used to allow / disallow 'restart' on a per deployment basis, if needed. + // For example: OneDeploy allows clients to enable / disable 'restart'. + public bool RestartAllowed { get; set; } } } \ No newline at end of file diff --git a/Kudu.Contracts/Deployment/IDeploymentStatusManager.cs b/Kudu.Contracts/Deployment/IDeploymentStatusManager.cs index b2802534..f75be539 100644 --- a/Kudu.Contracts/Deployment/IDeploymentStatusManager.cs +++ b/Kudu.Contracts/Deployment/IDeploymentStatusManager.cs @@ -5,9 +5,9 @@ namespace Kudu.Core.Deployment { public interface IDeploymentStatusManager { - IDeploymentStatusFile Create(string id); - IDeploymentStatusFile Open(string id); - void Delete(string id); + IDeploymentStatusFile Create(string id, IEnvironment environment); + IDeploymentStatusFile Open(string id, IEnvironment environment); + void Delete(string id, IEnvironment environment); IOperationLock Lock { get; } diff --git a/Kudu.Contracts/IEnvironment.cs b/Kudu.Contracts/IEnvironment.cs index abf6df4b..cc546b3a 100644 --- a/Kudu.Contracts/IEnvironment.cs +++ b/Kudu.Contracts/IEnvironment.cs @@ -2,12 +2,13 @@ { public interface IEnvironment { + string CurrId { get; set; } string RootPath { get; } // e.g. / string SiteRootPath { get; } // e.g. /site string RepositoryPath { get; set; } // e.g. /site/repository string WebRootPath { get; } // e.g. /site/wwwroot - string DeploymentsPath { get; } // e.g. /site/deployments - string DeploymentToolsPath { get; } // e.g. /site/deployments/tools + string DeploymentsPath { get; } // e.g. /site/deployments + string DeploymentToolsPath { get; } // e.g. /site/deployments/tools string SiteExtensionSettingsPath { get; } // e.g. /site/siteextensions string DiagnosticsPath { get; } // e.g. /site/diagnostics string LocksPath { get; } // e.g. /site/locks @@ -30,5 +31,9 @@ public interface IEnvironment string RequestId { get; } // e.g. x-arr-log-id or x-ms-request-id header value string KuduConsoleFullPath { get; } // e.g. KuduConsole/kudu.dll string SitePackagesPath { get; } // e.g. /data/SitePackages + bool IsOnLinuxConsumption { get; } // e.g. True on Linux Consumption. False on App Service. + bool IsK8SEApp { get; } + string K8SEAppName { get; } + string K8SEAppType { get; } } } diff --git a/Kudu.Contracts/ILinuxConsumptionEnvironment.cs b/Kudu.Contracts/ILinuxConsumptionEnvironment.cs new file mode 100644 index 00000000..cfa9f62c --- /dev/null +++ b/Kudu.Contracts/ILinuxConsumptionEnvironment.cs @@ -0,0 +1,37 @@ +using System.Threading.Tasks; + +namespace Kudu.Contracts +{ + public interface ILinuxConsumptionEnvironment + { + /// + /// Gets a value indicating whether requests should be delayed. + /// + bool DelayRequestsEnabled { get; } + + Task DelayCompletionTask { get; } + + /// + /// Gets a value indicating whether the current environment is in standby mode. + /// + bool InStandbyMode { get; } + + /// + /// Flags that requests under this environment should be delayed. + /// + void DelayRequests(); + + /// + /// Flags that requests under this environment should be resumed. + /// + void ResumeRequests(); + + /// + /// Flags the current environment as ready and specialized. + /// This sets to "0" + /// and to "1" against + /// the current environment. + /// + void FlagAsSpecializedAndReady(); + } +} diff --git a/Kudu.Contracts/Kudu.Contracts.csproj b/Kudu.Contracts/Kudu.Contracts.csproj index 1ea09fef..c9e6f745 100644 --- a/Kudu.Contracts/Kudu.Contracts.csproj +++ b/Kudu.Contracts/Kudu.Contracts.csproj @@ -1,7 +1,7 @@  - netcoreapp2.2 + netcoreapp3.1 true diff --git a/Kudu.Contracts/Scan/IScanManager.cs b/Kudu.Contracts/Scan/IScanManager.cs new file mode 100644 index 00000000..e69cff6a --- /dev/null +++ b/Kudu.Contracts/Scan/IScanManager.cs @@ -0,0 +1,29 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +namespace Kudu.Contracts.Scan +{ + public interface IScanManager + { + Task StartScan( + String timeout, + String mainScanDirPath, + String id, + String host, + Boolean checkModified); + + Task GetScanStatus( + String scanId, + String mainScanDirPath); + + Task GetScanResultFile( + String scanId, + String mainScanDirPath); + + IEnumerable GetResults(String mainScanDir); + + void StopScan(String mainScanDirPath); + } +} + diff --git a/Kudu.Contracts/Scan/InfectedFileObject.cs b/Kudu.Contracts/Scan/InfectedFileObject.cs new file mode 100644 index 00000000..43ee66a1 --- /dev/null +++ b/Kudu.Contracts/Scan/InfectedFileObject.cs @@ -0,0 +1,22 @@ +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Kudu.Contracts.Scan +{ + public class InfectedFileObject + { + [JsonProperty(PropertyName = "name")] + public String Name { get; set; } + + [JsonProperty(PropertyName = "threat_detected")] + public String ThreatDetected { get; set; } + + public InfectedFileObject(string name, string threat) + { + this.Name = name; + this.ThreatDetected = threat; + } + } +} diff --git a/Kudu.Contracts/Scan/ScanDetail.cs b/Kudu.Contracts/Scan/ScanDetail.cs new file mode 100644 index 00000000..48ff6acd --- /dev/null +++ b/Kudu.Contracts/Scan/ScanDetail.cs @@ -0,0 +1,25 @@ +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Kudu.Contracts.Scan +{ + public class ScanDetail + { + [JsonProperty(PropertyName = "total_scanned")] + public String TotalScanned { get; set; } + + [JsonProperty(PropertyName = "total_infected")] + public String TotalInfected { get; set; } + + [JsonProperty(PropertyName = "time_taken")] + public String TimeTaken { get; set; } + + [JsonProperty(PropertyName = "safe_files")] + public List SafeFiles { get; set; } + + [JsonProperty(PropertyName = "infected_files")] + public List InfectedFiles { get; set; } + } +} diff --git a/Kudu.Contracts/Scan/ScanOverviewResult.cs b/Kudu.Contracts/Scan/ScanOverviewResult.cs new file mode 100644 index 00000000..287bcfcc --- /dev/null +++ b/Kudu.Contracts/Scan/ScanOverviewResult.cs @@ -0,0 +1,24 @@ +using Kudu.Contracts.Infrastructure; +using Newtonsoft.Json; +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Kudu.Contracts.Scan +{ + + public class ScanOverviewResult : INamedObject + { + [JsonProperty(PropertyName = "status_info")] + public ScanStatusResult Status { get; set; } + + [JsonProperty(PropertyName = "scan_results_url")] + public String ScanResultsUrl { get; set; } + + /* [JsonIgnore] + public DateTime ReceivedTime { get; set; }*/ + + [JsonIgnore] + [SuppressMessage("Microsoft.Design", "CA1033:InterfaceMethodsShouldBeCallableByChildTypes", Justification = "to provide ARM spceific name")] + string INamedObject.Name { get { return Status.Id; } } + } +} diff --git a/Kudu.Contracts/Scan/ScanReport.cs b/Kudu.Contracts/Scan/ScanReport.cs new file mode 100644 index 00000000..5567f292 --- /dev/null +++ b/Kudu.Contracts/Scan/ScanReport.cs @@ -0,0 +1,23 @@ +using Kudu.Contracts.Infrastructure; +using Newtonsoft.Json; +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Kudu.Contracts.Scan +{ + public class ScanReport : INamedObject + { + [JsonProperty(PropertyName = "report")] + public ScanDetail Report { get; set; } + + [JsonIgnore] + [SuppressMessage("Microsoft.Design", "CA1033:InterfaceMethodsShouldBeCallableByChildTypes", Justification = "to provide ARM spceific name")] + string INamedObject.Name { get { return Id; } } + + [JsonProperty(PropertyName = "timestamp")] + public DateTime Timestamp { get; set; } + + [JsonIgnore] + public String Id { get; set; } + } +} diff --git a/Kudu.Contracts/Scan/ScanRequestResult.cs b/Kudu.Contracts/Scan/ScanRequestResult.cs new file mode 100644 index 00000000..2fa0993b --- /dev/null +++ b/Kudu.Contracts/Scan/ScanRequestResult.cs @@ -0,0 +1,12 @@ +namespace Kudu.Contracts.Scan +{ + public enum ScanRequestResult + { + RunningAynschronously, + RanSynchronously, + Pending, + AsyncScanFailed, + NoFileModifications, + ScanAlreadyInProgress + } +} diff --git a/Kudu.Contracts/Scan/ScanStatus.cs b/Kudu.Contracts/Scan/ScanStatus.cs new file mode 100644 index 00000000..081ff1f2 --- /dev/null +++ b/Kudu.Contracts/Scan/ScanStatus.cs @@ -0,0 +1,12 @@ +namespace Kudu.Contracts.Scan +{ + public enum ScanStatus + { + Starting, + Executing, + Failed, + TimeoutFailure, + Success, + ForceStopped + } +} diff --git a/Kudu.Contracts/Scan/ScanStatusResult.cs b/Kudu.Contracts/Scan/ScanStatusResult.cs new file mode 100644 index 00000000..121a620f --- /dev/null +++ b/Kudu.Contracts/Scan/ScanStatusResult.cs @@ -0,0 +1,21 @@ +using Kudu.Contracts.Infrastructure; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using System.Diagnostics.CodeAnalysis; + +namespace Kudu.Contracts.Scan +{ + public class ScanStatusResult : INamedObject + { + [JsonProperty(PropertyName = "id")] + public string Id { get; set; } + + [JsonProperty(PropertyName = "status")] + [JsonConverter(typeof(StringEnumConverter))] + public ScanStatus Status { get; set; } + + [JsonIgnore] + [SuppressMessage("Microsoft.Design", "CA1033:InterfaceMethodsShouldBeCallableByChildTypes", Justification = "to provide ARM spceific name")] + string INamedObject.Name { get { return Id; } } + } +} diff --git a/Kudu.Contracts/Scan/ScanUrl.cs b/Kudu.Contracts/Scan/ScanUrl.cs new file mode 100644 index 00000000..111077ae --- /dev/null +++ b/Kudu.Contracts/Scan/ScanUrl.cs @@ -0,0 +1,34 @@ +using Kudu.Contracts.Infrastructure; +using Newtonsoft.Json; +using System; +using System.Diagnostics.CodeAnalysis; + +namespace Kudu.Contracts.Scan +{ + public class ScanUrl : INamedObject + { + [JsonProperty(PropertyName = "track_url")] + public String TrackingURL { get; set; } + + [JsonProperty(PropertyName = "result_url")] + public String ResultURL { get; set; } + + [JsonIgnore] + [SuppressMessage("Microsoft.Design", "CA1033:InterfaceMethodsShouldBeCallableByChildTypes", Justification = "to provide ARM spceific name")] + string INamedObject.Name { get { return Id; } } + + [JsonProperty(PropertyName = "id")] + public String Id { get; set; } + + [JsonProperty(PropertyName = "message")] + public String Message { get; set; } + + public ScanUrl(string trackingURL, string resultURL, string id, string msg) + { + TrackingURL = trackingURL; + ResultURL = resultURL; + Id = id; + Message = msg; + } + } +} diff --git a/Kudu.Contracts/Settings/DeploymentSettingsExtension.cs b/Kudu.Contracts/Settings/DeploymentSettingsExtension.cs index bd8c1138..4a467c90 100644 --- a/Kudu.Contracts/Settings/DeploymentSettingsExtension.cs +++ b/Kudu.Contracts/Settings/DeploymentSettingsExtension.cs @@ -2,13 +2,14 @@ using System.Diagnostics; using Kudu.Contracts.Infrastructure; using Kudu.Contracts.SourceControl; +using Kudu.Contracts.Tracing; namespace Kudu.Contracts.Settings { public static class DeploymentSettingsExtension { public static readonly TimeSpan DefaultCommandIdleTimeout = TimeSpan.FromMinutes(1); - public static readonly TimeSpan DefaultLogStreamTimeout = TimeSpan.FromMinutes(30); + public static readonly TimeSpan DefaultLogStreamTimeout = TimeSpan.FromMinutes(120); public static readonly TimeSpan DefaultWebJobsRestartTime = TimeSpan.FromMinutes(1); public static readonly TimeSpan DefaultJobsIdleTimeout = TimeSpan.FromMinutes(2); public const TraceLevel DefaultTraceLevel = TraceLevel.Error; @@ -245,7 +246,6 @@ public static bool RestartAppContainerOnGitDeploy(this IDeploymentSettingsManage public static bool DoBuildDuringDeployment(this IDeploymentSettingsManager settings) { string value = settings.GetValue(SettingsKeys.DoBuildDuringDeployment); - // A default value should be set on a per-deployment basis depending on the context, but // returning true by default here as an indicator of generally expected behavior return value == null || StringUtils.IsTrueLike(value); diff --git a/Kudu.Contracts/Settings/IDeploymentSettingsManager.cs b/Kudu.Contracts/Settings/IDeploymentSettingsManager.cs index 43a8689f..2dd5eebb 100644 --- a/Kudu.Contracts/Settings/IDeploymentSettingsManager.cs +++ b/Kudu.Contracts/Settings/IDeploymentSettingsManager.cs @@ -5,7 +5,7 @@ namespace Kudu.Contracts.Settings public interface IDeploymentSettingsManager { void SetValue(string key, string value); - IEnumerable> GetValues(); + IEnumerable> GetValues(IDictionary injectedSettings); /// /// Gets a value for the key from an unified list of environment, per site settings and defaults. diff --git a/Kudu.Contracts/Settings/SettingsKeys.cs b/Kudu.Contracts/Settings/SettingsKeys.cs index a4a5f996..9ddbf698 100644 --- a/Kudu.Contracts/Settings/SettingsKeys.cs +++ b/Kudu.Contracts/Settings/SettingsKeys.cs @@ -46,5 +46,11 @@ public static class SettingsKeys public const string RunFromZipOld = "WEBSITE_RUN_FROM_ZIP"; // Old name, will eventually go away public const string RunFromZip = "WEBSITE_RUN_FROM_PACKAGE"; public const string MaxZipPackageCount = "SCM_MAX_ZIP_PACKAGE_COUNT"; + // Antares container specific settings + public const string PlaceholderMode = "WEBSITE_PLACEHOLDER_MODE"; + public const string ContainerReady = "WEBSITE_CONTAINER_READY"; + public const string WebsiteHostname = "WEBSITE_HOSTNAME"; + public const string AuthEncryptionKey = "WEBSITE_AUTH_ENCRYPTION_KEY"; + public const string ContainerEncryptionKey = "CONTAINER_ENCRYPTION_KEY"; } } diff --git a/Kudu.Core/AllSafeLinuxLock.cs b/Kudu.Core/AllSafeLinuxLock.cs index 2dc5955c..f9bd57f9 100644 --- a/Kudu.Core/AllSafeLinuxLock.cs +++ b/Kudu.Core/AllSafeLinuxLock.cs @@ -19,6 +19,9 @@ public class AllSafeLinuxLock :IOperationLock { private ITraceFactory _traceFactory; private static readonly string locksPath = "/home/site/locks"; + private const int lockTimeout = 1200; //in seconds + private string defaultMsg = Resources.DeploymentLockOccMsg; + private string Msg; public AllSafeLinuxLock(string path, ITraceFactory traceFactory) { _traceFactory = traceFactory; @@ -102,9 +105,9 @@ private static void CreateLockInfoFile(string operationName) var lockInfo = new LinuxLockInfo(); lockInfo.heldByPID = Process.GetCurrentProcess().Id; lockInfo.heldByTID = Thread.CurrentThread.ManagedThreadId; - lockInfo.heldByWorker = System.Environment.GetEnvironmentVariable("WEBSITE_INSTANCE_ID"); + lockInfo.heldByWorker = System.Environment.GetEnvironmentVariable(Constants.AzureWebsiteInstanceId); lockInfo.heldByOp = operationName; - lockInfo.lockExpiry = DateTime.UtcNow.AddSeconds(600); + lockInfo.lockExpiry = DateTime.UtcNow.AddSeconds(lockTimeout); //Console.WriteLine("CreatingLockDir - LockInfoObj : "+lockInfo); var json = JsonConvert.SerializeObject(lockInfo); FileSystemHelpers.WriteAllText(locksPath+"/deployment/info.lock",json); @@ -156,7 +159,24 @@ public void Release() Console.WriteLine("ReleasingLock - There is NO LOCK HELD | ERROR"); } } - + + public string GetLockMsg() + { + //throw new NotImplementedException(); + if(Msg == null || "".Equals(Msg)) + { + return defaultMsg; + } + + return Msg; + } + + public void SetLockMsg(string msg) + { + this.Msg = msg; + + } + private class LinuxLockInfo { public DateTime lockExpiry; @@ -171,4 +191,4 @@ public override string ToString() } } } -} \ No newline at end of file +} diff --git a/Kudu.Core/Deployment/ArtifactDeploymentInfo.cs b/Kudu.Core/Deployment/ArtifactDeploymentInfo.cs new file mode 100644 index 00000000..f560cd23 --- /dev/null +++ b/Kudu.Core/Deployment/ArtifactDeploymentInfo.cs @@ -0,0 +1,41 @@ +using Kudu.Core.SourceControl; +using Kudu.Core.Tracing; +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace Kudu.Core.Deployment +{ + public class ArtifactDeploymentInfo : DeploymentInfoBase + { + private readonly IEnvironment _environment; + private readonly ITraceFactory _traceFactory; + + public ArtifactDeploymentInfo(IEnvironment environment, ITraceFactory traceFactory) + { + _environment = environment; + _traceFactory = traceFactory; + } + + public override IRepository GetRepository() + { + // Artifact "repository" does not conflict with other types, including NoRepository, + // so there's no call to EnsureRepository + var path = Path.Combine(_environment.ZipTempPath, Constants.ArtifactStagingDirectoryName); + return new NullRepository(path, _traceFactory, DeploymentTrackingId); + } + + public string Author { get; set; } + + public string AuthorEmail { get; set; } + + public string Message { get; set; } + + // Optional file name. Used by certain features like run-from-zip. + public string ArtifactFileName { get; set; } + + // This is used when getting the artifact file from the remote URL + public string RemoteURL { get; set; } + } +} diff --git a/Kudu.Core/Deployment/DeploymentContext.cs b/Kudu.Core/Deployment/DeploymentContext.cs index 5e7d01c3..5a7b62b3 100644 --- a/Kudu.Core/Deployment/DeploymentContext.cs +++ b/Kudu.Core/Deployment/DeploymentContext.cs @@ -31,6 +31,11 @@ public class DeploymentContext /// public ILogger GlobalLogger { get; set; } + /// + /// Repository Path + /// + public string RepositoryPath { get; set; } + /// /// The output path. /// diff --git a/Kudu.Core/Deployment/DeploymentHelper.cs b/Kudu.Core/Deployment/DeploymentHelper.cs index c29ed21f..1db2d498 100644 --- a/Kudu.Core/Deployment/DeploymentHelper.cs +++ b/Kudu.Core/Deployment/DeploymentHelper.cs @@ -3,6 +3,8 @@ using System.IO; using System.Linq; using Kudu.Contracts.Tracing; +using Kudu.Core.Deployment.Generator; +using Kudu.Core.Deployment.Oryx; using Kudu.Core.Infrastructure; using Kudu.Core.SourceControl; using Kudu.Core.Tracing; @@ -52,13 +54,14 @@ public static bool IsDefaultWebRootContent(string webroot) return false; } - public static void PurgeZipsIfNecessary(string sitePackagesPath, ITracer tracer, int totalAllowedZips) + public static void PurgeBuildArtifactsIfNecessary(string sitePackagesPath, BuildArtifactType fileExtension, ITracer tracer, int totalAllowedFiles) { - IEnumerable zipFiles = FileSystemHelpers.GetFiles(sitePackagesPath, "*.zip"); - if (zipFiles.Count() > totalAllowedZips) + string extension = fileExtension.ToString().ToLowerInvariant(); + IEnumerable fileNames = FileSystemHelpers.GetFiles(sitePackagesPath, $"*.{extension}"); + if (fileNames.Count() > totalAllowedFiles) { // Order the files in descending order of the modified date and remove the last (N - allowed zip files). - var fileNamesToDelete = zipFiles.OrderByDescending(fileName => FileSystemHelpers.GetLastWriteTimeUtc(fileName)).Skip(totalAllowedZips); + var fileNamesToDelete = fileNames.OrderByDescending(fileName => FileSystemHelpers.GetLastWriteTimeUtc(fileName)).Skip(totalAllowedFiles); foreach (var fileName in fileNamesToDelete) { using (tracer.Step("Deleting outdated zip file {0}", fileName)) diff --git a/Kudu.Core/Deployment/DeploymentManager.cs b/Kudu.Core/Deployment/DeploymentManager.cs index 4cbae178..3ceffd52 100644 --- a/Kudu.Core/Deployment/DeploymentManager.cs +++ b/Kudu.Core/Deployment/DeploymentManager.cs @@ -12,9 +12,11 @@ using Kudu.Core.Helpers; using Kudu.Core.Hooks; using Kudu.Core.Infrastructure; +using Kudu.Core.K8SE; using Kudu.Core.Settings; using Kudu.Core.SourceControl; using Kudu.Core.Tracing; +using Microsoft.AspNetCore.Http; namespace Kudu.Core.Deployment { @@ -33,6 +35,7 @@ public class DeploymentManager : IDeploymentManager private readonly IDeploymentSettingsManager _settings; private readonly IDeploymentStatusManager _status; private readonly IWebHooksManager _hooksManager; + private readonly IDictionary _appSettings; private const string RestartTriggerReason = "App deployment"; private const string XmlLogFile = "log.xml"; @@ -48,8 +51,9 @@ public DeploymentManager(ISiteBuilderFactory builderFactory, IDeploymentStatusManager status, IDictionary namedLocks, ILogger globalLogger, - IWebHooksManager hooksManager) - : this(builderFactory, environment, traceFactory, analytics, settings, status, namedLocks["deployment"], globalLogger, hooksManager) + IWebHooksManager hooksManager, + IHttpContextAccessor httpContextAccessor) + : this(builderFactory, environment, traceFactory, analytics, settings, status, namedLocks["deployment"], globalLogger, hooksManager, httpContextAccessor) { } public DeploymentManager(ISiteBuilderFactory builderFactory, @@ -60,10 +64,12 @@ public DeploymentManager(ISiteBuilderFactory builderFactory, IDeploymentStatusManager status, IOperationLock deploymentLock, ILogger globalLogger, - IWebHooksManager hooksManager) + IWebHooksManager hooksManager, + IHttpContextAccessor httpContextAccessor) { _builderFactory = builderFactory; - _environment = environment; + _environment = GetEnvironment(httpContextAccessor, environment); + _appSettings = GetAppSettings(httpContextAccessor); _traceFactory = traceFactory; _analytics = analytics; _deploymentLock = deploymentLock; @@ -73,6 +79,35 @@ public DeploymentManager(ISiteBuilderFactory builderFactory, _hooksManager = hooksManager; } + private IEnvironment GetEnvironment(IHttpContextAccessor accessor, IEnvironment environment) + { + IEnvironment _environment; + if (!K8SEDeploymentHelper.IsK8SEEnvironment() || accessor == null) + { + _environment = environment; + } + else + { + var context = accessor.HttpContext; + _environment = (IEnvironment)context.Items["environment"]; + } + return _environment; + } + + internal static IDictionary GetAppSettings(IHttpContextAccessor accessor, Func isK8SeEnvironment = null) + { + isK8SeEnvironment = isK8SeEnvironment ?? K8SEDeploymentHelper.IsK8SEEnvironment; + + IDictionary appSettings = new Dictionary(); + if (isK8SeEnvironment() && accessor != null) + { + var context = accessor.HttpContext; + appSettings = (IDictionary) context.Items["appSettings"]; + } + + return appSettings; + } + private bool IsDeploying { get @@ -92,7 +127,7 @@ public IEnumerable GetResults() public DeployResult GetResult(string id) { - return GetResult(id, _status.ActiveDeploymentId, IsDeploying); + return GetResult(id, _status.ActiveDeploymentId, IsDeploying); } public IEnumerable GetLogEntries(string id) @@ -159,7 +194,7 @@ public void Delete(string id) throw new InvalidOperationException(String.Format(CultureInfo.CurrentCulture, Resources.Error_UnableToDeleteDeploymentActive, id)); } - _status.Delete(id); + _status.Delete(id, _environment); } } @@ -175,128 +210,149 @@ public async Task DeployAsync( Console.WriteLine("Deploy Async"); using (var deploymentAnalytics = new DeploymentAnalytics(_analytics, _settings)) { - Exception exception = null; - ITracer tracer = _traceFactory.GetTracer(); - IDisposable deployStep = null; - ILogger innerLogger = null; - string targetBranch = null; + Exception exception = null; + ITracer tracer = _traceFactory.GetTracer(); + IDisposable deployStep = null; + ILogger innerLogger = null; + string targetBranch = null; + + // If we don't get a changeset, find out what branch we should be deploying and get the commit ID from it + if (changeSet == null) + { + targetBranch = _settings.GetBranch(); + + changeSet = repository.GetChangeSet(targetBranch); - // If we don't get a changeset, find out what branch we should be deploying and get the commit ID from it if (changeSet == null) { - targetBranch = _settings.GetBranch(); - - changeSet = repository.GetChangeSet(targetBranch); - - if (changeSet == null) - { - throw new InvalidOperationException(String.Format( - "The current deployment branch is '{0}', but nothing has been pushed to it", - targetBranch)); - } + throw new InvalidOperationException(String.Format( + "The current deployment branch is '{0}', but nothing has been pushed to it", + targetBranch)); } + } - string id = changeSet.Id; - IDeploymentStatusFile statusFile = null; - try - { - deployStep = tracer.Step($"DeploymentManager.Deploy(id:{id})"); - // Remove the old log file for this deployment id - string logPath = GetLogPath(id); - FileSystemHelpers.DeleteFileSafe(logPath); + string id = changeSet.Id; + _environment.CurrId = id; + IDeploymentStatusFile statusFile = null; + try + { + deployStep = tracer.Step($"DeploymentManager.Deploy(id:{id})"); + // Remove the old log file for this deployment id + string logPath = GetLogPath(id); + FileSystemHelpers.DeleteFileSafe(logPath); - statusFile = GetOrCreateStatusFile(changeSet, tracer, deployer); - statusFile.MarkPending(); + statusFile = GetOrCreateStatusFile(changeSet, tracer, deployer); - ILogger logger = GetLogger(changeSet.Id); + statusFile.MarkPending(); - if (needFileUpdate) + ILogger logger = GetLogger(changeSet.Id); + if (needFileUpdate) + { + using (tracer.Step("Updating to specific changeset")) { - using (tracer.Step("Updating to specific changeset")) - { - innerLogger = logger.Log(Resources.Log_UpdatingBranch, targetBranch ?? id); - using (var writer = new ProgressWriter()) - { - // Update to the specific changeset or branch - repository.Update(targetBranch ?? id); - } + innerLogger = logger.Log(Resources.Log_UpdatingBranch, targetBranch ?? id); + + using (var writer = new ProgressWriter()) + { + // Update to the specific changeset or branch + repository.Update(targetBranch ?? id); } } + } - if (_settings.ShouldUpdateSubmodules()) + if (_settings.ShouldUpdateSubmodules()) + { + using (tracer.Step("Updating submodules")) { - using (tracer.Step("Updating submodules")) - { - innerLogger = logger.Log(Resources.Log_UpdatingSubmodules); + innerLogger = logger.Log(Resources.Log_UpdatingSubmodules); - repository.UpdateSubmodules(); - } + repository.UpdateSubmodules(); } + } - if (clean) - { - tracer.Trace("Cleaning {0} repository", repository.RepositoryType); + if (clean) + { + tracer.Trace("Cleaning {0} repository", repository.RepositoryType); - innerLogger = logger.Log(Resources.Log_CleaningRepository, repository.RepositoryType); + innerLogger = logger.Log(Resources.Log_CleaningRepository, repository.RepositoryType); - repository.Clean(); - } + repository.Clean(); + } - // set to null as Build() below takes over logging - innerLogger = null; + // set to null as Build() below takes over logging + innerLogger = null; - // Perform the build deployment of this changeset - await Build(changeSet, tracer, deployStep, repository, deploymentInfo, deploymentAnalytics, fullBuildByDefault); + // Perform the build deployment of this changeset + await Build(changeSet, tracer, deployStep, repository, deploymentInfo, deploymentAnalytics, fullBuildByDefault); - if (!(OSDetector.IsOnWindows() && - !EnvironmentHelper.IsWindowsContainers()) && - _settings.RestartAppContainerOnGitDeploy()) + if ((!(OSDetector.IsOnWindows() && + !EnvironmentHelper.IsWindowsContainers()) && + _settings.RestartAppContainerOnGitDeploy()) || + K8SEDeploymentHelper.IsK8SEEnvironment()) + { + if (K8SEDeploymentHelper.IsK8SEEnvironment()) + { + //logger.Log(Resources.Log_TriggeringK8SERestart); + } + else { logger.Log(Resources.Log_TriggeringContainerRestart); - DockerContainerRestartTrigger.RequestContainerRestart(_environment, RestartTriggerReason); } - } - catch (Exception ex) - { - exception = ex; - if (innerLogger != null) + string appName = _environment.K8SEAppName; + string repoUrl = deploymentInfo == null ? "empty" : deploymentInfo.RepositoryUrl; + if(deploymentInfo == null) { - innerLogger.Log(ex); + DockerContainerRestartTrigger.RequestContainerRestart(_environment, RestartTriggerReason, appSettings: _appSettings); } - - if (statusFile != null) + else { - MarkStatusComplete(statusFile, success: false); + DockerContainerRestartTrigger.RequestContainerRestart(_environment, RestartTriggerReason, deploymentInfo == null ? null : deploymentInfo.RepositoryUrl, deploymentInfo.TargetPath, appSettings:_appSettings); } + logger.Log($"Deployment Pod Rollout Started! Use 'kubectl -n k8se-apps get pods {appName} --watch' to monitor the rollout status"); + logger.Log($"Deployment Pod Rollout Started! Use kubectl watch deplotment {appName} to monitor the rollout status"); + } + } + catch (Exception ex) + { + exception = ex; - tracer.TraceError(ex); - - deploymentAnalytics.Error = ex.ToString(); + if (innerLogger != null) + { + innerLogger.Log(ex); + } - if (deployStep != null) - { - deployStep.Dispose(); - } + if (statusFile != null) + { + MarkStatusComplete(statusFile, success: false); } - // Reload status file with latest updates - statusFile = _status.Open(id); - using (tracer.Step("Reloading status file with latest updates")) + tracer.TraceError(ex); + + deploymentAnalytics.Error = ex.ToString(); + + if (deployStep != null) { - if (statusFile != null) - { - await _hooksManager.PublishEventAsync(HookEventTypes.PostDeployment, statusFile); - } + deployStep.Dispose(); } + } - if (exception != null) + // Reload status file with latest updates + statusFile = _status.Open(id, _environment); + using (tracer.Step("Reloading status file with latest updates")) + { + if (statusFile != null) { - tracer.TraceError(exception); - throw new DeploymentFailedException(exception); + await _hooksManager.PublishEventAsync(HookEventTypes.PostDeployment, statusFile); } - + } + + if (exception != null) + { + tracer.TraceError(exception); + throw new DeploymentFailedException(exception); + } } } @@ -306,7 +362,7 @@ public IDisposable CreateTemporaryDeployment(string statusText, out ChangeSet te using (tracer.Step("Creating temporary deployment")) { changeSet = changeSet != null && changeSet.IsTemporary ? changeSet : CreateTemporaryChangeSet(); - IDeploymentStatusFile statusFile = _status.Create(changeSet.Id); + IDeploymentStatusFile statusFile = _status.Create(changeSet.Id, _environment); statusFile.Id = changeSet.Id; statusFile.Message = changeSet.Message; statusFile.Author = changeSet.AuthorName; @@ -326,7 +382,7 @@ public IDisposable CreateTemporaryDeployment(string statusText, out ChangeSet te { if (changeSet.IsTemporary) { - _status.Delete(changeSet.Id); + _status.Delete(changeSet.Id, _environment); } }); } @@ -345,7 +401,7 @@ private IEnumerable PurgeAndGetDeployments() // Order the results by date (newest first). Previously, we supported OData to allow // arbitrary queries, but that was way overkill and brought in too many large binaries. IEnumerable results; - results = EnumerateResults().OrderByDescending(t => t.ReceivedTime).ToList(); + results = EnumerateResults().OrderByDescending(t => t.ReceivedTime).ToList(); try { results = PurgeDeployments(results); @@ -386,7 +442,7 @@ internal IEnumerable PurgeDeployments(IEnumerable re toDelete.AddRange(GetPurgeTemporaryDeployments(results)); toDelete.AddRange(GetPurgeFailedDeployments(results)); toDelete.AddRange(GetPurgeObsoleteDeployments(results)); - + if (toDelete.Any()) { var tracer = _traceFactory.GetTracer(); @@ -394,7 +450,7 @@ internal IEnumerable PurgeDeployments(IEnumerable re { foreach (DeployResult delete in toDelete) { - _status.Delete(delete.Id); + _status.Delete(delete.Id, _environment); tracer.Trace("Remove {0}, {1}, received at {2}", delete.Id.Substring(0, Math.Min(delete.Id.Length, 9)), @@ -498,11 +554,11 @@ IDeploymentStatusFile GetOrCreateStatusFile(ChangeSet changeSet, ITracer tracer, using (tracer.Step("Collecting changeset information")) { // Check if the status file already exists. This would happen when we're doing a redeploy - IDeploymentStatusFile statusFile = _status.Open(id); + IDeploymentStatusFile statusFile = _status.Open(id, _environment); if (statusFile == null) { // Create the status file and store information about the commit - statusFile = _status.Create(id); + statusFile = _status.Create(id, _environment); } statusFile.Message = changeSet.Message; statusFile.Author = changeSet.AuthorName; @@ -517,7 +573,7 @@ IDeploymentStatusFile GetOrCreateStatusFile(ChangeSet changeSet, ITracer tracer, private DeployResult GetResult(string id, string activeDeploymentId, bool isDeploying) { - var file = VerifyDeployment(id, isDeploying); + var file = VerifyDeployment(id, isDeploying); if (file == null) { return null; @@ -572,7 +628,7 @@ private async Task Build( logger = GetLogger(id); ILogger innerLogger = logger.Log(Resources.Log_PreparingDeployment, TrimId(id)); - currentStatus = _status.Open(id); + currentStatus = _status.Open(id, _environment); currentStatus.Complete = false; currentStatus.StartTime = DateTime.UtcNow; currentStatus.Status = DeployStatus.Building; @@ -589,6 +645,7 @@ private async Task Build( var perDeploymentSettings = DeploymentSettingsManager.BuildPerDeploymentSettingsManager(repository.RepositoryPath, settingsProviders); string delayMaxInStr = perDeploymentSettings.GetValue(SettingsKeys.MaxRandomDelayInSec); + perDeploymentSettings.SetValue(SettingsKeys.DoBuildDuringDeployment, fullBuildByDefault.ToString()); if (!String.IsNullOrEmpty(delayMaxInStr)) { int maxDelay; @@ -612,7 +669,7 @@ private async Task Build( { using (tracer.Step("Determining deployment builder")) { - builder = _builderFactory.CreateBuilder(tracer, innerLogger, perDeploymentSettings, repository); + builder = _builderFactory.CreateBuilder(tracer, innerLogger, perDeploymentSettings, repository, deploymentInfo); deploymentAnalytics.ProjectType = builder.ProjectType; tracer.Trace("Builder is {0}", builder.GetType().Name); } @@ -648,11 +705,11 @@ private async Task Build( NextManifestFilePath = GetDeploymentManifestPath(id), PreviousManifestFilePath = GetActiveDeploymentManifestPath(), IgnoreManifest = deploymentInfo != null && deploymentInfo.CleanupTargetDirectory, - // Ignoring the manifest will cause kudusync to delete sub-directories / files - // in the destination directory that are not present in the source directory, - // without checking the manifest to see if the file was copied over to the destination - // during a previous kudusync operation. This effectively performs a clean deployment - // from the source to the destination directory + // Ignoring the manifest will cause kudusync to delete sub-directories / files + // in the destination directory that are not present in the source directory, + // without checking the manifest to see if the file was copied over to the destination + // during a previous kudusync operation. This effectively performs a clean deployment + // from the source to the destination directory Tracer = tracer, Logger = logger, GlobalLogger = _globalLogger, @@ -682,21 +739,31 @@ private async Task Build( try { await builder.Build(context); + builder.PostBuild(context); + if (FunctionAppHelper.LooksLikeFunctionApp() && _environment.IsOnLinuxConsumption) + { + // A Linux consumption function app deployment requires (no matter whether it is oryx build or basic deployment) + // 1. packaging the output folder + // 2. upload the artifact to user's storage account + // 3. reset the container workers after deployment + await LinuxConsumptionDeploymentHelper.SetupLinuxConsumptionFunctionAppDeployment(_environment, _settings, context); + } + await PostDeploymentHelper.SyncFunctionsTriggers( - _environment.RequestId, - new PostDeploymentTraceListener(tracer, logger), + _environment.RequestId, + new PostDeploymentTraceListener(tracer, logger), deploymentInfo?.SyncFunctionsTriggersPath); if (_settings.TouchWatchedFileAfterDeployment()) { TryTouchWatchedFile(context, deploymentInfo); } - + if (_settings.RunFromLocalZip() && deploymentInfo is ZipDeploymentInfo) { - await PostDeploymentHelper.UpdateSiteVersion(deploymentInfo as ZipDeploymentInfo, _environment, logger); + await PostDeploymentHelper.UpdatePackageName(deploymentInfo as ZipDeploymentInfo, _environment, logger); } FinishDeployment(id, deployStep); @@ -727,7 +794,7 @@ await PostDeploymentHelper.SyncFunctionsTriggers( private void PreDeployment(ITracer tracer) { - if (Environment.IsAzureEnvironment() + if (Environment.IsAzureEnvironment() && FileSystemHelpers.DirectoryExists(_environment.SSHKeyPath) && OSDetector.IsOnWindows()) { @@ -736,7 +803,7 @@ private void PreDeployment(ITracer tracer) if (!String.Equals(src, dst, StringComparison.OrdinalIgnoreCase)) { - // copy %HOME%\.ssh to %USERPROFILE%\.ssh key to workaround + // copy %HOME%\.ssh to %USERPROFILE%\.ssh key to workaround // npm with private ssh git dependency using (tracer.Step("Copying SSH keys")) { @@ -777,17 +844,22 @@ private static void FailDeployment(ITracer tracer, IDisposable deployStep, Deplo private static string GetOutputPath(DeploymentInfoBase deploymentInfo, IEnvironment environment, IDeploymentSettingsManager perDeploymentSettings) { - string targetPath = perDeploymentSettings.GetTargetPath(); - - if (string.IsNullOrWhiteSpace(targetPath)) + string targetSubDirectoryRelativePath = perDeploymentSettings.GetTargetPath(); + + if (string.IsNullOrWhiteSpace(targetSubDirectoryRelativePath)) { - targetPath = deploymentInfo?.TargetPath; + targetSubDirectoryRelativePath = deploymentInfo?.TargetSubDirectoryRelativePath; } - - if (!string.IsNullOrWhiteSpace(targetPath)) + + if (deploymentInfo?.Deployer == Constants.OneDeploy) + { + return string.IsNullOrWhiteSpace(deploymentInfo?.TargetRootPath) ? environment.WebRootPath : deploymentInfo.TargetRootPath; + } + + if (!string.IsNullOrWhiteSpace(targetSubDirectoryRelativePath)) { - targetPath = targetPath.Trim('\\', '/'); - return Path.GetFullPath(Path.Combine(environment.WebRootPath, targetPath)); + targetSubDirectoryRelativePath = targetSubDirectoryRelativePath.Trim('\\', '/'); + return Path.GetFullPath(Path.Combine(environment.WebRootPath, targetSubDirectoryRelativePath)); } return environment.WebRootPath; @@ -819,14 +891,14 @@ private IEnumerable EnumerateResults() /// private IDeploymentStatusFile VerifyDeployment(string id, bool isDeploying) { - IDeploymentStatusFile statusFile = _status.Open(id); - + IDeploymentStatusFile statusFile = _status.Open(id, _environment); + if (statusFile == null) { return null; } - - + + if (statusFile.Complete) { return statusFile; @@ -857,7 +929,7 @@ private void FinishDeployment(string id, IDisposable deployStep) ILogger logger = GetLogger(id); logger.Log(Resources.Log_DeploymentSuccessful); - IDeploymentStatusFile currentStatus = _status.Open(id); + IDeploymentStatusFile currentStatus = _status.Open(id, _environment); MarkStatusComplete(currentStatus, success: true); _status.ActiveDeploymentId = id; @@ -868,7 +940,7 @@ private void FinishDeployment(string id, IDisposable deployStep) } private static void TryTouchWatchedFile(DeploymentContext context, DeploymentInfoBase deploymentInfo) - { + { try { string watchedFileRelativePath = deploymentInfo?.WatchedFilePath; @@ -876,7 +948,7 @@ private static void TryTouchWatchedFile(DeploymentContext context, DeploymentInf { watchedFileRelativePath = "web.config"; } - + string watchedFileAbsolutePath = Path.Combine(context.OutputPath, watchedFileRelativePath); if (File.Exists(watchedFileAbsolutePath)) @@ -922,7 +994,7 @@ public ILogger GetLogger(string id) { var path = GetLogPath(id); var logger = GetLoggerForFile(path); - return new ProgressLogger(id, _status, new CascadeLogger(logger, _globalLogger)); + return new ProgressLogger(id, _status, new CascadeLogger(logger, _globalLogger), _environment); } /// diff --git a/Kudu.Core/Deployment/DeploymentStatusFile.cs b/Kudu.Core/Deployment/DeploymentStatusFile.cs index ae4327e3..877e90da 100644 --- a/Kudu.Core/Deployment/DeploymentStatusFile.cs +++ b/Kudu.Core/Deployment/DeploymentStatusFile.cs @@ -53,7 +53,6 @@ public static DeploymentStatusFile Open(string id, IEnvironment environment, IAn return statusLock.LockOperation(() => { string path = Path.Combine(environment.DeploymentsPath, id, StatusFile); - if (!FileSystemHelpers.FileExists(path)) { return null; @@ -220,8 +219,11 @@ private static string GetSiteName(IEnvironment environment) string siteName = ServerConfiguration.GetApplicationName(); if (String.IsNullOrEmpty(siteName)) { + Console.WriteLine("Gettin Site Root"); // Otherwise get it from the root directory name siteName = Path.GetFileName(environment.RootPath); + Console.WriteLine("siteName "+siteName); + } return siteName; diff --git a/Kudu.Core/Deployment/DeploymentStatusManager.cs b/Kudu.Core/Deployment/DeploymentStatusManager.cs index 0cff1902..b4d219fb 100644 --- a/Kudu.Core/Deployment/DeploymentStatusManager.cs +++ b/Kudu.Core/Deployment/DeploymentStatusManager.cs @@ -10,7 +10,6 @@ namespace Kudu.Core.Deployment public class DeploymentStatusManager : IDeploymentStatusManager { public static readonly TimeSpan LockTimeout = TimeSpan.FromSeconds(10); - private readonly IEnvironment _environment; private readonly IAnalytics _analytics; private readonly IOperationLock _statusLock; private readonly string _activeFile; @@ -25,25 +24,24 @@ public DeploymentStatusManager(IEnvironment environment, IAnalytics analytics, IOperationLock statusLock) { - _environment = environment; _analytics = analytics; _statusLock = statusLock; _activeFile = Path.Combine(environment.DeploymentsPath, Constants.ActiveDeploymentFile); } - public IDeploymentStatusFile Create(string id) + public IDeploymentStatusFile Create(string id, IEnvironment environment) { - return DeploymentStatusFile.Create(id, _environment, _statusLock); + return DeploymentStatusFile.Create(id, environment, _statusLock); } - public IDeploymentStatusFile Open(string id) + public IDeploymentStatusFile Open(string id, IEnvironment environment) { - return DeploymentStatusFile.Open(id, _environment, _analytics, _statusLock); + return DeploymentStatusFile.Open(id, environment, _analytics, _statusLock); } - public void Delete(string id) + public void Delete(string id, IEnvironment environment) { - string path = Path.Combine(_environment.DeploymentsPath, id); + string path = Path.Combine(environment.DeploymentsPath, id); _statusLock.LockOperation(() => { diff --git a/Kudu.Core/Deployment/FetchDeploymentManager.cs b/Kudu.Core/Deployment/FetchDeploymentManager.cs index ba651c01..1e03c2a6 100644 --- a/Kudu.Core/Deployment/FetchDeploymentManager.cs +++ b/Kudu.Core/Deployment/FetchDeploymentManager.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.IO.Abstractions; using System.Threading.Tasks; using Kudu.Contracts.Infrastructure; using Kudu.Contracts.Settings; @@ -10,8 +11,10 @@ using Kudu.Core.Helpers; using Kudu.Core.Hooks; using Kudu.Core.Infrastructure; +using Kudu.Core.K8SE; using Kudu.Core.SourceControl; using Kudu.Core.Tracing; +using Microsoft.AspNetCore.Http; namespace Kudu.Core.Deployment { @@ -23,6 +26,7 @@ public class FetchDeploymentManager : IFetchDeploymentManager private readonly IOperationLock _deploymentLock; private readonly IDeploymentManager _deploymentManager; private readonly IDeploymentStatusManager _status; + private IHttpContextAccessor _httpContextAccessor = null; private readonly string _markerFilePath; public FetchDeploymentManager( @@ -32,8 +36,9 @@ public FetchDeploymentManager( //IOperationLock deploymentLock, IDictionary namedLocks, IDeploymentManager deploymentManager, - IDeploymentStatusManager status) - : this(settings, environment, tracer, namedLocks["deployment"], deploymentManager, status) + IDeploymentStatusManager status, + IHttpContextAccessor httpContextAccessor) + : this(settings, environment, tracer, namedLocks["deployment"], deploymentManager, status, httpContextAccessor) { } public FetchDeploymentManager( @@ -42,15 +47,16 @@ public FetchDeploymentManager( ITracer tracer, IOperationLock deploymentLock, IDeploymentManager deploymentManager, - IDeploymentStatusManager status) + IDeploymentStatusManager status, + IHttpContextAccessor httpContextAccessor) { _settings = settings; - _environment = environment; + _environment = GetEnvironment(httpContextAccessor, environment); + _httpContextAccessor = httpContextAccessor; _tracer = tracer; _deploymentLock = deploymentLock; _deploymentManager = deploymentManager; _status = status; - _markerFilePath = Path.Combine(environment.DeploymentsPath, "pending"); // Prefer marker creation in ctor to delay create when needed. @@ -68,6 +74,21 @@ public FetchDeploymentManager( } } + private IEnvironment GetEnvironment(IHttpContextAccessor accessor, IEnvironment environment) + { + IEnvironment _environment; + var context = accessor.HttpContext; + if (!K8SEDeploymentHelper.IsK8SEEnvironment()) + { + _environment = environment; + } + else + { + _environment = (IEnvironment)context.Items["environment"]; + } + return _environment; + } + public async Task FetchDeploy( DeploymentInfoBase deployInfo, bool asyncRequested, @@ -83,7 +104,7 @@ public async Task FetchDeploy( { return FetchDeploymentRequestResult.ForbiddenScmDisabled; } - + // Else if this app is configured with a url in WEBSITE_USE_ZIP, then fail the deployment // since this is a RunFromZip site and the deployment has no chance of succeeding. else if (_settings.RunFromRemoteZip()) @@ -106,7 +127,8 @@ public async Task FetchDeploy( _settings, _tracer.TraceLevel, requestUri, - waitForTempDeploymentCreation); + waitForTempDeploymentCreation, + _httpContextAccessor); return successfullyRequested ? FetchDeploymentRequestResult.RunningAynschronously @@ -125,7 +147,6 @@ public async Task FetchDeploy( } await PerformDeployment(deployInfo); - Console.WriteLine("\n\n\n\n Perform deployment Over\n\n\n"); return FetchDeploymentRequestResult.RanSynchronously; }, "Performing continuous deployment", TimeSpan.Zero); } @@ -151,8 +172,8 @@ public async Task FetchDeploy( } } - public async Task PerformDeployment(DeploymentInfoBase deploymentInfo, - IDisposable tempDeployment = null, + public async Task PerformDeployment(DeploymentInfoBase deploymentInfo, + IDisposable tempDeployment = null, ChangeSet tempChangeSet = null) { DateTime currentMarkerFileUTC; @@ -240,7 +261,7 @@ await _deploymentManager.DeployAsync( // In case the commit or perhaps fetch do no-op. if (deploymentInfo.TargetChangeset != null) { - IDeploymentStatusFile statusFile = _status.Open(deploymentInfo.TargetChangeset.Id); + IDeploymentStatusFile statusFile = _status.Open(deploymentInfo.TargetChangeset.Id, _environment); if (statusFile != null) { statusFile.MarkFailed(); @@ -260,14 +281,14 @@ await _deploymentManager.DeployAsync( if (lastChange != null && PostDeploymentHelper.IsAutoSwapEnabled()) { - IDeploymentStatusFile statusFile = _status.Open(lastChange.Id); + IDeploymentStatusFile statusFile = _status.Open(lastChange.Id, _environment); if (statusFile.Status == DeployStatus.Success) { // if last change is not null and finish successfully, mean there was at least one deployoment happened // since deployment is now done, trigger swap if enabled await PostDeploymentHelper.PerformAutoSwap( - _environment.RequestId, - new PostDeploymentTraceListener(_tracer, + _environment.RequestId, + new PostDeploymentTraceListener(_tracer, _deploymentManager.GetLogger(lastChange.Id))); } } @@ -293,7 +314,8 @@ public static async Task PerformBackgroundDeployment( IDeploymentSettingsManager settings, TraceLevel traceLevel, Uri uri, - bool waitForTempDeploymentCreation) + bool waitForTempDeploymentCreation, + IHttpContextAccessor _httpContextAccessor) { var tracer = traceLevel <= TraceLevel.Off ? NullTracer.Instance : new CascadeTracer(new XmlTracer(environment.TracePath, traceLevel), new ETWTracer(environment.RequestId, "POST")); var traceFactory = new TracerFactory(() => tracer); @@ -311,7 +333,7 @@ public static async Task PerformBackgroundDeployment( // Needed for deployments where deferred deployment is not allowed. Will be set to false if // lock contention occurs and AllowDeferredDeployment is false, otherwise true. var deploymentWillOccurTcs = new TaskCompletionSource(); - + // This task will be let out of scope intentionally var deploymentTask = Task.Run(() => { @@ -328,10 +350,10 @@ public static async Task PerformBackgroundDeployment( var analytics = new Analytics(settings, new ServerConfiguration(), traceFactory); var deploymentStatusManager = new DeploymentStatusManager(environment, analytics, statusLock); - var siteBuilderFactory = new SiteBuilderFactory(new BuildPropertyProvider(), environment); + var siteBuilderFactory = new SiteBuilderFactory(new BuildPropertyProvider(), environment, _httpContextAccessor); var webHooksManager = new WebHooksManager(tracer, environment, hooksLock); - var deploymentManager = new DeploymentManager(siteBuilderFactory, environment, traceFactory, analytics, settings, deploymentStatusManager, deploymentLock, NullLogger.Instance, webHooksManager); - var fetchDeploymentManager = new FetchDeploymentManager(settings, environment, tracer, deploymentLock, deploymentManager, deploymentStatusManager); + var deploymentManager = new DeploymentManager(siteBuilderFactory, environment, traceFactory, analytics, settings, deploymentStatusManager, deploymentLock, NullLogger.Instance, webHooksManager, _httpContextAccessor); + var fetchDeploymentManager = new FetchDeploymentManager(settings, environment, tracer, deploymentLock, deploymentManager, deploymentStatusManager, _httpContextAccessor); IDisposable tempDeployment = null; diff --git a/Kudu.Core/Deployment/Generator/BasicBuilder.cs b/Kudu.Core/Deployment/Generator/BasicBuilder.cs index 933749ad..6f1a0974 100644 --- a/Kudu.Core/Deployment/Generator/BasicBuilder.cs +++ b/Kudu.Core/Deployment/Generator/BasicBuilder.cs @@ -1,12 +1,73 @@ using Kudu.Contracts.Settings; +using Kudu.Core.Deployment.Oryx; +using Kudu.Core.Infrastructure; +using Kudu.Core.K8SE; +using System; +using System.IO; +using System.Threading.Tasks; namespace Kudu.Core.Deployment.Generator { public class BasicBuilder : BaseBasicBuilder { + IEnvironment _environment; public BasicBuilder(IEnvironment environment, IDeploymentSettingsManager settings, IBuildPropertyProvider propertyProvider, string repositoryPath, string projectPath) : base(environment, settings, propertyProvider, repositoryPath, projectPath, "--basic") { + _environment = environment; + } + + public override Task Build(DeploymentContext context) + { + if (K8SEDeploymentHelper.IsK8SEEnvironment()) + { + // K8SE TODO: Move to a resources file + ILogger customLogger = context.Logger.Log("Builder : K8SE Basic Builder"); + string src = _environment.ZipTempPath; + string artifactDir = Path.Combine(_environment.SiteRootPath, "artifacts", _environment.CurrId); + FileSystemHelpers.EnsureDirectory(Path.Combine(_environment.ZipTempPath, "artifacts")); + FileSystemHelpers.EnsureDirectory(artifactDir); + return Task.Factory.StartNew(() => PackageArtifactFromFolder(context, Path.Combine(_environment.ZipTempPath, "extracted"), Path.Combine(_environment.SiteRootPath, "artifacts", _environment.CurrId), "artifact.zip", BuildArtifactType.Squashfs, 2)); + } + else + { + return Task.CompletedTask; + } + } + + private string PackageArtifactFromFolder(DeploymentContext context, string srcDirectory, string artifactDirectory, string artifactFilename, BuildArtifactType artifactType, int numBuildArtifacts = 0) + { + context.Logger.Log($"Writing the artifacts to {artifactType.ToString()} file at {artifactDirectory}"); + FileSystemHelpers.EnsureDirectory(artifactDirectory); + string file = Path.Combine(artifactDirectory, artifactFilename); + var exe = ExternalCommandFactory.BuildExternalCommandExecutable(srcDirectory, artifactDirectory, context.Logger); + try + { + switch (artifactType) + { + case BuildArtifactType.Zip: + exe.ExecuteWithProgressWriter(context.Logger, context.Tracer, $"zip -r -0 -q {file} ."); + break; + case BuildArtifactType.Squashfs: + exe.ExecuteWithProgressWriter(context.Logger, context.Tracer, $"mksquashfs . {file} -noappend"); + break; + default: + throw new ArgumentException($"Received unknown file extension {artifactType.ToString()}"); + } + } + catch (Exception) + { + context.GlobalLogger.LogError(); + throw; + } + + // Just to be sure that we don't keep adding build artifacts here + if (numBuildArtifacts > 0) + { + DeploymentHelper.PurgeBuildArtifactsIfNecessary(artifactDirectory, artifactType, context.Tracer, numBuildArtifacts); + } + + return file; } public override string ProjectType diff --git a/Kudu.Core/Deployment/Generator/Deploymentv2BasicBuilder.cs b/Kudu.Core/Deployment/Generator/Deploymentv2BasicBuilder.cs new file mode 100644 index 00000000..13fe36fe --- /dev/null +++ b/Kudu.Core/Deployment/Generator/Deploymentv2BasicBuilder.cs @@ -0,0 +1,70 @@ +using Kudu.Contracts.Settings; +using Kudu.Contracts.Tracing; +using Kudu.Core.Infrastructure; +using Kudu.Core.SourceControl; +using System; +using System.IO; + +namespace Kudu.Core.Deployment.Generator +{ + class DeploymentV2BasicBuilder : BaseBasicBuilder + { + private readonly IEnvironment _environment; + private readonly ILogger _logger; + private readonly ITracer _tracer; + private readonly IRepository _repository; + + public DeploymentV2BasicBuilder(IEnvironment environment, + IDeploymentSettingsManager settings, + IBuildPropertyProvider propertyProvider, + DeploymentInfoBase deploymentInfo, + IRepository repository, + ILogger logger, + ITracer tracer, + string repositoryPath, + string projectPath) + : base(environment, settings, propertyProvider, repositoryPath, projectPath, "--basic") + { + this._environment = environment; + this._logger = logger; + this._tracer = tracer; + this._repository = repository; + if(deploymentInfo is ZipDeploymentInfo) + { + var zipDeploymentInfo = ((ZipDeploymentInfo)deploymentInfo); + SetupArtifacts(zipDeploymentInfo); + } + } + + public override string ProjectType + { + get { return "BASICV2"; } + } + + private void SetupArtifacts(ZipDeploymentInfo deploymentInfo) + { + string deploymentsPath = _environment.DeploymentsPath; + string artifactPath = Path.Combine(deploymentsPath, _repository.CurrentId, "artifact"); + + FileSystemHelpers.EnsureDirectory(artifactPath); + + string zipAppName = $"{DateTime.UtcNow.ToString("yyyyMMddHHmmss")}.zip"; + + var copyExe = ExternalCommandFactory.BuildExternalCommandExecutable(_environment.ZipTempPath, artifactPath, _logger); + var copyToPath = Path.Combine(artifactPath, zipAppName); + + try + { + copyExe.ExecuteWithProgressWriter(_logger, _tracer, $"cp {deploymentInfo.RepositoryUrl} {copyToPath}"); + } + catch (Exception) + { + throw; + } + + // Update the packagename file to ensure latest app restart loads the new zip file + // K8SE TODO: Uncomment this and test + //DeploymentHelper.UpdateLatestAndPurgeOldArtifacts(_environment, zipAppName, artifactPath, _tracer); + } + } +} diff --git a/Kudu.Core/Deployment/Generator/ExternalCommandBuilder.cs b/Kudu.Core/Deployment/Generator/ExternalCommandBuilder.cs index b1cc0894..c1ded864 100644 --- a/Kudu.Core/Deployment/Generator/ExternalCommandBuilder.cs +++ b/Kudu.Core/Deployment/Generator/ExternalCommandBuilder.cs @@ -8,6 +8,7 @@ using Kudu.Contracts.Settings; using Kudu.Core.Infrastructure; using Kudu.Core.Tracing; +using Kudu.Core.Helpers; using NuGet.Versioning; namespace Kudu.Core.Deployment.Generator @@ -16,6 +17,7 @@ namespace Kudu.Core.Deployment.Generator // // ExternalCommandBuilder // CustomBuilder + // OryxBuilder // GeneratorSiteBuilder // BaseBasicBuilder // BasicBuilder @@ -61,11 +63,17 @@ public void PostBuild(DeploymentContext context) { var fi = new FileInfo(file); string scriptFilePath = null; - if (string.Equals(".ps1", fi.Extension, StringComparison.OrdinalIgnoreCase)) + + if (OSDetector.IsOnWindows()) + { + scriptFilePath = GetWindowsPostBuildFilepath(context, fi); + } + else if (!OSDetector.IsOnWindows()) { - scriptFilePath = string.Format(CultureInfo.InvariantCulture, "PowerShell.exe -ExecutionPolicy RemoteSigned -File \"{0}\"", file); + scriptFilePath = GetLinuxPostBuildFilepath(context, fi); } - else + + if (string.IsNullOrEmpty(scriptFilePath)) { scriptFilePath = string.Format(CultureInfo.InvariantCulture, "\"{0}\"", file); } @@ -195,6 +203,7 @@ public IList GetPostBuildActionScripts() { var files = FileSystemHelpers.GetFiles(sf, "*") .Where(f => f.EndsWith(".cmd", StringComparison.OrdinalIgnoreCase) + || f.EndsWith(".sh", StringComparison.OrdinalIgnoreCase) || f.EndsWith(".bat", StringComparison.OrdinalIgnoreCase) || f.EndsWith(".ps1", StringComparison.OrdinalIgnoreCase)).ToList(); @@ -204,5 +213,26 @@ public IList GetPostBuildActionScripts() return scriptFilesGroupedAndSorted; } + + private string GetLinuxPostBuildFilepath(DeploymentContext context, FileInfo fi) + { + if (string.Equals(".sh", fi.Extension, StringComparison.OrdinalIgnoreCase)) + { + context.Logger.Log("Add execute permission for post build script " + fi.FullName); + PermissionHelper.Chmod("ugo+x", fi.FullName, Environment, DeploymentSettings, context.Logger); + return string.Format(CultureInfo.InvariantCulture, "\"{0}\"", fi.FullName); + } + return null; + } + + private string GetWindowsPostBuildFilepath(DeploymentContext context, FileInfo fi) + { + if (string.Equals(".ps1", fi.Extension, StringComparison.OrdinalIgnoreCase)) + { + context.Logger.Log("Execute post build script with RemoteSigned policy " + fi.FullName); + return string.Format(CultureInfo.InvariantCulture, "PowerShell.exe -ExecutionPolicy RemoteSigned -File \"{0}\"", fi.FullName); + } + return null; + } } } diff --git a/Kudu.Core/Deployment/Generator/ExternalCommandFactory.cs b/Kudu.Core/Deployment/Generator/ExternalCommandFactory.cs index cf7dbc88..7791e697 100644 --- a/Kudu.Core/Deployment/Generator/ExternalCommandFactory.cs +++ b/Kudu.Core/Deployment/Generator/ExternalCommandFactory.cs @@ -75,7 +75,6 @@ public Executable BuildCommandExecutable(string commandPath, string workingDirec UpdateToDefaultIfNotSet(exe, WellKnownEnvironmentVariables.MSBuild15Dir, PathUtilityFactory.Instance.ResolveMSBuild15Dir(), logger); UpdateToDefaultIfNotSet(exe, WellKnownEnvironmentVariables.KuduSyncCommandKey, KuduSyncCommand, logger); UpdateToDefaultIfNotSet(exe, WellKnownEnvironmentVariables.NuGetExeCommandKey, NuGetExeCommand, logger); - UpdateToDefaultIfNotSet(exe, WellKnownEnvironmentVariables.NpmJsPathKey, PathUtilityFactory.Instance.ResolveNpmJsPath(), logger); UpdateToDefaultIfNotSet(exe, WellKnownEnvironmentVariables.DnvmPath, DnvmPath, logger); UpdateToDefaultIfNotSet(exe, WellKnownEnvironmentVariables.GypMsvsVersion, Constants.VCVersion, logger); UpdateToDefaultIfNotSet(exe, WellKnownEnvironmentVariables.VCTargetsPath, PathUtilityFactory.Instance.ResolveVCTargetsPath(), logger); diff --git a/Kudu.Core/Deployment/Generator/OryxBuilder.cs b/Kudu.Core/Deployment/Generator/OryxBuilder.cs index b180f6ef..ce3fc9a6 100644 --- a/Kudu.Core/Deployment/Generator/OryxBuilder.cs +++ b/Kudu.Core/Deployment/Generator/OryxBuilder.cs @@ -1,6 +1,11 @@ -using System.Threading.Tasks; +using System; +using System.IO; +using System.Threading.Tasks; +using Kudu.Core.Infrastructure; using Kudu.Core.Helpers; using Kudu.Contracts.Settings; +using Kudu.Core.Deployment.Oryx; +using Kudu.Core.K8SE; namespace Kudu.Core.Deployment.Generator { @@ -8,79 +13,165 @@ public class OryxBuilder : ExternalCommandBuilder { public override string ProjectType => "Oryx-Build"; + IEnvironment environment; + IDeploymentSettingsManager settings; + IBuildPropertyProvider propertyProvider; + string sourcePath; + public OryxBuilder(IEnvironment environment, IDeploymentSettingsManager settings, IBuildPropertyProvider propertyProvider, string sourcePath) : base(environment, settings, propertyProvider, sourcePath) { + this.environment = environment; + this.settings = settings; + this.propertyProvider = propertyProvider; + this.sourcePath = sourcePath; } public override Task Build(DeploymentContext context) { FileLogHelper.Log("In oryx build..."); - // Step 1: Run kudusync - - string kuduSyncCommand = string.Format("kudusync -v 50 -f {0} -t {1} -n {2} -p {3} -i \".git;.hg;.deployment;.deploy.sh\"", - RepositoryPath, - context.OutputPath, - context.NextManifestFilePath, - context.PreviousManifestFilePath - ); - - FileLogHelper.Log("Running KuduSync with " + kuduSyncCommand); + // initialize the repository Path for the build + context.RepositoryPath = RepositoryPath; - RunCommand(context, kuduSyncCommand, false, "Oryx-Build: Running kudu sync..."); + // Initialize Oryx Args. + IOryxArguments args = OryxArgumentsFactory.CreateOryxArguments(environment); - string framework = System.Environment.GetEnvironmentVariable("FRAMEWORK"); - string version = System.Environment.GetEnvironmentVariable("FRAMEWORK_VERSION"); + if (!args.SkipKuduSync) + { + // Step 1: Run kudusync + string kuduSyncCommand = string.Format("kudusync -v 50 -f {0} -t {1} -n {2} -p {3} -i \".git;.hg;.deployment;.deploy.sh\"", + RepositoryPath, + context.OutputPath, + context.NextManifestFilePath, + context.PreviousManifestFilePath + ); - string oryxLanguage = ""; - string additionalOptions = ""; - bool runOryxBuild = false; + FileLogHelper.Log("Running KuduSync with " + kuduSyncCommand); - if (framework.StartsWith("NODE")) - { - oryxLanguage = "nodejs"; - runOryxBuild = true; + //RunCommand(context, kuduSyncCommand, false, "Oryx-Build: Running kudu sync..."); } - else if (framework.StartsWith("PYTHON")) + + if (args.RunOryxBuild) { - oryxLanguage = "python"; - runOryxBuild = true; + PreOryxBuild(context); + + args.Flags = BuildOptimizationsFlags.UseExpressBuild; - string virtualEnvName = "antenv"; + string buildCommand = args.GenerateOryxBuildCommand(context, environment); + RunCommand(context, buildCommand, false, "Running oryx build..."); - if (version.StartsWith("3.6")) + // + // Run express build setups if needed + // + + if (args.Flags == BuildOptimizationsFlags.UseExpressBuild) { - virtualEnvName = "antenv3.6"; + if (FunctionAppHelper.LooksLikeFunctionApp()) + { + SetupFunctionAppExpressArtifacts(context); + } + else + { + ExpressBuilder appServiceExpressBuilder = new ExpressBuilder(environment, settings, propertyProvider, sourcePath); + appServiceExpressBuilder.SetupExpressBuilderArtifacts(context.OutputPath, context, args); + } } - else if (version.StartsWith("2.7")) + else { - virtualEnvName = "antenv2.7"; + Console.WriteLine("No Express :("); } - - additionalOptions = string.Format("-p virtualenv_name={0}", virtualEnvName); } + return Task.CompletedTask; + } - if (runOryxBuild) + private static void PreOryxBuild(DeploymentContext context) + { + if (FunctionAppHelper.LooksLikeFunctionApp()) { - string oryxBuildCommand = string.Format("oryx build {0} -o {1} -l {2} --language-version {3} {4}", - context.OutputPath, - context.OutputPath, - oryxLanguage, - version, - additionalOptions); + // We need to delete this directory in order to avoid issues with + // reinstalling Python dependencies on a target directory + var pythonPackagesDir = Path.Combine(context.OutputPath, ".python_packages"); + if (Directory.Exists(pythonPackagesDir)) + { + FileSystemHelpers.DeleteDirectorySafe(pythonPackagesDir); + } + } + } + + private void SetupFunctionAppExpressArtifacts(DeploymentContext context) + { + string sitePackages = "/home/data/SitePackages"; + string packageNameFile = Path.Combine(sitePackages, "packagename.txt"); + string packagePathFile = Path.Combine(sitePackages, "packagepath.txt"); + + FileSystemHelpers.EnsureDirectory(sitePackages); + + string zipAppName = $"{DateTime.UtcNow.ToString("yyyyMMddHHmmss")}.zip"; + string createdZip = PackageArtifactFromFolder(context, OryxBuildConstants.FunctionAppBuildSettings.ExpressBuildSetup, + OryxBuildConstants.FunctionAppBuildSettings.ExpressBuildSetup, zipAppName, BuildArtifactType.Zip, numBuildArtifacts: -1); - RunCommand(context, oryxBuildCommand, false, "Running oryx build..."); + var copyExe = ExternalCommandFactory.BuildExternalCommandExecutable(OryxBuildConstants.FunctionAppBuildSettings.ExpressBuildSetup, sitePackages, context.Logger); + var copyToPath = Path.Combine(sitePackages, zipAppName); + try + { + copyExe.ExecuteWithProgressWriter(context.Logger, context.Tracer, $"cp {createdZip} {copyToPath}"); + } + catch (Exception) + { + context.GlobalLogger.LogError(); + throw; } - return Task.CompletedTask; + // Gotta remove the old zips + DeploymentHelper.PurgeBuildArtifactsIfNecessary(sitePackages, BuildArtifactType.Zip, context.Tracer, totalAllowedFiles: 2); + + File.WriteAllText(packageNameFile, zipAppName); + File.WriteAllText(packagePathFile, sitePackages); } - //public override void PostBuild(DeploymentContext context) - //{ - // // no-op - // context.Logger.Log($"Skipping post build. Project type: {ProjectType}"); - // FileLogHelper.Log("Completed PostBuild oryx...."); - //} + /// + /// Package every files and sub directories from a source folder + /// + /// The deployment context in current scope + /// The source directory to be packed + /// The destination directory to eject the build artifact + /// The filename of the build artifact + /// The method for packing the artifact + /// The number of temporary artifacts should be hold in the destination directory + /// + private string PackageArtifactFromFolder(DeploymentContext context, string srcDirectory, string artifactDirectory, string artifactFilename, BuildArtifactType artifactType, int numBuildArtifacts = 0) + { + context.Logger.Log($"Writing the artifacts to a {artifactType.ToString()} file"); + string file = Path.Combine(artifactDirectory, artifactFilename); + var exe = ExternalCommandFactory.BuildExternalCommandExecutable(srcDirectory, artifactDirectory, context.Logger); + try + { + switch(artifactType) + { + case BuildArtifactType.Zip: + exe.ExecuteWithProgressWriter(context.Logger, context.Tracer, $"zip -r -0 -q {file} ."); + break; + case BuildArtifactType.Squashfs: + exe.ExecuteWithProgressWriter(context.Logger, context.Tracer, $"mksquashfs . {file} -noappend"); + break; + default: + throw new ArgumentException($"Received unknown file extension {artifactType.ToString()}"); + } + } + catch (Exception) + { + context.GlobalLogger.LogError(); + throw; + } + + // Just to be sure that we don't keep adding build artifacts here + if (numBuildArtifacts > 0) + { + DeploymentHelper.PurgeBuildArtifactsIfNecessary(artifactDirectory, artifactType, context.Tracer, numBuildArtifacts); + } + + return file; + } } -} \ No newline at end of file +} diff --git a/Kudu.Core/Deployment/Generator/SiteBuilderFactory.cs b/Kudu.Core/Deployment/Generator/SiteBuilderFactory.cs index 07360174..cdfb38e9 100644 --- a/Kudu.Core/Deployment/Generator/SiteBuilderFactory.cs +++ b/Kudu.Core/Deployment/Generator/SiteBuilderFactory.cs @@ -3,11 +3,14 @@ using System.Globalization; using System.IO; using System.Linq; +using Kudu.Contracts.Infrastructure; using Kudu.Contracts.Settings; using Kudu.Contracts.Tracing; using Kudu.Core.Infrastructure; +using Kudu.Core.K8SE; using Kudu.Core.SourceControl; using Kudu.Core.Tracing; +using Microsoft.AspNetCore.Http; namespace Kudu.Core.Deployment.Generator { @@ -16,23 +19,34 @@ public class SiteBuilderFactory : ISiteBuilderFactory private readonly IEnvironment _environment; private readonly IBuildPropertyProvider _propertyProvider; - public SiteBuilderFactory(IBuildPropertyProvider propertyProvider, IEnvironment environment) + public SiteBuilderFactory(IBuildPropertyProvider propertyProvider, IEnvironment environment, IHttpContextAccessor accessor) { _propertyProvider = propertyProvider; - _environment = environment; + _environment = GetEnvironment(accessor, environment); } - public ISiteBuilder CreateBuilder(ITracer tracer, ILogger logger, IDeploymentSettingsManager settings, IRepository repository) + private IEnvironment GetEnvironment(IHttpContextAccessor accessor, IEnvironment environment) { - string repositoryRoot = repository.RepositoryPath; + IEnvironment _environment; + if (!K8SEDeploymentHelper.IsK8SEEnvironment() || accessor==null) + { + _environment = environment; + } + else + { + var context = accessor.HttpContext; + _environment = (IEnvironment)context.Items["environment"]; + } + return _environment; + } - string enableOryxBuild = System.Environment.GetEnvironmentVariable("ENABLE_ORYX_BUILD"); - if (!string.IsNullOrEmpty(enableOryxBuild)) + public ISiteBuilder CreateBuilder(ITracer tracer, ILogger logger, IDeploymentSettingsManager settings, IRepository repository, DeploymentInfoBase deploymentInfo) + { + + string repositoryRoot = _environment.RepositoryPath; + if (!string.IsNullOrEmpty(repository.RepositoryPath)) { - if (enableOryxBuild.Equals("true", StringComparison.OrdinalIgnoreCase)) - { - return new OryxBuilder(_environment, settings, _propertyProvider, repositoryRoot); - } + repositoryRoot = repository.RepositoryPath; } // Use the cached vs projects file finder for: a. better performance, b. ignoring solutions/projects under node_modules @@ -65,13 +79,25 @@ public ISiteBuilder CreateBuilder(ITracer tracer, ILogger logger, IDeploymentSet { return new RunFromZipSiteBuilder(); } - - if (!settings.DoBuildDuringDeployment()) + + if (deploymentInfo != null && (!deploymentInfo.ShouldBuildArtifact && !settings.DoBuildDuringDeployment() && repository.RepositoryType != RepositoryType.Git)) { var projectPath = !String.IsNullOrEmpty(targetProjectPath) ? targetProjectPath : repositoryRoot; return new BasicBuilder(_environment, settings, _propertyProvider, repositoryRoot, projectPath); } + string enableOryxBuild = System.Environment.GetEnvironmentVariable("ENABLE_ORYX_BUILD"); + if (!string.IsNullOrEmpty(enableOryxBuild) && (deploymentInfo != null && deploymentInfo.ShouldBuildArtifact) || settings.DoBuildDuringDeployment()) + { + if (StringUtils.IsTrueLike(enableOryxBuild)) + { + return new OryxBuilder(_environment, settings, _propertyProvider, repositoryRoot); + } + } + + tracer.Trace("After Oryx determination."); + + if (!String.IsNullOrEmpty(targetProjectPath)) { // Try to resolve the project @@ -417,4 +443,4 @@ public IEnumerable ListFiles(string path, SearchOption searchOption, par } } } -} \ No newline at end of file +} diff --git a/Kudu.Core/Deployment/ISiteBuilderFactory.cs b/Kudu.Core/Deployment/ISiteBuilderFactory.cs index 3c421819..609849c1 100644 --- a/Kudu.Core/Deployment/ISiteBuilderFactory.cs +++ b/Kudu.Core/Deployment/ISiteBuilderFactory.cs @@ -6,6 +6,6 @@ namespace Kudu.Core.Deployment { public interface ISiteBuilderFactory { - ISiteBuilder CreateBuilder(ITracer tracer, ILogger logger, IDeploymentSettingsManager settings, IRepository fileFinder); + ISiteBuilder CreateBuilder(ITracer tracer, ILogger logger, IDeploymentSettingsManager settings, IRepository fileFinder, DeploymentInfoBase deploymentInfo); } } diff --git a/Kudu.Core/Deployment/Oryx/AppServiceOryxArguments.cs b/Kudu.Core/Deployment/Oryx/AppServiceOryxArguments.cs new file mode 100644 index 00000000..08a0ad4c --- /dev/null +++ b/Kudu.Core/Deployment/Oryx/AppServiceOryxArguments.cs @@ -0,0 +1,266 @@ +using System; +using System.Collections.Generic; +using System.Text; +using k8s; +using Kudu.Core.Deployment.Oryx; +using Kudu.Core.Helpers; +using Kudu.Core.K8SE; +using LibGit2Sharp; + +namespace Kudu.Core.Deployment +{ + public class AppServiceOryxArguments : IOryxArguments + { + public bool RunOryxBuild { get; set; } + + public BuildOptimizationsFlags Flags { get; set; } + + public Framework Language { get; set; } + + public string Version { get; set; } + + public string PublishFolder { get; set; } + + public string VirtualEnv { get; set; } + + public string AppName { get; set; } + + public AppServiceOryxArguments(IEnvironment environment) + { + RunOryxBuild = false; + SkipKuduSync = false; + string framework = ""; + string version = ""; + + if (K8SEDeploymentHelper.IsK8SEEnvironment()) + { + this.AppName = environment.K8SEAppName; + + // K8SE TODO: Inject Environment + var frameworkArr = K8SEDeploymentHelper.GetLinuxFxVersion(AppName); + framework = frameworkArr.Split("|")[0]; + version = frameworkArr.Split("|")[1]; + } + else + { + framework = System.Environment.GetEnvironmentVariable(OryxBuildConstants.OryxEnvVars.FrameworkSetting); + version = System.Environment.GetEnvironmentVariable(OryxBuildConstants.OryxEnvVars.FrameworkVersionSetting); + } + string buildFlags = System.Environment.GetEnvironmentVariable(OryxBuildConstants.OryxEnvVars.BuildFlagsSetting); + + if (string.IsNullOrEmpty(framework) || + string.IsNullOrEmpty(version)) + { + return; + } + + Language = SupportedFrameworks.ParseLanguage(framework); + if (Language == Framework.None) + { + return; + } + else if (Language == Framework.DotNETCore) + { + // Skip kudu sync for .NET core builds + SkipKuduSync = true; + } + + RunOryxBuild = true; + Version = version; + + // Parse Build Flags + Flags = BuildFlagsHelper.Parse(buildFlags); + + // Set language specific + SetLanguageOptions(); + } + + private void SetLanguageOptions() + { + switch(Language) + { + case Framework.None: + return; + + case Framework.Python: + SetVirtualEnvironment(); + // For python, enable compress option by default + if (Flags == BuildOptimizationsFlags.None) + { + Flags = BuildOptimizationsFlags.CompressModules; + } + return; + + case Framework.DotNETCore: + if (Flags == BuildOptimizationsFlags.None) + { + Flags = BuildOptimizationsFlags.UseTempDirectory; + } + + return; + + case Framework.NodeJs: + // For node, enable compress option by default + if (Flags == BuildOptimizationsFlags.None) + { + Flags = BuildOptimizationsFlags.CompressModules; + } + + return; + + case Framework.PHP: + return; + } + } + + private void SetVirtualEnvironment() + { + string virtualEnvName = "antenv"; + if (Version.StartsWith("3.6")) + { + virtualEnvName = "antenv3.6"; + } + else if (Version.StartsWith("2.7")) + { + virtualEnvName = "antenv2.7"; + } + + VirtualEnv = virtualEnvName; + } + + public string GenerateOryxBuildCommand(DeploymentContext context, IEnvironment environment) + { + StringBuilder args = new StringBuilder(); + // Language + switch (Language) + { + case Framework.None: + // Input/Output + OryxArgumentsHelper.AddOryxBuildCommand(args, source: context.OutputPath, destination: context.OutputPath); + break; + + case Framework.NodeJs: + // Input/Output + OryxArgumentsHelper.AddOryxBuildCommand(args, source: context.RepositoryPath, destination: context.BuildTempPath); + OryxArgumentsHelper.AddLanguage(args, "nodejs"); + break; + + case Framework.Python: + // Input/Output + OryxArgumentsHelper.AddOryxBuildCommand(args, source: context.RepositoryPath, destination: context.BuildTempPath); + OryxArgumentsHelper.AddLanguage(args, "python"); + break; + + case Framework.DotNETCore: + if (Flags == BuildOptimizationsFlags.UseExpressBuild) + { + // We don't want to copy the built artifacts to wwwroot for ExpressBuild scenario + OryxArgumentsHelper.AddOryxBuildCommand(args, source: context.RepositoryPath, destination: context.BuildTempPath); + } + else + { + // Input/Output [For .NET core, the source path is the RepositoryPath] + OryxArgumentsHelper.AddOryxBuildCommand(args, source: context.RepositoryPath, destination: context.OutputPath); + } + OryxArgumentsHelper.AddLanguage(args, "dotnet"); + break; + + case Framework.PHP: + // Input/Output + OryxArgumentsHelper.AddOryxBuildCommand(args, source: context.RepositoryPath, destination: context.OutputPath); + OryxArgumentsHelper.AddLanguage(args, "php"); + break; + } + + // Version + switch (Language) + { + case Framework.None: + break; + case Framework.PHP: + case Framework.NodeJs: + if (Version.Contains("LTS", StringComparison.OrdinalIgnoreCase)) + { + // 10-LTS, 12-LTS should use versions 10, 12 etc + // Oryx Builder uses lts for major versions + Version = Version.Replace("LTS", "").Replace("lts", "").Replace("-", ""); + if (string.IsNullOrEmpty(Version)) + { + // Current LTS + Version = "10"; + } + OryxArgumentsHelper.AddLanguageVersion(args, Version); + } + break; + case Framework.Python: + OryxArgumentsHelper.AddLanguageVersion(args, Version); + break; + + // work around issue regarding sdk version vs runtime version + case Framework.DotNETCore: + if (Version == "1.0") + { + Version = "1.1"; + } + else if (Version == "2.0") + { + Version = "2.1"; + } + else if (Version == "3.0") + { + Version = "3.1"; + } + + OryxArgumentsHelper.AddLanguageVersion(args, Version); + break; + + default: + break; + } + + // Build Flags + switch (Flags) + { + case BuildOptimizationsFlags.Off: + case BuildOptimizationsFlags.None: + break; + + case BuildOptimizationsFlags.CompressModules: + OryxArgumentsHelper.AddTempDirectoryOption(args, context.BuildTempPath); + if (Language == Framework.NodeJs) + { + OryxArgumentsHelper.AddNodeCompressOption(args, "tar-gz"); + } + else if (Language == Framework.Python) + { + OryxArgumentsHelper.AddPythonCompressOption(args, "tar-gz"); + } + + break; + + case BuildOptimizationsFlags.UseExpressBuild: + OryxArgumentsHelper.AddTempDirectoryOption(args, context.BuildTempPath); + break; + + case BuildOptimizationsFlags.UseTempDirectory: + OryxArgumentsHelper.AddTempDirectoryOption(args, context.BuildTempPath); + break; + case BuildOptimizationsFlags.UseK8SquashFs: + OryxArgumentsHelper.AddTempDirectoryOption(args, context.BuildTempPath); + break; + } + + // Virtual Env? + if (!String.IsNullOrEmpty(VirtualEnv)) + { + OryxArgumentsHelper.AddPythonVirtualEnv(args, VirtualEnv); + } + // OryxArgumentsHelper.AddDebugLog(args); + + return args.ToString(); + } + + public bool SkipKuduSync { get; set; } + + } +} diff --git a/Kudu.Core/Deployment/Oryx/BuildArtifactType.cs b/Kudu.Core/Deployment/Oryx/BuildArtifactType.cs new file mode 100644 index 00000000..65050bc8 --- /dev/null +++ b/Kudu.Core/Deployment/Oryx/BuildArtifactType.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; + +namespace Kudu.Core.Deployment.Oryx +{ + public enum BuildArtifactType + { + Zip, + Squashfs + } +} diff --git a/Kudu.Core/Deployment/Oryx/BuildFlags.cs b/Kudu.Core/Deployment/Oryx/BuildFlags.cs new file mode 100644 index 00000000..df326609 --- /dev/null +++ b/Kudu.Core/Deployment/Oryx/BuildFlags.cs @@ -0,0 +1,37 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Kudu.Core.Deployment +{ + public enum BuildOptimizationsFlags + { + Off, + None, + CompressModules, + UseExpressBuild, + UseTempDirectory, + UseK8SquashFs + } + + public class BuildFlagsHelper + { + public static BuildOptimizationsFlags Parse(string value) + { + if (string.IsNullOrEmpty(value)) + { + return BuildOptimizationsFlags.None; + } + + try + { + var result = (BuildOptimizationsFlags)Enum.Parse(typeof(BuildOptimizationsFlags), value); + return result; + } + catch (Exception) + { + return BuildOptimizationsFlags.None; + } + } + } +} diff --git a/Kudu.Core/Deployment/Oryx/ExpressBuilder.cs b/Kudu.Core/Deployment/Oryx/ExpressBuilder.cs new file mode 100644 index 00000000..aa947d7f --- /dev/null +++ b/Kudu.Core/Deployment/Oryx/ExpressBuilder.cs @@ -0,0 +1,148 @@ +using Kudu.Contracts.Settings; +using Kudu.Core.Deployment.Generator; +using Kudu.Core.Infrastructure; +using Kudu.Core.K8SE; +using System; +using System.Diagnostics; +using System.IO; +using System.Threading.Tasks; + +namespace Kudu.Core.Deployment.Oryx +{ + public class ExpressBuilder : ExternalCommandBuilder + { + public override string ProjectType => "OryxBuild"; + public IEnvironment environment; + IDeploymentSettingsManager settings; + + public ExpressBuilder(IEnvironment environment, IDeploymentSettingsManager settings, IBuildPropertyProvider propertyProvider, string sourcePath) + : base(environment, settings, propertyProvider, sourcePath) + { + this.environment = environment; + this.settings = settings; + } + + public void SetupExpressBuilderArtifacts(string outputPath, DeploymentContext context, IOryxArguments args) + { + if(args.Flags != BuildOptimizationsFlags.UseExpressBuild) + { + return; + } + + if (K8SEDeploymentHelper.IsK8SEEnvironment()) + { + SetupK8Artifacts(context, environment, outputPath); + } + else + { + + string sitePackagesDir = "/home/data/SitePackages"; + string packageNameFile = Path.Combine(sitePackagesDir, "packagename.txt"); + string packagePathFile = Path.Combine(sitePackagesDir, "packagepath.txt"); + + FileSystemHelpers.EnsureDirectory(sitePackagesDir); + + string packageName = ""; + + if (args.Language == Framework.NodeJs) + { + // For App service express mode + // Generate packagename.txt and packagepath + //packageName = "node_modules.zip:/node_modules"; + //SetupNodeAppExpressArtifacts(context, sitePackagesDir, outputPath); + } + else if (args.Language == Framework.Python) + { + packageName = $"{args.VirtualEnv}.zip:/home/site/wwwroot/{args.VirtualEnv}"; + } + else if (args.Language == Framework.DotNETCore) + { + // store the zipped artifacts at site packages dir + string artifactName = SetupNetCoreAppExpressArtifacts(context, sitePackagesDir, outputPath); + packageName = $"{artifactName:/home/site/wwwroot}"; + } + + File.WriteAllText(packageNameFile, packageName); + File.WriteAllText(packagePathFile, outputPath); + } + } + + public string SetupK8Artifacts(DeploymentContext context, IEnvironment environment, string outputPath) + { + context.Logger.Log($"Building for K8."); + + // Create NetCore Zip at tm build folder where artifact were build and copy it to sitePackages, .GetBranch() + string zipAppName = $"artifact.zip"; + + context.Logger.Log($"From {context.BuildTempPath} to {(environment.RepositoryPath + "/../artifacts/" + environment.CurrId)} "); + FileSystemHelpers.EnsureDirectory(environment.RepositoryPath + "/../artifacts/"); + FileSystemHelpers.EnsureDirectory(environment.RepositoryPath + "/../artifacts/" + environment.CurrId); + string createdZip = PackageArtifactFromFolder(context, context.BuildTempPath, environment.RepositoryPath + "/../artifacts/" + environment.CurrId, zipAppName, BuildArtifactType.Squashfs, numBuildArtifacts: -1); + return zipAppName; + } + + private string SetupNetCoreAppExpressArtifacts(DeploymentContext context, string sitePackages,string outputPath) + { + context.Logger.Log($"Express Build enabled for NETCore app"); + + // Create NetCore Zip at tm build folder where artifact were build and copy it to sitePackages + string zipAppName = $"{DateTime.UtcNow.ToString("yyyyMMddHHmmss")}.zip"; + + string createdZip = PackageArtifactFromFolder(context, context.BuildTempPath, outputPath, zipAppName, BuildArtifactType.Zip, numBuildArtifacts: -1); + + // Remove the old zips + DeploymentHelper.PurgeBuildArtifactsIfNecessary(sitePackages, BuildArtifactType.Zip, context.Tracer, totalAllowedFiles: 2); + + return zipAppName; + } + + /// + /// Package every files and sub directories from a source folder + /// + /// The deployment context in current scope + /// The source directory to be packed + /// The destination directory to eject the build artifact + /// The filename of the build artifact + /// The method for packing the artifact + /// The number of temporary artifacts should be hold in the destination directory + /// + private string PackageArtifactFromFolder(DeploymentContext context, string srcDirectory, string artifactDirectory, string artifactFilename, BuildArtifactType artifactType, int numBuildArtifacts = 0) + { + context.Logger.Log($"Writing the artifacts to {artifactType.ToString()} file at {artifactDirectory}"); + string file = Path.Combine(artifactDirectory, artifactFilename); + var exe = ExternalCommandFactory.BuildExternalCommandExecutable(srcDirectory, artifactDirectory, context.Logger); + try + { + switch (artifactType) + { + case BuildArtifactType.Zip: + exe.ExecuteWithProgressWriter(context.Logger, context.Tracer, $"zip -r -0 -q {file} ."); + break; + case BuildArtifactType.Squashfs: + exe.ExecuteWithProgressWriter(context.Logger, context.Tracer, $"mksquashfs . {file} -noappend"); + break; + default: + throw new ArgumentException($"Received unknown file extension {artifactType.ToString()}"); + } + } + catch (Exception) + { + context.GlobalLogger.LogError(); + throw; + } + + // Just to be sure that we don't keep adding build artifacts here + if (numBuildArtifacts > 0) + { + DeploymentHelper.PurgeBuildArtifactsIfNecessary(artifactDirectory, artifactType, context.Tracer, numBuildArtifacts); + } + + return file; + } + + public override Task Build(DeploymentContext context) + { + throw new NotImplementedException(); + } + } +} diff --git a/Kudu.Core/Deployment/Oryx/FunctionAppOryxArguments.cs b/Kudu.Core/Deployment/Oryx/FunctionAppOryxArguments.cs new file mode 100644 index 00000000..1bb77449 --- /dev/null +++ b/Kudu.Core/Deployment/Oryx/FunctionAppOryxArguments.cs @@ -0,0 +1,147 @@ +using Kudu.Core.Infrastructure; +using System.IO; +using System.Text; + +namespace Kudu.Core.Deployment.Oryx +{ + public class FunctionAppOryxArguments : IOryxArguments + { + public bool RunOryxBuild { get; set; } + + public BuildOptimizationsFlags Flags { get; set; } + + protected readonly WorkerRuntime FunctionsWorkerRuntime; + public bool SkipKuduSync { get; set; } + public string Version { get; set; } + public Framework Language { get; set; } + public string PublishFolder { get; set; } + public string VirtualEnv { get; set; } + public string AppName { get => throw new System.NotImplementedException(); set => throw new System.NotImplementedException(); } + + public FunctionAppOryxArguments(IEnvironment environment) + { + SkipKuduSync = false; + FunctionsWorkerRuntime = ResolveWorkerRuntime(); + RunOryxBuild = FunctionsWorkerRuntime != WorkerRuntime.None; + var buildFlags = GetEnvironmentVariableOrNull(OryxBuildConstants.OryxEnvVars.BuildFlagsSetting); + Flags = BuildFlagsHelper.Parse(buildFlags); + SkipKuduSync = Flags == BuildOptimizationsFlags.UseExpressBuild; + } + + public virtual string GenerateOryxBuildCommand(DeploymentContext context, IEnvironment environment) + { + StringBuilder args = new StringBuilder(); + + AddOryxBuildCommand(args, context, source: context.OutputPath, destination: context.OutputPath); + AddLanguage(args, FunctionsWorkerRuntime); + AddLanguageVersion(args, FunctionsWorkerRuntime); + AddBuildOptimizationFlags(args, context, Flags); + AddWorkerRuntimeArgs(args, FunctionsWorkerRuntime); + + return args.ToString(); + } + + protected void AddOryxBuildCommand(StringBuilder args, DeploymentContext context, string source, string destination) + { + // If it is express build, we don't directly need to write to /home/site/wwwroot + // So, we build into a different directory to avoid overlap + // Additionally, we didn't run kudusync, and can just build directly from repository path + if (Flags == BuildOptimizationsFlags.UseExpressBuild) + { + source = context.RepositoryPath; + destination = OryxBuildConstants.FunctionAppBuildSettings.ExpressBuildSetup; + // It is important to clean and recreate the directory to make sure no overwrite occurs + if (FileSystemHelpers.DirectoryExists(destination)) + { + FileSystemHelpers.DeleteDirectorySafe(destination); + } + FileSystemHelpers.EnsureDirectory(destination); + } + OryxArgumentsHelper.AddOryxBuildCommand(args, source, destination); + } + + protected void AddLanguage(StringBuilder args, WorkerRuntime workerRuntime) + { + switch (workerRuntime) + { + case WorkerRuntime.DotNet: + Language = Framework.DotNETCore; + OryxArgumentsHelper.AddLanguage(args, "dotnet"); + break; + + case WorkerRuntime.Node: + Language = Framework.NodeJs; + OryxArgumentsHelper.AddLanguage(args, "nodejs"); + break; + + case WorkerRuntime.Python: + Language = Framework.Python; + OryxArgumentsHelper.AddLanguage(args, "python"); + break; + + case WorkerRuntime.PHP: + Language = Framework.PHP; + OryxArgumentsHelper.AddLanguage(args, "php"); + break; + } + } + + protected void AddLanguageVersion(StringBuilder args, WorkerRuntime workerRuntime) + { + var workerVersion = ResolveWorkerRuntimeVersion(FunctionsWorkerRuntime); + if (!string.IsNullOrEmpty(workerVersion)) + { + Version = workerVersion; + OryxArgumentsHelper.AddLanguageVersion(args, workerVersion); + } + } + + protected void AddBuildOptimizationFlags(StringBuilder args, DeploymentContext context, BuildOptimizationsFlags optimizationFlags) + { + switch (Flags) + { + // By default, we always want to use the temp directory path + // However, it's good to have an off option here. + // Ideally, we would always use ExpressBuild, as that also performs run from package + case BuildOptimizationsFlags.Off: + break; + case BuildOptimizationsFlags.None: + case BuildOptimizationsFlags.CompressModules: + case BuildOptimizationsFlags.UseExpressBuild: + OryxArgumentsHelper.AddTempDirectoryOption(args, context.BuildTempPath); + break; + } + } + + protected void AddWorkerRuntimeArgs(StringBuilder args, WorkerRuntime workerRuntime) + { + switch (workerRuntime) + { + case WorkerRuntime.Python: + OryxArgumentsHelper.AddPythonPackageDir(args, OryxBuildConstants.FunctionAppBuildSettings.PythonPackagesTargetDir); + break; + } + } + + private WorkerRuntime ResolveWorkerRuntime() + { + var functionsWorkerRuntimeStr = GetEnvironmentVariableOrNull(OryxBuildConstants.FunctionAppEnvVars.WorkerRuntimeSetting); + return FunctionAppSupportedWorkerRuntime.ParseWorkerRuntime(functionsWorkerRuntimeStr); + } + + private string ResolveWorkerRuntimeVersion(WorkerRuntime workerRuntime) + { + return FunctionAppSupportedWorkerRuntime.GetDefaultLanguageVersion(workerRuntime); + } + + private string GetEnvironmentVariableOrNull(string environmentVarName) + { + var environmentVarValue = System.Environment.GetEnvironmentVariable(environmentVarName); + if (string.IsNullOrEmpty(environmentVarValue)) + { + return null; + } + return environmentVarValue; + } + } +} diff --git a/Kudu.Core/Deployment/Oryx/FunctionAppSupportedWorkerRuntime.cs b/Kudu.Core/Deployment/Oryx/FunctionAppSupportedWorkerRuntime.cs new file mode 100644 index 00000000..513a4b5b --- /dev/null +++ b/Kudu.Core/Deployment/Oryx/FunctionAppSupportedWorkerRuntime.cs @@ -0,0 +1,63 @@ +using System; + +namespace Kudu.Core.Deployment.Oryx +{ + public enum WorkerRuntime + { + None, + Node, + Python, + DotNet, + PHP + } + + public class FunctionAppSupportedWorkerRuntime + { + public static WorkerRuntime ParseWorkerRuntime(string value) + { + if (string.IsNullOrEmpty(value)) + { + return WorkerRuntime.None; + } + else if (value.StartsWith("NODE", StringComparison.OrdinalIgnoreCase)) + { + return WorkerRuntime.Node; + } + else if (value.StartsWith("PYTHON", StringComparison.OrdinalIgnoreCase)) + { + return WorkerRuntime.Python; + } + else if (value.StartsWith("DOTNET", StringComparison.OrdinalIgnoreCase)) + { + return WorkerRuntime.DotNet; + } + else if (value.StartsWith("PHP", StringComparison.OrdinalIgnoreCase)) + { + return WorkerRuntime.PHP; + } + + return WorkerRuntime.None; + } + + public static string GetDefaultLanguageVersion(WorkerRuntime workerRuntime) + { + switch (workerRuntime) + { + case WorkerRuntime.DotNet: + return OryxBuildConstants.FunctionAppWorkerRuntimeDefaults.Dotnet; + + case WorkerRuntime.Node: + return OryxBuildConstants.FunctionAppWorkerRuntimeDefaults.Node; + + case WorkerRuntime.Python: + return OryxBuildConstants.FunctionAppWorkerRuntimeDefaults.Python; + + case WorkerRuntime.PHP: + return OryxBuildConstants.FunctionAppWorkerRuntimeDefaults.PHP; + + default: + return ""; + } + } + } +} diff --git a/Kudu.Core/Deployment/Oryx/IOryxArguments.cs b/Kudu.Core/Deployment/Oryx/IOryxArguments.cs new file mode 100644 index 00000000..8d7d7dff --- /dev/null +++ b/Kudu.Core/Deployment/Oryx/IOryxArguments.cs @@ -0,0 +1,25 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Kudu.Core.Deployment.Oryx +{ + public interface IOryxArguments + { + bool RunOryxBuild { get; set; } + + BuildOptimizationsFlags Flags { get; set; } + + bool SkipKuduSync { get; set; } + + string GenerateOryxBuildCommand(DeploymentContext context, IEnvironment environment); + + string Version { get; set; } + + Framework Language { get; set; } + + string PublishFolder { get; set; } + string VirtualEnv { get; set; } + string AppName { get; set; } + } +} diff --git a/Kudu.Core/Deployment/Oryx/LinuxConsumptionFunctionAppOryxArguments.cs b/Kudu.Core/Deployment/Oryx/LinuxConsumptionFunctionAppOryxArguments.cs new file mode 100644 index 00000000..64ca0195 --- /dev/null +++ b/Kudu.Core/Deployment/Oryx/LinuxConsumptionFunctionAppOryxArguments.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Kudu.Core.Deployment.Oryx +{ + public class LinuxConsumptionFunctionAppOryxArguments : FunctionAppOryxArguments + { + public LinuxConsumptionFunctionAppOryxArguments(IEnvironment env) : base(env) + { + SkipKuduSync = true; + Flags = BuildOptimizationsFlags.Off; + } + + public override string GenerateOryxBuildCommand(DeploymentContext context, IEnvironment environment) + { + StringBuilder args = new StringBuilder(); + + base.AddOryxBuildCommand(args, context, source: context.RepositoryPath, destination: context.OutputPath); + base.AddLanguage(args, base.FunctionsWorkerRuntime); + base.AddLanguageVersion(args, base.FunctionsWorkerRuntime); + base.AddBuildOptimizationFlags(args, context, Flags); + base.AddWorkerRuntimeArgs(args, base.FunctionsWorkerRuntime); + + return args.ToString(); + } + } +} diff --git a/Kudu.Core/Deployment/Oryx/OryxArguments.cs b/Kudu.Core/Deployment/Oryx/OryxArguments.cs new file mode 100644 index 00000000..f356c053 --- /dev/null +++ b/Kudu.Core/Deployment/Oryx/OryxArguments.cs @@ -0,0 +1,188 @@ +using System; +using System.Collections.Generic; +using System.Text; +using Kudu.Core.Deployment.Oryx; + +namespace Kudu.Core.Deployment +{ + public class OryxArguments + { + public bool RunOryxBuild { get; set; } + + public Framework Language { get; set; } + + public string Version { get; set; } + + public string PublishFolder { get; set; } + + public string VirtualEnv { get; private set; } + + public BuildOptimizationsFlags Flags { get; set; } + + public OryxArguments() + { + RunOryxBuild = false; + + string framework = System.Environment.GetEnvironmentVariable("FRAMEWORK"); + string version = System.Environment.GetEnvironmentVariable("FRAMEWORK_VERSION"); + string buildFlags = System.Environment.GetEnvironmentVariable("BUILD_FLAGS"); + + if (string.IsNullOrEmpty(framework) || + string.IsNullOrEmpty(version)) + { + return; + } + + Language = SupportedFrameworks.ParseLanguage(framework); + if (Language == Framework.None) + { + return; + } + + RunOryxBuild = true; + Version = version; + + // Parse Build Flags + Flags = BuildFlagsHelper.Parse(buildFlags); + + // Set language specific + SetLanguageOptions(); + } + + private void SetLanguageOptions() + { + switch(Language) + { + case Framework.None: + return; + + case Framework.Python: + SetVirtualEnvironment(); + return; + + case Framework.DotNETCore: + return; + + case Framework.NodeJs: + // For node, enable compress option by default + if (Flags == BuildOptimizationsFlags.None) + { + Flags = BuildOptimizationsFlags.CompressModules; + } + + return; + + case Framework.PHP: + return; + } + } + + private void SetVirtualEnvironment() + { + string virtualEnvName = "antenv"; + if (Version.StartsWith("3.6")) + { + virtualEnvName = "antenv3.6"; + } + else if (Version.StartsWith("2.7")) + { + virtualEnvName = "antenv2.7"; + } + + VirtualEnv = virtualEnvName; + } + + public string GenerateOryxBuildCommand(DeploymentContext context) + { + StringBuilder args = new StringBuilder(); + + // Input/Output + args.AppendFormat("oryx build {0} -o {1}", context.OutputPath, context.OutputPath); + + // Language + switch (Language) + { + case Framework.None: + break; + + case Framework.NodeJs: + args.AppendFormat(" --platform nodejs"); + break; + + case Framework.Python: + args.AppendFormat(" --platform python"); + break; + + case Framework.DotNETCore: + args.AppendFormat(" --platform dotnet"); + break; + + case Framework.PHP: + args.AppendFormat(" --platform php"); + break; + } + + // Version + args.AppendFormat(" --platform-version {0}", Version); + + // Build Flags + switch (Flags) + { + case BuildOptimizationsFlags.Off: + case BuildOptimizationsFlags.None: + break; + + case BuildOptimizationsFlags.CompressModules: + AddTempDirectoryOption(args, context.BuildTempPath); + if (Language == Framework.NodeJs) + { + AddNodeCompressOption(args, "tar-gz"); + } + else if (Language == Framework.Python) + { + AddPythonCompressOption(args); + } + + break; + + case BuildOptimizationsFlags.UseExpressBuild: + AddTempDirectoryOption(args, context.BuildTempPath); + if (Language == Framework.NodeJs) + { + AddNodeCompressOption(args, "zip"); + } + + break; + } + + // Virtual Env? + if (!String.IsNullOrEmpty(VirtualEnv)) + { + args.AppendFormat(" -p virtualenv_name={0}", VirtualEnv); + } + + // Publish Output? + if (!String.IsNullOrEmpty(PublishFolder)) + { + args.AppendFormat(" -publishedOutputPath {0}", PublishFolder); + } + + return args.ToString(); + } + + private static void AddTempDirectoryOption(StringBuilder args, string tempDir) + { + args.AppendFormat(" -i {0}", tempDir); + } + + private static void AddNodeCompressOption(StringBuilder args, string format) + { + args.AppendFormat(" -p compress_node_modules={0}", format); + } + + private static void AddPythonCompressOption(StringBuilder args) + { + args.AppendFormat(" -p zip_venv_dir"); + } + } +} diff --git a/Kudu.Core/Deployment/Oryx/OryxArgumentsFactory.cs b/Kudu.Core/Deployment/Oryx/OryxArgumentsFactory.cs new file mode 100644 index 00000000..768b16c7 --- /dev/null +++ b/Kudu.Core/Deployment/Oryx/OryxArgumentsFactory.cs @@ -0,0 +1,21 @@ +using Kudu.Core.Infrastructure; + +namespace Kudu.Core.Deployment.Oryx +{ + public class OryxArgumentsFactory + { + public static IOryxArguments CreateOryxArguments(IEnvironment environment) + { + if (FunctionAppHelper.LooksLikeFunctionApp()) + { + if (FunctionAppHelper.HasScmRunFromPackage()) + { + return new LinuxConsumptionFunctionAppOryxArguments(environment); + } else { + return new FunctionAppOryxArguments(environment); + } + } + return new AppServiceOryxArguments(environment); + } + } +} diff --git a/Kudu.Core/Deployment/Oryx/OryxArgumentsHelper.cs b/Kudu.Core/Deployment/Oryx/OryxArgumentsHelper.cs new file mode 100644 index 00000000..098f202e --- /dev/null +++ b/Kudu.Core/Deployment/Oryx/OryxArgumentsHelper.cs @@ -0,0 +1,59 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Kudu.Core.Deployment.Oryx +{ + internal static class OryxArgumentsHelper + { + internal static void AddOryxBuildCommand(StringBuilder args, string source, string destination) + { + args.AppendFormat("oryx build {0} -o {1}", source, destination); + } + + internal static void AddLanguage(StringBuilder args, string language) + { + args.AppendFormat(" --platform {0}", language); + } + + internal static void AddLanguageVersion(StringBuilder args, string languageVer) + { + args.AppendFormat(" --platform-version {0}", languageVer); + } + + internal static void AddTempDirectoryOption(StringBuilder args, string tempDir) + { + args.AppendFormat(" -i {0}", tempDir); + } + + internal static void AddNodeCompressOption(StringBuilder args, string format) + { + args.AppendFormat(" -p compress_node_modules={0}", format); + } + + internal static void AddPythonCompressOption(StringBuilder args, string format) + { + args.AppendFormat(" -p compress_virtualenv={0}", format); + } + + internal static void AddPythonVirtualEnv(StringBuilder args, string virtualEnv) + { + args.AppendFormat(" -p virtualenv_name={0}", virtualEnv); + } + + internal static void AddPythonPackageDir(StringBuilder args, string packageDir) + { + args.AppendFormat(" -p packagedir={0}", packageDir); + } + + internal static void AddPublishedOutputPath(StringBuilder args, string path) + { + args.AppendFormat(" -publishedOutputPath {0}", path); + } + + internal static void AddDebugLog(StringBuilder args) + { + args.AppendFormat(" --log-file /tmp/test.log "); + } + } +} diff --git a/Kudu.Core/Deployment/Oryx/OryxBuildConstants.cs b/Kudu.Core/Deployment/Oryx/OryxBuildConstants.cs new file mode 100644 index 00000000..44d817ac --- /dev/null +++ b/Kudu.Core/Deployment/Oryx/OryxBuildConstants.cs @@ -0,0 +1,40 @@ +using System.IO; + +namespace Kudu.Core.Deployment.Oryx +{ + internal static class OryxBuildConstants + { + internal static readonly string EnableOryxBuild = "ENABLE_ORYX_BUILD"; + + internal static class OryxEnvVars + { + public static readonly string FrameworkSetting = "FRAMEWORK"; + public static readonly string FrameworkVersionSetting = "FRAMEWORK_VERSION"; + public static readonly string BuildFlagsSetting = "BUILD_FLAGS"; + } + + internal static class FunctionAppEnvVars + { + public static readonly string WorkerRuntimeSetting = "FUNCTIONS_WORKER_RUNTIME"; + } + + internal static class FunctionAppWorkerRuntimeDefaults + { + public static readonly string Node = "8.15"; + public static readonly string Python = "3.6"; + public static readonly string Dotnet = "3.1"; + public static readonly string PHP = "7.3"; + } + + internal static class FunctionAppBuildSettings + { + public static readonly string ExpressBuildSetup = "/tmp/build/expressbuild"; + public static readonly string LinuxConsumptionArtifactName = "functionappartifact.squashfs"; + public static readonly string PythonPackagesTargetDir = Path.Combine(".python_packages", "lib", "python3.6", "site-packages"); + + // Determine how many built files should be kept in the container + public static readonly int ExpressBuildMaxFiles = 3; + public static readonly int ConsumptionBuildMaxFiles = 1; + } + } +} diff --git a/Kudu.Core/Deployment/Oryx/OryxDeploymentContext.cs b/Kudu.Core/Deployment/Oryx/OryxDeploymentContext.cs new file mode 100644 index 00000000..a1432ec7 --- /dev/null +++ b/Kudu.Core/Deployment/Oryx/OryxDeploymentContext.cs @@ -0,0 +1,21 @@ + +namespace Kudu.Core.Deployment.Oryx +{ + class OryxDeploymentContext + { + public bool RunOryxBuild { get; set; } + + public BuildOptimizationsFlags Flags { get; set; } + + public Framework Language { get; set; } + + public string Version { get; set; } + + public string PublishFolder { get; set; } + + public string VirtualEnv { get; set; } + + public bool SkipKuduSync { get; set; } + + } +} diff --git a/Kudu.Core/Deployment/Oryx/SupportedLanguages.cs b/Kudu.Core/Deployment/Oryx/SupportedLanguages.cs new file mode 100644 index 00000000..622359e0 --- /dev/null +++ b/Kudu.Core/Deployment/Oryx/SupportedLanguages.cs @@ -0,0 +1,38 @@ +using System; + +namespace Kudu.Core.Deployment.Oryx +{ + public enum Framework + { + None, + NodeJs, + Python, + DotNETCore, + PHP + } + + public class SupportedFrameworks + { + public static Framework ParseLanguage(string value) + { + if (value.StartsWith("NODE", StringComparison.OrdinalIgnoreCase)) + { + return Framework.NodeJs; + } + else if (value.StartsWith("PYTHON", StringComparison.OrdinalIgnoreCase)) + { + return Framework.Python; + } + else if (value.StartsWith("DOTNETCORE", StringComparison.OrdinalIgnoreCase)) + { + return Framework.DotNETCore; + } + else if (value.StartsWith("PHP", StringComparison.OrdinalIgnoreCase)) + { + return Framework.PHP; + } + + return Framework.None; + } + } +} diff --git a/Kudu.Core/Deployment/ProgressLogger.cs b/Kudu.Core/Deployment/ProgressLogger.cs index f68c0a09..c1057bb3 100644 --- a/Kudu.Core/Deployment/ProgressLogger.cs +++ b/Kudu.Core/Deployment/ProgressLogger.cs @@ -7,17 +7,19 @@ public class ProgressLogger : ILogger private readonly string _id; private readonly IDeploymentStatusManager _status; private readonly ILogger _innerLogger; + private readonly IEnvironment _environment; - public ProgressLogger(string id, IDeploymentStatusManager status, ILogger innerLogger) + public ProgressLogger(string id, IDeploymentStatusManager status, ILogger innerLogger, IEnvironment environment) { _id = id; _status = status; _innerLogger = innerLogger; + _environment = environment; } public ILogger Log(string value, LogEntryType type) { - IDeploymentStatusFile statusFile = _status.Open(_id); + IDeploymentStatusFile statusFile = _status.Open(_id, _environment); if (statusFile != null) { statusFile.UpdateProgress(value); diff --git a/Kudu.Core/Deployment/ZipDeploymentInfo.cs b/Kudu.Core/Deployment/ZipDeploymentInfo.cs index d336ff35..e37eee8d 100644 --- a/Kudu.Core/Deployment/ZipDeploymentInfo.cs +++ b/Kudu.Core/Deployment/ZipDeploymentInfo.cs @@ -32,5 +32,8 @@ public override IRepository GetRepository() // This is used if the deployment is Run-From-Zip public string ZipName { get; set; } + + // This is used when getting the zipfile from the zipURL + public string ZipURL { get; set; } } } diff --git a/Kudu.Core/Environment.cs b/Kudu.Core/Environment.cs index 56d33e67..89ed2d0a 100644 --- a/Kudu.Core/Environment.cs +++ b/Kudu.Core/Environment.cs @@ -10,6 +10,8 @@ using Microsoft.Win32; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http.Extensions; +using Kudu.Core.Settings; +using Kudu.Core.K8SE; namespace Kudu.Core { @@ -39,6 +41,10 @@ public class Environment : IEnvironment private readonly string _jobsBinariesPath; private readonly string _sitePackagesPath; private readonly string _secondaryJobsBinariesPath; + private readonly string _k8seAppName; + private readonly string _k8seAppNamespace; + private readonly string _k8seAppType; + // This ctor is used only in unit tests public Environment( @@ -58,7 +64,10 @@ public Environment( string siteExtensionSettingsPath, string sitePackagesPath, string requestId, - IHttpContextAccessor httpContextAccessor) + IHttpContextAccessor httpContextAccessor, + string k8seAppName = null, + string k8seAppNamespace = null, + string k8seAppType = null) { if (repositoryPath == null) { @@ -77,8 +86,8 @@ public Environment( _diagnosticsPath = diagnosticsPath; _locksPath = locksPath; _sshKeyPath = sshKeyPath; - Console.WriteLine("Root Path : "+rootPath); - Console.WriteLine("SSH Key Path : "+_sshKeyPath); + Console.WriteLine("Root Path : " + rootPath); + Console.WriteLine("SSH Key Path : " + _sshKeyPath); _scriptPath = scriptPath; _nodeModulesPath = nodeModulesPath; @@ -97,6 +106,9 @@ public Environment( RequestId = !string.IsNullOrEmpty(requestId) ? requestId : Guid.Empty.ToString(); _httpContextAccessor = httpContextAccessor; + _k8seAppName = k8seAppName; + _k8seAppNamespace = k8seAppNamespace; + _k8seAppType = k8seAppType; } public Environment( @@ -105,7 +117,10 @@ public Environment( string repositoryPath, string requestId, string kuduConsoleFullPath, - IHttpContextAccessor httpContextAccessor) + IHttpContextAccessor httpContextAccessor, + string k8seAppName = null, + string k8seAppNamespace = null, + string k8seAppType = null) { RootPath = rootPath; @@ -120,7 +135,10 @@ public Environment( _siteExtensionSettingsPath = Path.Combine(SiteRootPath, Constants.SiteExtensionsCachePath); _diagnosticsPath = Path.Combine(SiteRootPath, Constants.DiagnosticsPath); _locksPath = Path.Combine(SiteRootPath, Constants.LocksPath); - + _k8seAppName = k8seAppName; + _k8seAppNamespace = k8seAppNamespace; + _k8seAppType = k8seAppType; + if (OSDetector.IsOnWindows()) { _sshKeyPath = Path.Combine(rootPath, Constants.SSHKeyPath); @@ -128,7 +146,12 @@ public Environment( else { // in linux, rootPath is "/home", while .ssh folder need to under "/home/{user}" - _sshKeyPath = Path.Combine(rootPath, System.Environment.GetEnvironmentVariable("KUDU_RUN_USER"), Constants.SSHKeyPath); + string path2 = System.Environment.GetEnvironmentVariable("KUDU_RUN_USER"); + if (path2 == null || path2.Equals("")) + { + path2 = "root"; + } + _sshKeyPath = Path.Combine(rootPath, path2, Constants.SSHKeyPath); } _scriptPath = Path.Combine(binPath, Constants.ScriptsPath); _nodeModulesPath = Path.Combine(binPath, Constants.NodeModulesPath); @@ -152,7 +175,7 @@ public Environment( _jobsBinariesPath = Path.Combine(_webRootPath, userDefinedWebJobRoot); } _sitePackagesPath = Path.Combine(_dataPath, Constants.SitePackages); - + RequestId = !string.IsNullOrEmpty(requestId) ? requestId : Guid.Empty.ToString(); _httpContextAccessor = httpContextAccessor; @@ -224,7 +247,7 @@ public string SSHKeyPath public string RootPath { get; - private set; + set; } public string SiteRootPath @@ -343,7 +366,7 @@ public string FunctionsPath return this.WebRootPath; } } - + public string SitePackagesPath { get @@ -385,11 +408,21 @@ public string RequestId private set; } + public bool IsOnLinuxConsumption + { + get + { + bool isOnAppService = !string.IsNullOrEmpty(System.Environment.GetEnvironmentVariable(Constants.AzureWebsiteInstanceId)); + bool isOnLinuxContainer = !string.IsNullOrEmpty(System.Environment.GetEnvironmentVariable(Constants.ContainerName)); + return isOnLinuxContainer && !isOnAppService; + } + } + public string KuduConsoleFullPath { get; } public static bool IsAzureEnvironment() { - return !String.IsNullOrEmpty(System.Environment.GetEnvironmentVariable("WEBSITE_INSTANCE_ID")); + return !String.IsNullOrEmpty(System.Environment.GetEnvironmentVariable(Constants.AzureWebsiteInstanceId)); } public static bool SkipSslValidation @@ -420,6 +453,65 @@ public static bool SkipSslValidation } } + + public static string ContainerName + { + get + { + return System.Environment.GetEnvironmentVariable(Constants.ContainerName)?.ToLowerInvariant(); + } + } + + public static string StampName + { + get + { + return System.Environment.GetEnvironmentVariable(Constants.WebSiteHomeStampName)?.ToLowerInvariant(); + } + } + + public static string TenantId + { + get + { + return System.Environment.GetEnvironmentVariable(Constants.WebSiteStampDeploymentId)?.ToLowerInvariant(); + } + } + + public bool IsK8SEApp + { + get + { + return K8SEDeploymentHelper.IsK8SEEnvironment(); + } + } + + public string CurrId { get; set; } + + public string K8SEAppName + { + get + { + return _k8seAppName; + } + } + + public string K8SEAppNamespace + { + get + { + return _k8seAppNamespace; + } + } + + public string K8SEAppType + { + get + { + return _k8seAppType; + } + } + public static string GetFreeSpaceHtml(string path) { try diff --git a/Kudu.Core/Functions/CodeSpec.cs b/Kudu.Core/Functions/CodeSpec.cs new file mode 100644 index 00000000..4ba5c9e6 --- /dev/null +++ b/Kudu.Core/Functions/CodeSpec.cs @@ -0,0 +1,10 @@ +namespace Kudu.Core.Functions +{ + using Newtonsoft.Json; + + public class CodeSpec + { + [JsonProperty(PropertyName = "packageRef")] + public PackageReference PackageRef { get; set; } + } +} diff --git a/Kudu.Core/Functions/FunctionManager.cs b/Kudu.Core/Functions/FunctionManager.cs index fa1f98c3..f17792a6 100644 --- a/Kudu.Core/Functions/FunctionManager.cs +++ b/Kudu.Core/Functions/FunctionManager.cs @@ -444,7 +444,7 @@ private string GetFunctionSecretsFilePath(string functionName) /// the to be populated with function app content. /// Optional: indicates whether to add local.settings.json or not to the archive. Default is false. /// Optional: indicates whether to add a .csproj to the archive. Default is false. - /// Optional: the name for *.csproj file if is true. Default is appName. + /// Optional: the name for *.csproj file if is true. Default is AppName. public void CreateArchive(ZipArchive zip, bool includeAppSettings = false, bool includeCsproj = false, string projectName = null) { var tracer = _traceFactory.GetTracer(); diff --git a/Kudu.Core/Functions/KedaFunctionTriggerProvider.cs b/Kudu.Core/Functions/KedaFunctionTriggerProvider.cs new file mode 100644 index 00000000..b4b098a2 --- /dev/null +++ b/Kudu.Core/Functions/KedaFunctionTriggerProvider.cs @@ -0,0 +1,428 @@ +using Kudu.Core.Helpers; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Text.RegularExpressions; + +namespace Kudu.Core.Functions +{ + public static class KedaFunctionTriggerProvider + { + public static IEnumerable GetFunctionTriggers(string zipFilePath, string appName = null, string appType = null, IDictionary appSettings = null) + { + appSettings = appSettings ?? new Dictionary(); + + if (!File.Exists(zipFilePath)) + { + return null; + } + + string hostJsonText = null; + var triggerBindings = new List(); + using (var zip = ZipFile.OpenRead(zipFilePath)) + { + var hostJsonEntry = zip.Entries.FirstOrDefault(e => IsHostJson(e.FullName)); + if (hostJsonEntry != null) + { + using (var reader = new StreamReader(hostJsonEntry.Open())) + { + hostJsonText = reader.ReadToEnd(); + } + } + + var entries = zip.Entries + .Where(e => IsFunctionJson(e.FullName)); + + foreach (var entry in entries) + { + using (var stream = entry.Open()) + { + using (var reader = new StreamReader(stream)) + { + triggerBindings.AddRange(ParseFunctionJson(GetFunctionName(entry), JObject.Parse(reader.ReadToEnd()))); + } + } + } + } + + bool IsFunctionJson(string fullName) + { + return fullName.EndsWith(Constants.FunctionsConfigFile) && + fullName.Count(c => c == '/' || c == '\\') == 1; + } + + bool IsHostJson(string fullName) + { + return fullName.Equals(Constants.FunctionsHostConfigFile, StringComparison.OrdinalIgnoreCase); + } + + var triggers = CreateScaleTriggers(triggerBindings, hostJsonText, appSettings).ToList(); + + var isWorkflowApp = appType?.ToLowerInvariant()?.Contains(Constants.WorkflowAppKind.ToLowerInvariant()); + if (isWorkflowApp.GetValueOrDefault(defaultValue: false)) + { + // NOTE(haassyad) Check if the host json has the workflow extension loaded. If so we will add a queue scale trigger for the job dispatcher queue. + if (TryGetWorkflowKedaTrigger(hostJsonText, appName, out ScaleTrigger workflowScaleTrigger)) + { + triggers.Add(workflowScaleTrigger); + } + } + + return triggers; + } + + internal static void UpdateFunctionTriggerBindingExpression( + IEnumerable scaleTriggers, IDictionary appSettings) + { + string ReplaceMatchedBindingExpression(Match match) + { + var bindingExpressionTarget = match.Value.Replace("%", ""); + if (appSettings.ContainsKey(bindingExpressionTarget)) + { + return appSettings[bindingExpressionTarget]; + } + + return bindingExpressionTarget; + } + + var matchEvaluator = new MatchEvaluator((Func) ReplaceMatchedBindingExpression); + + foreach (var scaleTrigger in scaleTriggers) + { + IDictionary newMetadata = new Dictionary(); + foreach (var metadata in scaleTrigger.Metadata) + { + var replacedValue = Regex.Replace(metadata.Value, Constants.AppSettingsRegex, matchEvaluator); + newMetadata[metadata.Key] = replacedValue; + } + + scaleTrigger.Metadata = newMetadata; + } + } + + public static IEnumerable GetFunctionTriggers(IEnumerable functionsJson, string hostJsonText, IDictionary appSettings) + { + var triggerBindings = functionsJson + .Select(o => ParseFunctionJson(o["functionName"]?.ToString(), o)) + .SelectMany(i => i); + + return CreateScaleTriggers(triggerBindings, hostJsonText, appSettings); + } + + public static IEnumerable GetFunctionTriggersFromSyncTriggerPayload(string synctriggerPayload, + IDictionary appSettings) + { + return CreateScaleTriggers(ParseSyncTriggerPayload(synctriggerPayload), ParseHostJsonPayload(synctriggerPayload), appSettings); + } + + internal static IEnumerable CreateScaleTriggers(IEnumerable triggerBindings, string hostJsonText, IDictionary appSettings) + { + + var durableTriggers = triggerBindings.Where(b => IsDurable(b)); + var standardTriggers = triggerBindings.Where(b => !IsDurable(b)); + + var kedaScaleTriggers = new List(); + kedaScaleTriggers.AddRange(GetStandardScaleTriggers(standardTriggers)); + + // Update Binding Expression for %..% notation + UpdateFunctionTriggerBindingExpression(kedaScaleTriggers, appSettings); + + // Durable Functions triggers are treated as a group and get configuration from host.json + if (durableTriggers.Any() && TryGetDurableKedaTrigger(hostJsonText, out ScaleTrigger durableScaleTrigger)) + { + kedaScaleTriggers.Add(durableScaleTrigger); + } + + bool IsDurable(FunctionTrigger function) => + function.Type.Equals("orchestrationTrigger", StringComparison.OrdinalIgnoreCase) || + function.Type.Equals("activityTrigger", StringComparison.OrdinalIgnoreCase) || + function.Type.Equals("entityTrigger", StringComparison.OrdinalIgnoreCase); + + return kedaScaleTriggers; + } + + internal static string ParseHostJsonPayload(string payload) + { + var payloadJson = JObject.Parse(payload); + var extensions = (JObject) payloadJson["extensions"]; + if (extensions != null) + { + var hostJsonPayload = new JObject {{"extensions", extensions}}; + return hostJsonPayload.ToString(); + } + else + { + return string.Empty; + } + } + + internal static IEnumerable ParseSyncTriggerPayload(string payload) + { + var payloadJson = JObject.Parse(payload); + var triggers = (JArray)payloadJson["triggers"]; + return triggers.Select(o => o.ToObject()) + .Select(o => new FunctionTrigger(o["functionName"].ToString(), o, o["type"].ToString())); + } + + internal static IEnumerable ParseFunctionJson(string functionName, JObject functionJson) + { + if (functionJson.TryGetValue("disabled", out JToken value)) + { + string stringValue = value.ToString(); + if (!bool.TryParse(stringValue, out bool disabled)) + { + string expandValue = System.Environment.GetEnvironmentVariable(stringValue); + disabled = string.Equals(expandValue, "1", StringComparison.OrdinalIgnoreCase) || + string.Equals(expandValue, "true", StringComparison.OrdinalIgnoreCase); + } + + if (disabled) + { + yield break; + } + } + + var excluded = functionJson.TryGetValue("excluded", out value) && (bool)value; + if (excluded) + { + yield break; + } + + foreach (JObject binding in (JArray)functionJson["bindings"]) + { + var type = (string)binding["type"]; + if (type.EndsWith("Trigger", StringComparison.OrdinalIgnoreCase)) + { + yield return new FunctionTrigger(functionName, binding, type); + } + } + } + + internal static IEnumerable GetStandardScaleTriggers(IEnumerable standardTriggers) + { + foreach (FunctionTrigger function in standardTriggers) + { + var triggerType = GetKedaTriggerType(function.Type); + if (!string.IsNullOrEmpty(triggerType)) + { + var scaleTrigger = new ScaleTrigger + { + Type = triggerType, + Metadata = PopulateMetadataDictionary(function.Binding, function.FunctionName) + }; + yield return scaleTrigger; + } + } + } + + internal static string GetFunctionName(ZipArchiveEntry zipEntry) + { + if (string.IsNullOrWhiteSpace(zipEntry?.FullName)) + { + return string.Empty; + } + + return zipEntry.FullName.Split('/').Length == 2 ? zipEntry.FullName.Split('/')[0] : zipEntry.FullName.Split('\\')[0]; + } + + internal static string GetKedaTriggerType(string triggerType) + { + if (string.IsNullOrEmpty(triggerType)) + { + throw new ArgumentNullException(nameof(triggerType)); + } + + triggerType = triggerType.ToLower(); + + switch (triggerType) + { + case "queuetrigger": + return "azure-queue"; + + case "kafkatrigger": + return "kafka"; + + case "blobtrigger": + return "azure-blob"; + + case "servicebustrigger": + return "azure-servicebus"; + + case "eventhubtrigger": + return "azure-eventhub"; + + case "rabbitmqtrigger": + return "rabbitmq"; + + case "httptrigger": + return "httpTrigger"; + + default: + return string.Empty; + } + } + + internal static bool TryGetDurableKedaTrigger(string hostJsonText, out ScaleTrigger scaleTrigger) + { + scaleTrigger = null; + if (string.IsNullOrEmpty(hostJsonText)) + { + return false; + } + + JObject hostJson = JObject.Parse(hostJsonText); + + // Reference: https://docs.microsoft.com/azure/azure-functions/durable/durable-functions-bindings#durable-functions-2-0-host-json + string durableStorageProviderPath = $"{Constants.Extensions}.{Constants.DurableTask}.{Constants.DurableTaskStorageProvider}"; + JObject storageProviderConfig = hostJson.SelectToken(durableStorageProviderPath) as JObject; + string storageType = storageProviderConfig?["type"]?.ToString(); + + // Custom storage types are supported starting in Durable Functions v2.4.2 + // Minimum required version of Microsoft.DurableTask.SqlServer.AzureFunctions is v0.7.0-alpha + if (string.Equals(storageType, Constants.DurableTaskMicrosoftSqlProviderType, StringComparison.OrdinalIgnoreCase)) + { + scaleTrigger = new ScaleTrigger + { + // MSSQL scaler reference: https://keda.sh/docs/2.2/scalers/mssql/ + Type = Constants.MicrosoftSqlScaler, + Metadata = new Dictionary + { + // Durable SQL scaling: https://microsoft.github.io/durabletask-mssql/#/scaling?id=worker-auto-scale + ["query"] = "SELECT dt.GetScaleRecommendation(10, 1)", // max 10 orchestrations and 1 activity per replica + ["targetValue"] = "1", + ["connectionStringFromEnv"] = storageProviderConfig?[Constants.DurableTaskSqlConnectionName]?.ToString(), + } + }; + } + else + { + // TODO: Support for the Azure Storage and Netherite backends + } + + return scaleTrigger != null; + } + + /// + /// Tries to add a scale trigger if the app is a workflow app. + /// + /// The host.json text. + /// The app name. + /// The scale trigger. + /// true if a scale trigger was found + internal static bool TryGetWorkflowKedaTrigger(string hostJsonText, string appName, out ScaleTrigger scaleTrigger) + { + JObject hostJson = JObject.Parse(hostJsonText); + + // Check the host.json file for workflow settings. + JObject workflowSettings = hostJson + .SelectToken(path: $"{Constants.Extensions}.{Constants.WorkflowExtensionName}.{Constants.WorkflowSettingsName}") as JObject; + + // Get the queue length if specified, otherwise default to arbitrary value. + var queueLengthObject = workflowSettings?["Runtime.ScaleMonitor.KEDA.TargetQueueLength"]; + var queueLength = queueLengthObject != null ? queueLengthObject.ToString() : "20"; + + // Get the host id if specified, otherwise default to app name. + var hostIdObject = workflowSettings?["Runtime.HostId"]; + var hostId = hostIdObject != null ? hostIdObject.ToString() : appName; + + // Hash the host id. + var hostSpecificStorageId = StringHelper + .EscapeAndTrimStorageKeyPrefix(HashHelper.MurmurHash64(hostId).ToString("X"), 32) + .ToLowerInvariant(); + + var queuePrefix = $"flow{hostSpecificStorageId}jobtriggers"; + + scaleTrigger = new ScaleTrigger + { + // Azure queue scaler reference: https://keda.sh/docs/2.2/scalers/azure-storage-queue/ + Type = Constants.AzureQueueScaler, + Metadata = new Dictionary + { + // NOTE(haassyad): We only have one queue partition in single tenant. + ["queueName"] = StringHelper.GetWorkflowQueueNameInternal(queuePrefix, 1), + ["queueLength"] = queueLength, + ["connectionFromEnv"] = "AzureWebJobsStorage", + } + }; + + return true; + } + + // match https://github.com/Azure/azure-functions-core-tools/blob/6bfab24b2743f8421475d996402c398d2fe4a9e0/src/Azure.Functions.Cli/Kubernetes/KEDA/V2/KedaV2Resource.cs#L91 + internal static IDictionary PopulateMetadataDictionary(JToken t, string functionName) + { + const string ConnectionField = "connection"; + const string ConnectionFromEnvField = "connectionFromEnv"; + + IDictionary metadata = t.ToObject>() + .Where(i => i.Value.Type == JTokenType.String) + .ToDictionary(k => k.Key, v => v.Value.ToString()); + + var triggerType = t["type"].ToString().ToLower(); + + switch (triggerType) + { + case TriggerTypes.AzureBlobStorage: + case TriggerTypes.AzureStorageQueue: + metadata[ConnectionFromEnvField] = metadata[ConnectionField] ?? "AzureWebJobsStorage"; + metadata.Remove(ConnectionField); + break; + case TriggerTypes.AzureServiceBus: + metadata[ConnectionFromEnvField] = metadata[ConnectionField] ?? "AzureWebJobsServiceBus"; + metadata.Remove(ConnectionField); + break; + case TriggerTypes.AzureEventHubs: + metadata[ConnectionFromEnvField] = metadata[ConnectionField]; + metadata.Remove(ConnectionField); + break; + + case TriggerTypes.Kafka: + metadata["bootstrapServersFromEnv"] = metadata["brokerList"]; + metadata.Remove("brokerList"); + metadata.Remove("protocol"); + metadata.Remove("authenticationMode"); + break; + + case TriggerTypes.RabbitMq: + metadata["hostFromEnv"] = metadata["connectionStringSetting"]; + metadata.Remove("connectionStringSetting"); + break; + } + + // Clean-up for all triggers + + metadata.Remove("type"); + metadata.Remove("name"); + + metadata["functionName"] = functionName; + return metadata; + } + + internal class FunctionTrigger + { + public FunctionTrigger(string functionName, JObject binding, string type) + { + this.FunctionName = functionName; + this.Binding = binding; + this.Type = type; + } + + public string FunctionName { get; } + public JObject Binding { get; } + public string Type { get; } + } + + static class TriggerTypes + { + public const string AzureBlobStorage = "blobtrigger"; + public const string AzureEventHubs = "eventhubtrigger"; + public const string AzureServiceBus = "servicebustrigger"; + public const string AzureStorageQueue = "queuetrigger"; + public const string Kafka = "kafkatrigger"; + public const string RabbitMq = "rabbitmqtrigger"; + } + } +} diff --git a/Kudu.Core/Functions/PackageReference.cs b/Kudu.Core/Functions/PackageReference.cs new file mode 100644 index 00000000..b13b835e --- /dev/null +++ b/Kudu.Core/Functions/PackageReference.cs @@ -0,0 +1,12 @@ +using Kudu.Contracts.Deployment; + +namespace Kudu.Core.Functions +{ + using Newtonsoft.Json; + + public class PackageReference + { + [JsonProperty(PropertyName = "buildMetadata")] + public string BuildMetadata { get; set; } + } +} diff --git a/Kudu.Core/Functions/PatchAppJson.cs b/Kudu.Core/Functions/PatchAppJson.cs new file mode 100644 index 00000000..b0fb6705 --- /dev/null +++ b/Kudu.Core/Functions/PatchAppJson.cs @@ -0,0 +1,13 @@ +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Kudu.Core.Functions +{ + public class PatchAppJson + { + [JsonProperty(PropertyName = "spec")] + public PatchSpec PatchSpec { get; set; } + } +} diff --git a/Kudu.Core/Functions/PatchSpec.cs b/Kudu.Core/Functions/PatchSpec.cs new file mode 100644 index 00000000..d80f5df5 --- /dev/null +++ b/Kudu.Core/Functions/PatchSpec.cs @@ -0,0 +1,16 @@ +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Kudu.Core.Functions +{ + public class PatchSpec + { + [JsonProperty(PropertyName = "triggerOptions", DefaultValueHandling = DefaultValueHandling.Ignore)] + public TriggerOptions TriggerOptions { get; set; } + + [JsonProperty(PropertyName = "code", DefaultValueHandling = DefaultValueHandling.Ignore)] + public CodeSpec Code { get; set; } + } +} diff --git a/Kudu.Core/Functions/ScaleTrigger.cs b/Kudu.Core/Functions/ScaleTrigger.cs new file mode 100644 index 00000000..3d97c5c0 --- /dev/null +++ b/Kudu.Core/Functions/ScaleTrigger.cs @@ -0,0 +1,14 @@ +using Newtonsoft.Json; +using System.Collections.Generic; + +namespace Kudu.Core.Functions +{ + public class ScaleTrigger + { + [JsonProperty(PropertyName = "type")] + public string Type { get; set; } + + [JsonProperty(PropertyName = "metadata")] + public IDictionary Metadata { get; set; } + } +} diff --git a/Kudu.Core/Functions/SyncTriggerHandler.cs b/Kudu.Core/Functions/SyncTriggerHandler.cs new file mode 100644 index 00000000..bc2addf9 --- /dev/null +++ b/Kudu.Core/Functions/SyncTriggerHandler.cs @@ -0,0 +1,75 @@ +using Kudu.Contracts.Deployment; +using Kudu.Contracts.Tracing; +using Kudu.Core.K8SE; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Kudu.Core.Tracing; + +namespace Kudu.Core.Functions +{ + public class SyncTriggerHandler + { + private readonly IEnvironment _environment; + private readonly ITracer _tracer; + private readonly IDictionary _appSettings; + + public SyncTriggerHandler(IEnvironment environment, + ITracer tracer, + IDictionary appSettings) + { + _environment = environment; + _tracer = tracer; + _appSettings = appSettings ?? new Dictionary(); + } + + public async Task SyncTriggers(string functionTriggersPayload) + { + using (_tracer.Step("SyncTriggerHandler.SyncTrigger()")) + { + var scaleTriggersContent = GetScaleTriggers(functionTriggersPayload); + if (!string.IsNullOrEmpty(scaleTriggersContent.Item2)) + { + return scaleTriggersContent.Item2; + } + + var scaleTriggers = scaleTriggersContent.Item1; + string appName = _environment.K8SEAppName; + + await Task.Run(() => K8SEDeploymentHelper.UpdateFunctionAppTriggers(appName, scaleTriggers, null)); + } + + return null; + } + + public Tuple, string> GetScaleTriggers(string functionTriggersPayload) + { + IEnumerable scaleTriggers = new List(); + try + { + if (string.IsNullOrEmpty(functionTriggersPayload)) + { + return new Tuple, string>(null, "Function trigger payload is null or empty."); + } + + scaleTriggers = + KedaFunctionTriggerProvider.GetFunctionTriggersFromSyncTriggerPayload(functionTriggersPayload, + _appSettings); + if (!scaleTriggers.Any()) + { + return new Tuple, string>(null, "No triggers in the payload"); + } + } + catch (Exception e) + { + return new Tuple, string>(null, e.Message); + } + + return new Tuple, string>(scaleTriggers, null); ; + } + + } +} diff --git a/Kudu.Core/Functions/TriggerOptions.cs b/Kudu.Core/Functions/TriggerOptions.cs new file mode 100644 index 00000000..68493871 --- /dev/null +++ b/Kudu.Core/Functions/TriggerOptions.cs @@ -0,0 +1,17 @@ +using Newtonsoft.Json; +using System.Collections.Generic; + +namespace Kudu.Core.Functions +{ + public class TriggerOptions + { + [JsonProperty(PropertyName = "triggers")] + public IEnumerable Triggers { get; set; } + + [JsonProperty(PropertyName = "pollingInterval")] + public int? PollingInterval { get; set; } + + [JsonProperty(PropertyName = "cooldownPeriod")] + public int? cooldownPeriod { get; set; } + } +} diff --git a/Kudu.Core/Helpers/FunctionAppSpecializationHelper.cs b/Kudu.Core/Helpers/FunctionAppSpecializationHelper.cs new file mode 100644 index 00000000..3bd5163b --- /dev/null +++ b/Kudu.Core/Helpers/FunctionAppSpecializationHelper.cs @@ -0,0 +1,31 @@ +using Kudu.Contracts.Settings; +using Kudu.Core.Deployment.Oryx; +using System; +using System.Collections.Generic; +using System.Text; + +namespace Kudu.Core.Helpers +{ + /// + /// Handle Function App container specialization when running in Service Fabric Mesh + /// + public static class FunctionAppSpecializationHelper + { + /// + /// According to the existing environment variables key-value pair, change the container environment. + /// + /// The key of environment variable + /// The value of environment variable + /// The new environment variables should be set + public static Dictionary HandleLinuxConsumption(string envKey, string envValue) + { + Dictionary toBeUpdatedEnv = new Dictionary(); + if (envKey == OryxBuildConstants.FunctionAppEnvVars.WorkerRuntimeSetting) + { + toBeUpdatedEnv.Add(SettingsKeys.DoBuildDuringDeployment, "true"); + toBeUpdatedEnv.Add(OryxBuildConstants.EnableOryxBuild, "true"); + } + return toBeUpdatedEnv; + } + } +} diff --git a/Kudu.Core/Helpers/HashHelper.cs b/Kudu.Core/Helpers/HashHelper.cs new file mode 100644 index 00000000..5fa35d3a --- /dev/null +++ b/Kudu.Core/Helpers/HashHelper.cs @@ -0,0 +1,138 @@ +using System.Text; + +namespace Kudu.Core.Helpers +{ + public static class HashHelper + { + /// + /// Computes 64-bit Murmur hash. + /// + /// The input string. + /// The input seed. + public static ulong MurmurHash64(string str, uint seed = 0) + { + return HashHelper.MurmurHash64(Encoding.UTF8.GetBytes(str), seed); + } + + /// + /// Computes 64-bit Murmur hash. + /// + /// The input data. + /// The input seed. + public static ulong MurmurHash64(byte[] data, uint seed = 0) + { + const uint C1 = 0x239b961b; + const uint C2 = 0xab0e9789; + const uint C3 = 0x561ccd1b; + const uint C4 = 0x0bcaa747; + const uint C5 = 0x85ebca6b; + const uint C6 = 0xc2b2ae35; + + int length = data.Length; + + unchecked + { + uint h1 = seed; + uint h2 = seed; + + int index = 0; + while (index + 7 < length) + { + uint k1 = (uint)(data[index + 0] | data[index + 1] << 8 | data[index + 2] << 16 | data[index + 3] << 24); + uint k2 = (uint)(data[index + 4] | data[index + 5] << 8 | data[index + 6] << 16 | data[index + 7] << 24); + + k1 *= C1; + k1 = k1.RotateLeft32(15); + k1 *= C2; + h1 ^= k1; + h1 = h1.RotateLeft32(19); + h1 += h2; + h1 = (h1 * 5) + C3; + + k2 *= C2; + k2 = k2.RotateLeft32(17); + k2 *= C1; + h2 ^= k2; + h2 = h2.RotateLeft32(13); + h2 += h1; + h2 = (h2 * 5) + C4; + + index += 8; + } + + int tail = length - index; + if (tail > 0) + { + uint k1 = (tail >= 4) ? (uint)(data[index + 0] | data[index + 1] << 8 | data[index + 2] << 16 | data[index + 3] << 24) : + (tail == 3) ? (uint)(data[index + 0] | data[index + 1] << 8 | data[index + 2] << 16) : + (tail == 2) ? (uint)(data[index + 0] | data[index + 1] << 8) : + (uint)data[index + 0]; + + k1 *= C1; + k1 = k1.RotateLeft32(15); + k1 *= C2; + h1 ^= k1; + + if (tail > 4) + { + uint k2 = (tail == 7) ? (uint)(data[index + 4] | data[index + 5] << 8 | data[index + 6] << 16) : + (tail == 6) ? (uint)(data[index + 4] | data[index + 5] << 8) : + (uint)data[index + 4]; + + k2 *= C2; + k2 = k2.RotateLeft32(17); + k2 *= C1; + h2 ^= k2; + } + } + + h1 ^= (uint)length; + h2 ^= (uint)length; + + h1 += h2; + h2 += h1; + + h1 ^= h1 >> 16; + h1 *= C5; + h1 ^= h1 >> 13; + h1 *= C6; + h1 ^= h1 >> 16; + + h2 ^= h2 >> 16; + h2 *= C5; + h2 ^= h2 >> 13; + h2 *= C6; + h2 ^= h2 >> 16; + + h1 += h2; + h2 += h1; + + return ((ulong)h2 << 32) | (ulong)h1; + } + } + + #region RotateLeft + + /// + /// Rotates the bits in the provided value to the left (where the number of bits is specified). + /// + /// The value to be rotated. + /// The number of bits to rotate. + private static uint RotateLeft32(this uint value, int count) + { + return (value << count) | (value >> (32 - count)); + } + + /// + /// Rotates the bits in the provided value to the right (where the number of bits is specified). + /// + /// The value to be rotated. + /// The number of bits to rotate. + private static ulong RotateLeft64(this ulong value, int count) + { + return (value << count) | (value >> (64 - count)); + } + + #endregion + } +} diff --git a/Kudu.Core/Helpers/LinuxConsumptionDeploymentHelper.cs b/Kudu.Core/Helpers/LinuxConsumptionDeploymentHelper.cs new file mode 100644 index 00000000..c569de85 --- /dev/null +++ b/Kudu.Core/Helpers/LinuxConsumptionDeploymentHelper.cs @@ -0,0 +1,255 @@ +using Kudu.Contracts.Settings; +using Kudu.Core.Deployment; +using Kudu.Core.Deployment.Generator; +using Kudu.Core.Deployment.Oryx; +using Kudu.Core.Infrastructure; +using Kudu.Core.Tracing; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.IO; +using System.Net.Http; +using System.Threading.Tasks; +using System.Xml.Linq; + +namespace Kudu.Core.Helpers +{ + public class LinuxConsumptionDeploymentHelper + { + /// + /// Specifically used for Linux Consumption to support Server Side build scenario + /// + /// + public static async Task SetupLinuxConsumptionFunctionAppDeployment(IEnvironment env, IDeploymentSettingsManager settings, DeploymentContext context) + { + string sas = System.Environment.GetEnvironmentVariable(Constants.ScmRunFromPackage); + string builtFolder = context.OutputPath; + string packageFolder = env.DeploymentsPath; + string packageFileName = OryxBuildConstants.FunctionAppBuildSettings.LinuxConsumptionArtifactName; + + // Package built content from oryx build artifact + string filePath = PackageArtifactFromFolder(env, settings, context, builtFolder, packageFolder, packageFileName); + + // Log function app dependencies to kusto (requirements.txt, package.json, .csproj) + await LogDependenciesFile(context.RepositoryPath); + + // Upload from DeploymentsPath + await UploadLinuxConsumptionFunctionAppBuiltContent(context, sas, filePath); + + // Clean up local built content + FileSystemHelpers.DeleteDirectoryContentsSafe(context.OutputPath); + + // Remove Linux consumption plan functionapp workers for the site + await RemoveLinuxConsumptionFunctionAppWorkers(context); + } + + private static async Task LogDependenciesFile(string builtFolder) + { + try + { + await PrintRequirementsTxtDependenciesAsync(builtFolder); + await PrintPackageJsonDependenciesAsync(builtFolder); + PrintCsprojDependenciesAsync(builtFolder); + } catch (Exception) + { + KuduEventGenerator.Log().GenericEvent( + ServerConfiguration.GetApplicationName(), + $"dependencies,failed to parse function app dependencies", + Guid.Empty.ToString(), + string.Empty, + string.Empty, + string.Empty); + } + } + + private static async Task PrintRequirementsTxtDependenciesAsync(string builtFolder) + { + string filename = "requirements.txt"; + string requirementsTxtPath = Path.Combine(builtFolder, filename); + if (File.Exists(requirementsTxtPath)) + { + string[] lines = await File.ReadAllLinesAsync(requirementsTxtPath); + foreach (string line in lines) + { + int separatorIndex; + if (line.IndexOf("==") >= 0) + { + separatorIndex = line.IndexOf("=="); + } else if (line.IndexOf(">=") >= 0) + { + separatorIndex = line.IndexOf(">="); + } else if (line.IndexOf("<=") >= 0) + { + separatorIndex = line.IndexOf("<="); + } else if (line.IndexOf(">") >= 0) + { + separatorIndex = line.IndexOf(">"); + } else if (line.IndexOf("<") >= 0) + { + separatorIndex = line.IndexOf("<"); + } else + { + separatorIndex = line.Length; + } + + string package = line.Substring(0, separatorIndex).Trim(); + string version = line.Substring(separatorIndex).Trim(); + + KuduEventGenerator.Log().GenericEvent( + ServerConfiguration.GetApplicationName(), + $"dependencies,python,{filename},{package},{version}", + Guid.Empty.ToString(), + string.Empty, + string.Empty, + string.Empty); + } + } + } + + private static async Task PrintPackageJsonDependenciesAsync(string builtFolder) + { + string filename = "package.json"; + string packageJsonPath = Path.Combine(builtFolder, filename); + if (File.Exists(packageJsonPath)) + { + string content = await File.ReadAllTextAsync(packageJsonPath); + JObject jobj = JObject.Parse(content); + if (jobj.ContainsKey("devDependencies")) + { + Dictionary dictObj = jobj["devDependencies"].ToObject>(); + foreach (string key in dictObj.Keys) + { + KuduEventGenerator.Log().GenericEvent( + ServerConfiguration.GetApplicationName(), + $"dependencies,node,{filename},{key},{dictObj[key]},devDependencies", + Guid.Empty.ToString(), + string.Empty, + string.Empty, + string.Empty); + } + } + + if (jobj.ContainsKey("dependencies")) + { + Dictionary dictObj = jobj["dependencies"].ToObject>(); + foreach (string key in dictObj.Keys) + { + KuduEventGenerator.Log().GenericEvent( + ServerConfiguration.GetApplicationName(), + $"dependencies,node,{filename},{key},{dictObj[key]},dependencies", + Guid.Empty.ToString(), + string.Empty, + string.Empty, + string.Empty); + } + } + } + } + + private static void PrintCsprojDependenciesAsync(string builtFolder) + { + foreach (string csprojPath in Directory.GetFiles(builtFolder, "*.csproj", SearchOption.TopDirectoryOnly)) + { + string filename = Path.GetFileName(csprojPath); + XElement purchaseOrder = XElement.Load(csprojPath); + foreach (var itemGroup in purchaseOrder.Elements("ItemGroup")) + { + foreach (var packageReference in itemGroup.Elements("PackageReference")) + { + string include = packageReference.Attribute("Include").Value; + string version = packageReference.Attribute("Version").Value; + KuduEventGenerator.Log().GenericEvent( + ServerConfiguration.GetApplicationName(), + $"dependencies,dotnet,{filename},{include},{version}", + Guid.Empty.ToString(), + string.Empty, + string.Empty, + string.Empty); + } + } + } + } + + private static async Task UploadLinuxConsumptionFunctionAppBuiltContent(DeploymentContext context, string sas, string filePath) + { + context.Logger.Log($"Uploading built content {filePath} -> {sas}"); + + // Check if SCM_RUN_FROM_PACKAGE does exist + if (string.IsNullOrEmpty(sas)) + { + context.Logger.Log($"Failed to upload because SCM_RUN_FROM_PACKAGE is not provided."); + throw new DeploymentFailedException(new ArgumentException("Failed to upload because SAS is empty.")); + } + + // Parse SAS + Uri sasUri = null; + if (!Uri.TryCreate(sas, UriKind.Absolute, out sasUri)) + { + context.Logger.Log($"Malformed SAS when uploading built content."); + throw new DeploymentFailedException(new ArgumentException("Failed to upload because SAS is malformed.")); + } + + // Upload blob to Azure Storage + CloudBlockBlob blob = new CloudBlockBlob(sasUri); + try + { + await blob.UploadFromFileAsync(filePath); + } + catch (StorageException se) + { + context.Logger.Log($"Failed to upload because Azure Storage responds {se.RequestInformation.HttpStatusCode}."); + context.Logger.Log(se.Message); + throw new DeploymentFailedException(se); + } + } + + private static async Task RemoveLinuxConsumptionFunctionAppWorkers(DeploymentContext context) + { + string webSiteHostName = System.Environment.GetEnvironmentVariable(SettingsKeys.WebsiteHostname); + string sitename = ServerConfiguration.GetApplicationName(); + + context.Logger.Log($"Reseting all workers for {webSiteHostName}"); + + try + { + await OperationManager.AttemptAsync(async () => + { + await PostDeploymentHelper.RemoveAllWorkersAsync(webSiteHostName, sitename); + }, retries: 3, delayBeforeRetry: 2000); + } + catch (ArgumentException ae) + { + context.Logger.Log($"Reset all workers has malformed webSiteHostName or sitename {ae.Message}"); + throw new DeploymentFailedException(ae); + } + catch (HttpRequestException hre) + { + context.Logger.Log($"Reset all workers endpoint responded with {hre.Message}"); + throw new DeploymentFailedException(hre); + } + } + + private static string PackageArtifactFromFolder(IEnvironment environment, IDeploymentSettingsManager settings, DeploymentContext context, string srcDirectory, string artifactDirectory, string artifactFilename) + { + context.Logger.Log("Writing the artifacts to a squashfs file"); + string file = Path.Combine(artifactDirectory, artifactFilename); + ExternalCommandFactory commandFactory = new ExternalCommandFactory(environment, settings, context.RepositoryPath); + Executable exe = commandFactory.BuildExternalCommandExecutable(srcDirectory, artifactDirectory, context.Logger); + try + { + exe.ExecuteWithProgressWriter(context.Logger, context.Tracer, $"mksquashfs . {file} -noappend"); + } + catch (Exception) + { + context.GlobalLogger.LogError(); + throw; + } + + int numOfArtifacts = OryxBuildConstants.FunctionAppBuildSettings.ConsumptionBuildMaxFiles; + DeploymentHelper.PurgeBuildArtifactsIfNecessary(artifactDirectory, BuildArtifactType.Squashfs, context.Tracer, numOfArtifacts); + return file; + } + } +} diff --git a/Kudu.Core/Helpers/PermissionHelper.cs b/Kudu.Core/Helpers/PermissionHelper.cs index 305b7bad..378a4ded 100644 --- a/Kudu.Core/Helpers/PermissionHelper.cs +++ b/Kudu.Core/Helpers/PermissionHelper.cs @@ -1,9 +1,12 @@ -using System.IO; +using System; +using System.Diagnostics; +using System.IO; +using System.Threading; using Kudu.Contracts.Settings; +using Kudu.Contracts.Tracing; using Kudu.Core.Deployment; using Kudu.Core.Deployment.Generator; using Kudu.Core.Infrastructure; -using System; namespace Kudu.Core.Helpers { @@ -16,5 +19,31 @@ public static void Chmod(string permission, string filePath, IEnvironment enviro Executable exe = exeFactory.BuildCommandExecutable("/bin/chmod", folder, deploymentSettingManager.GetCommandIdleTimeout(), logger); exe.Execute("{0} \"{1}\"", permission, filePath); } + + public static void ChmodRecursive(string permission, string directoryPath, ITracer tracer, TimeSpan timeout) + { + string cmd = String.Format("timeout {0}s chmod {1} -R {2}",timeout.TotalSeconds, permission, directoryPath); + var escapedArgs = cmd.Replace("\"", "\\\""); + + var process = new Process + { + StartInfo = new ProcessStartInfo + { + RedirectStandardOutput = true, + UseShellExecute = false, + CreateNoWindow = true, + WindowStyle = ProcessWindowStyle.Hidden, + FileName = "/bin/bash", + Arguments = $"-c \"{escapedArgs}\"" + } + }; + process.Start(); + process.WaitForExit(); + + if(process.ExitCode != 0) + { + throw new Exception(string.Format("Error in changing file permissions : {0}",process.ExitCode)); + } + } } } diff --git a/Kudu.Core/Helpers/PostDeploymentHelper.cs b/Kudu.Core/Helpers/PostDeploymentHelper.cs index 79be973f..38cbc4c1 100644 --- a/Kudu.Core/Helpers/PostDeploymentHelper.cs +++ b/Kudu.Core/Helpers/PostDeploymentHelper.cs @@ -11,6 +11,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using System.Web; using Kudu.Contracts.Settings; using Kudu.Core.Deployment; using Kudu.Core.Infrastructure; @@ -90,9 +91,9 @@ private static string WebSiteElasticScaleEnabled // WEBSITE_INSTANCE_ID not null or empty public static bool IsAzureEnvironment() { - return !String.IsNullOrEmpty(System.Environment.GetEnvironmentVariable("WEBSITE_INSTANCE_ID")); + return !String.IsNullOrEmpty(System.Environment.GetEnvironmentVariable(Constants.AzureWebsiteInstanceId)); } - + // WEBSITE_HOME_STAMPNAME = waws-prod-bay-001 private static string HomeStamp { @@ -348,6 +349,41 @@ public static async Task PerformAutoSwap(string requestId, TraceListener tracer) } } + /// + /// Remove all site workers after cloudbuilt content is uploaded + /// + /// WEBSITE_HOSTNAME + /// WEBSITE_SITE_NAME + /// Thrown when RemoveAllWorkers url is malformed. + /// Thrown when request to RemoveAllWorkers is not OK. + public static async Task RemoveAllWorkersAsync(string websiteHostname, string sitename) + { + // Generate URL encoded auth token + string websiteAuthEncryptionKey = System.Environment.GetEnvironmentVariable(SettingsKeys.AuthEncryptionKey); + DateTime expiry = DateTime.UtcNow.AddMinutes(5); + string authToken = SimpleWebTokenHelper.CreateToken(expiry, websiteAuthEncryptionKey.ToKeyBytes()); + string authTokenEncoded = HttpUtility.UrlEncode(authToken); + + // Generate RemoveAllWorker request URI + string baseUrl = $"http://{websiteHostname}/operations/removeworker/{sitename}/allStandard?token={authTokenEncoded}"; + Uri baseUri = null; + if (!Uri.TryCreate(baseUrl, UriKind.Absolute, out baseUri)) + { + throw new ArgumentException($"Malformed URI is used in RemoveAllWorkers"); + } + Trace(TraceEventType.Information, "Calling RemoveAllWorkers to refresh the function app"); + + // Initiate GET request + using (var client = HttpClientFactory()) + using (var response = await client.GetAsync(baseUri)) + { + response.EnsureSuccessStatusCode(); + Trace(TraceEventType.Information, "RemoveAllWorkers, statusCode = {0}", response.StatusCode); + } + + return; + } + private static void VerifyEnvironments() { if (string.IsNullOrEmpty(HttpHost)) @@ -699,11 +735,11 @@ private static void Trace(TraceEventType eventType, string format, params object } } - public static async Task UpdateSiteVersion(ZipDeploymentInfo deploymentInfo, IEnvironment environment, ILogger logger) + public static async Task UpdatePackageName(ZipDeploymentInfo deploymentInfo, IEnvironment environment, ILogger logger) { - var siteVersionPath = Path.Combine(environment.SitePackagesPath, Constants.SiteVersionTxt); - logger.Log($"Updating {siteVersionPath} with deployment {deploymentInfo.ZipName}"); - await FileSystemHelpers.WriteAllTextToFileAsync(siteVersionPath, deploymentInfo.ZipName); + var packageNamePath = Path.Combine(environment.SitePackagesPath, Constants.PackageNameTxt); + logger.Log($"Updating {packageNamePath} with deployment {deploymentInfo.ZipName}"); + await FileSystemHelpers.WriteAllTextToFileAsync(packageNamePath, deploymentInfo.ZipName); } } } diff --git a/Kudu.Core/Helpers/PostDeploymentTraceListener.cs b/Kudu.Core/Helpers/PostDeploymentTraceListener.cs index 96395293..0386aeef 100644 --- a/Kudu.Core/Helpers/PostDeploymentTraceListener.cs +++ b/Kudu.Core/Helpers/PostDeploymentTraceListener.cs @@ -24,7 +24,7 @@ public override void TraceEvent(TraceEventCache eventCache, string source, Trace { _logger.Log(format, args); - KuduEventSource.Log.GenericEvent( + KuduEventGenerator.Log().GenericEvent( ServerConfiguration.GetApplicationName(), string.Format(format, args), System.Environment.GetEnvironmentVariable("x-ms-request-id") ?? string.Empty, diff --git a/Kudu.Core/Helpers/SimpleWebTokenHelper.cs b/Kudu.Core/Helpers/SimpleWebTokenHelper.cs index fd3f896f..b8e345d2 100644 --- a/Kudu.Core/Helpers/SimpleWebTokenHelper.cs +++ b/Kudu.Core/Helpers/SimpleWebTokenHelper.cs @@ -1,11 +1,13 @@ using System; -using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using System.Security.Cryptography; using System.Text; - namespace Kudu.Core.Helpers +using Kudu.Contracts.Settings; +using Microsoft.AspNetCore.Authentication; + +namespace Kudu.Core.Helpers { public static class SimpleWebTokenHelper { @@ -15,19 +17,25 @@ public static class SimpleWebTokenHelper /// The SWT is then returned as an encrypted string /// /// Datetime for when the token should expire + /// Optional key to encrypt the token with /// a SWT signed by this app - public static string CreateToken(DateTime validUntil) => Encrypt($"exp={validUntil.Ticks}"); + public static string CreateToken(DateTime validUntil, byte[] key = null) => Encrypt($"exp={validUntil.Ticks}", key); [SuppressMessage("Microsoft.Usage", "CA2202:Object 'cipherStream' and 'cryptoStream' can be disposed mo re than once", Justification = "MemoeryStream, CryptoStream, and BinaryWriter handle multiple disposal correctly. The alternative is pretty ugly code for clearing each variable, checking for null, and manual dispose.")] - private static string Encrypt(string value) + public static string Encrypt(string value, byte[] key = null) { - using (var aes = new AesManaged { Key = GetWebSiteAuthEncryptionKey() }) + if (key == null) + { + TryGetEncryptionKey(SettingsKeys.AuthEncryptionKey, out key); + } + + using (var aes = new AesManaged { Key = key }) { // IV is always generated for the key every time aes.GenerateIV(); var input = Encoding.UTF8.GetBytes(value); var iv = Convert.ToBase64String(aes.IV); - using (var encrypter = aes.CreateEncryptor(aes.Key, aes.IV)) + using (var encrypter = aes.CreateEncryptor(aes.Key, aes.IV)) using (var cipherStream = new MemoryStream()) { using (var cryptoStream = new CryptoStream(cipherStream, encrypter, CryptoStreamMode.Write)) @@ -41,21 +49,114 @@ private static string Encrypt(string value) } } } - private static string GetSHA256Base64String(byte[] key) + + public static bool TryValidateToken(string token, ISystemClock systemClock) + { + try + { + return ValidateToken(token, systemClock); + } + catch + { + return false; + } + } + + public static bool ValidateToken(string token, ISystemClock systemClock) + { + // Use WebSiteAuthEncryptionKey if available else fallback to ContainerEncryptionKey. + // Until the container is specialized to a specific site WebSiteAuthEncryptionKey will not be available. + byte[] key; + if (!TryGetEncryptionKey(SettingsKeys.AuthEncryptionKey, out key, false)) + { + TryGetEncryptionKey(SettingsKeys.ContainerEncryptionKey, out key); + } + + var data = Decrypt(key, token); + + var parsedToken = data + // token = key1=value1;key2=value2 + .Split(';', StringSplitOptions.RemoveEmptyEntries) + // ["key1=value1", "key2=value2"] + .Select(v => v.Split('=', StringSplitOptions.RemoveEmptyEntries)) + // [["key1", "value1"], ["key2", "value2"]] + .ToDictionary(k => k[0], v => v[1]); + + return parsedToken.ContainsKey("exp") && systemClock.UtcNow.UtcDateTime < new DateTime(long.Parse(parsedToken["exp"])); + } + + public static string Decrypt(byte[] encryptionKey, string value) + { + var parts = value.Split(new[] { '.' }, StringSplitOptions.RemoveEmptyEntries); + if (parts.Length != 2 && parts.Length != 3) + { + throw new InvalidOperationException("Malformed token."); + } + + var iv = Convert.FromBase64String(parts[0]); + var data = Convert.FromBase64String(parts[1]); + var base64KeyHash = parts.Length == 3 ? parts[2] : null; + + if (!string.IsNullOrEmpty(base64KeyHash) && !string.Equals(GetSHA256Base64String(encryptionKey), base64KeyHash)) + { + throw new InvalidOperationException(string.Format("Key with hash {0} does not exist.", base64KeyHash)); + } + + using (var aes = new AesManaged { Key = encryptionKey }) + { + using (var ms = new MemoryStream()) + { + using (var cs = new CryptoStream(ms, aes.CreateDecryptor(aes.Key, iv), CryptoStreamMode.Write)) + using (var binaryWriter = new BinaryWriter(cs)) + { + binaryWriter.Write(data, 0, data.Length); + } + + return Encoding.UTF8.GetString(ms.ToArray()); + } + } + } + + private static bool TryGetEncryptionKey(string keyName, out byte[] encryptionKey, bool throwIfFailed = true) + { + encryptionKey = null; + var hexOrBase64 = System.Environment.GetEnvironmentVariable(keyName); + if (string.IsNullOrEmpty(hexOrBase64)) + { + if (throwIfFailed) + { + throw new InvalidOperationException($"No {keyName} defined in the environment"); + } + + return false; + } + + encryptionKey = hexOrBase64.ToKeyBytes(); + + return true; + } + + private static string GetSHA256Base64String(byte[] key) { using (var sha256 = new SHA256Managed()) { return Convert.ToBase64String(sha256.ComputeHash(key)); } } - private static byte[] GetWebSiteAuthEncryptionKey() + + private static byte[] GetWebSiteAuthEncryptionKey() { var hexOrBase64 = System.Environment.GetEnvironmentVariable(Constants.SiteAuthEncryptionKey); if (string.IsNullOrEmpty(hexOrBase64)) { throw new InvalidOperationException($"No {Constants.SiteAuthEncryptionKey} defined in the environment"); } - + + return hexOrBase64.ToKeyBytes(); + } + + public static byte[] ToKeyBytes(this string hexOrBase64) + { // only support 32 bytes (256 bits) key length if (hexOrBase64.Length == 64) { @@ -64,6 +165,7 @@ private static byte[] GetWebSiteAuthEncryptionKey() .Select(x => Convert.ToByte(hexOrBase64.Substring(x, 2), 16)) .ToArray(); } + return Convert.FromBase64String(hexOrBase64); } } diff --git a/Kudu.Core/Helpers/StringHelper.cs b/Kudu.Core/Helpers/StringHelper.cs new file mode 100644 index 00000000..3c4084a1 --- /dev/null +++ b/Kudu.Core/Helpers/StringHelper.cs @@ -0,0 +1,99 @@ +using System; +using System.Globalization; +using System.Text; + +namespace Kudu.Core.Helpers +{ + public static class StringHelper + { + /// + /// The storage key trim padding. + /// + public const int StorageKeyTrimPadding = 17; + + /// + /// The escaped storage keys. + /// + private static readonly string[] EscapedStorageKeys = new string[128] + { + ":00", ":01", ":02", ":03", ":04", ":05", ":06", ":07", ":08", ":09", ":0A", ":0B", ":0C", ":0D", ":0E", ":0F", + ":10", ":11", ":12", ":13", ":14", ":15", ":16", ":17", ":18", ":19", ":1A", ":1B", ":1C", ":1D", ":1E", ":1F", + ":20", ":21", ":22", ":23", ":24", ":25", ":26", ":27", ":28", ":29", ":2A", ":2B", ":2C", ":2D", ":2E", ":2F", + "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", ":3A", ":3B", ":3C", ":3D", ":3E", ":3F", + ":40", "A", "B", "C", "D", "E", "F", "G", "H", "I", "J", "K", "L", "M", "N", "O", + "P", "Q", "R", "S", "T", "U", "V", "W", "X", "Y", "Z", ":5B", ":5C", ":5D", ":5E", ":5F", + ":60", "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", + "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z", ":7B", ":7C", ":7D", ":7E", ":7F", + }; + + /// + /// Escapes the and trim storage key prefix. + /// + /// The storage key prefix. + /// The storage key limit. + public static string EscapeAndTrimStorageKeyPrefix(string storageKeyPrefix, int limit) + { + return StringHelper.TrimStorageKeyPrefix(StringHelper.EscapeStorageKey(storageKeyPrefix), limit); + } + + /// + /// Trims the storage key prefix. + /// + /// The storage key prefix. + /// The storage key limit. + private static string TrimStorageKeyPrefix(string storageKeyPrefix, int limit) + { + if (limit < StringHelper.StorageKeyTrimPadding) + { + throw new ArgumentException(message: $"The storage key limit should be at least {StringHelper.StorageKeyTrimPadding} characters.", paramName: nameof(limit)); + } + + return storageKeyPrefix.Length > limit - StringHelper.StorageKeyTrimPadding + ? storageKeyPrefix.Substring(0, limit - StringHelper.StorageKeyTrimPadding) + : storageKeyPrefix; + } + + /// + /// Escapes the storage key. + /// + /// The storage key. + public static string EscapeStorageKey(string storageKey) + { + var sb = new StringBuilder(storageKey.Length); + for (var index = 0; index < storageKey.Length; ++index) + { + var c = storageKey[index]; + if (c < 128) + { + sb.Append(StringHelper.EscapedStorageKeys[c]); + } + else if (char.IsLetterOrDigit(c)) + { + sb.Append(c); + } + else if (c < 0x100) + { + sb.Append(':'); + sb.Append(((int)c).ToString("X2", CultureInfo.InvariantCulture)); + } + else + { + sb.Append(':'); + sb.Append(':'); + sb.Append(((int)c).ToString("X4", CultureInfo.InvariantCulture)); + } + } + + return sb.ToString(); + } + + /// + /// Gets the name of the workflow job trigger queue. + /// + /// Index of the queue. + internal static string GetWorkflowQueueNameInternal(string prefix, int numPartitionsInJobTriggersQueue) + { + return string.Concat(prefix, (1 % numPartitionsInJobTriggersQueue).ToString("d2", CultureInfo.InvariantCulture)); + } + } +} diff --git a/Kudu.Core/Infrastructure/ContainerInfo.cs b/Kudu.Core/Infrastructure/ContainerInfo.cs new file mode 100644 index 00000000..66189a56 --- /dev/null +++ b/Kudu.Core/Infrastructure/ContainerInfo.cs @@ -0,0 +1,155 @@ +using System; +using System.Collections.Generic; +using System.Text.RegularExpressions; +using Newtonsoft.Json; + +namespace Kudu.Core.Infrastructure +{ + + public class SiteInstanceStats + { + [JsonProperty(PropertyName = "containers")] + public Dictionary appContainersOnThisInstance = new Dictionary(); + } + + public class ContainerInfo + { + + [JsonProperty(PropertyName = "read")] + public DateTime CurrentTimeStamp { get; set; } + + [JsonProperty(PropertyName = "preread")] + public DateTime PreviousTimeStamp { get; set; } + + [JsonProperty(PropertyName = "cpu_stats")] + public ContainerCpuStatistics CurrentCpuStats { get; set; } + + [JsonProperty(PropertyName = "precpu_stats")] + public ContainerCpuStatistics PreviousCpuStats { get; set; } + + [JsonProperty(PropertyName = "memory_stats")] + public ContainerMemoryStatistics MemoryStats { get; set; } + + [JsonProperty(PropertyName = "name")] public string Name { get; set; } + + [JsonProperty(PropertyName = "id")] public string Id { get; set; } + + [JsonProperty(PropertyName = "eth0")] public ContainerNetworkInterfaceStatistics Eth0 { get; set; } + + public string GetSiteName() + { + if (String.IsNullOrWhiteSpace(this.Name)) + { + return ""; + } + var startIndex = this.Name[0] == '/' ? 1 : 0; + // Remove the last _0 or something from the container name + var siteName = Regex.Replace(this.Name.Substring(startIndex), @"_\d+$", ""); + return siteName; + } + + } + + public class ContainerCpuUsage + { + [JsonProperty(PropertyName = "total_usage")] + public long TotalUsage { get; set; } + + [JsonProperty(PropertyName = "percpu_usage")] + public List PerCpuUsage { get; set; } + + [JsonProperty(PropertyName = "usage_in_kernelmode")] + public long KernelModeUsage { get; set; } + + [JsonProperty(PropertyName = "usage_in_usermode")] + public long UserModeUsage { get; set; } + } + + public class ContainerThrottlingData + { + [JsonProperty(PropertyName = "periods")] + public int Periods { get; set; } + + [JsonProperty(PropertyName = "throttled_periods")] + public int ThrolltledPeriods { get; set; } + + [JsonProperty(PropertyName = "throttled_time")] + public int ThrottledTime { get; set; } + } + + public class ContainerCpuStatistics + { + [JsonProperty(PropertyName = "cpu_usage")] + public ContainerCpuUsage CpuUsage { get; set; } + + [JsonProperty(PropertyName = "system_cpu_usage")] + public long SystemCpuUsage { get; set; } + + [JsonProperty(PropertyName = "online_cpus")] + public int OnlineCpuCount { get; set; } + + public ContainerThrottlingData ThrottlingData { get; set; } + } + + public class ContainerMemoryStatistics + { + [JsonProperty(PropertyName = "usage")] public long Usage { get; set; } + + [JsonProperty(PropertyName = "max_usage")] + public long MaxUsage { get; set; } + + [JsonProperty(PropertyName = "limit")] public long Limit { get; set; } + } + + public class ContainerNetworkInterfaceStatistics + { + [JsonProperty(PropertyName = "rx_bytes")] + public long RxBytes { get; set; } + + [JsonProperty(PropertyName = "rx_packets")] + public long RxPackets { get; set; } + + [JsonProperty(PropertyName = "rx_errors")] + public long RxErrors { get; set; } + + [JsonProperty(PropertyName = "rx_dropped")] + public long RxDropped { get; set; } + + [JsonProperty(PropertyName = "tx_bytes")] + public long TxBytes { get; set; } + + [JsonProperty(PropertyName = "tx_packets")] + public long TxPackets { get; set; } + + [JsonProperty(PropertyName = "tx_errors")] + public long TxErrors { get; set; } + + [JsonProperty(PropertyName = "tx_dropped")] + public long TxDropped { get; set; } + } + + + public class ContainerSimpleStatus + { + [JsonProperty(PropertyName = "TimeStamp")] + public DateTime TimeStamp { get; set; } + + [JsonProperty(PropertyName = "name")] public string Name { get; set; } + + [JsonProperty(PropertyName = "cpuUsage")] + public long CpuUsage { get; set; } + + [JsonProperty(PropertyName = "systemCpuUsage")] + public long SystemCpuUsage { get; set; } + + [JsonProperty(PropertyName = "memoryUsage")] + public long MemoryUsage { get; set; } + + [JsonProperty(PropertyName = "maxMemoryUsage")] + public long MaxMemoryUsage { get; set; } + + [JsonProperty(PropertyName = "memoryLimit")] + public long MemoryLimit { get; set; } + } + +} \ No newline at end of file diff --git a/Kudu.Core/Infrastructure/DockerContainerRestartTrigger.cs b/Kudu.Core/Infrastructure/DockerContainerRestartTrigger.cs index 5b4e1ec0..f082acac 100644 --- a/Kudu.Core/Infrastructure/DockerContainerRestartTrigger.cs +++ b/Kudu.Core/Infrastructure/DockerContainerRestartTrigger.cs @@ -1,7 +1,14 @@ using System; +using System.Collections.Generic; +using System.Diagnostics; using System.Globalization; using System.IO; +using System.Linq; +using Kudu.Contracts.Deployment; +using Kudu.Core.Functions; using Kudu.Core.Helpers; +using Kudu.Core.K8SE; +using Newtonsoft.Json; namespace Kudu.Core.Infrastructure { @@ -20,8 +27,39 @@ public static class DockerContainerRestartTrigger "The last modification Kudu made to this file was at {0}, for the following reason: {1}.", System.Environment.NewLine); - public static void RequestContainerRestart(IEnvironment environment, string reason) + public static void RequestContainerRestart(IEnvironment environment, string reason, string repositoryUrl = null, string appSubPath = "", IDictionary appSettings = null) { + if (appSettings is null) + { + appSettings = new Dictionary(); + } + + if (K8SEDeploymentHelper.IsK8SEEnvironment()) + { + string appName = environment.K8SEAppName; + string appType = environment.K8SEAppType; + string buildNumber = environment.CurrId; + var functionTriggers = KedaFunctionTriggerProvider.GetFunctionTriggers(repositoryUrl, appName, appType, appSettings); + var buildMetadata = new BuildMetadata() + { + AppName = appName, + BuildVersion = buildNumber, + AppSubPath = appSubPath + }; + + //Only for function apps functionTriggers will be non-null/non-empty + if (functionTriggers?.Any() == true) + { + K8SEDeploymentHelper.UpdateFunctionAppTriggers(appName, functionTriggers, buildMetadata); + } + else + { + K8SEDeploymentHelper.UpdateBuildNumber(appName, buildMetadata); + } + + return; + } + if (OSDetector.IsOnWindows() && !EnvironmentHelper.IsWindowsContainers()) { throw new NotSupportedException("RequestContainerRestart is only supported on Linux and Windows Containers"); diff --git a/Kudu.Core/Infrastructure/ExecutableExtensions.cs b/Kudu.Core/Infrastructure/ExecutableExtensions.cs index dc85ed34..a8290058 100644 --- a/Kudu.Core/Infrastructure/ExecutableExtensions.cs +++ b/Kudu.Core/Infrastructure/ExecutableExtensions.cs @@ -27,7 +27,7 @@ public static void PrependToPath(this Executable exe, IEnumerable paths) public static void AddDeploymentSettingsAsEnvironmentVariables(this Executable exe, IDeploymentSettingsManager deploymentSettingsManager) { - IEnumerable> deploymentSettings = deploymentSettingsManager.GetValues(); + IEnumerable> deploymentSettings = deploymentSettingsManager.GetValues(new Dictionary()); foreach (var keyValuePair in deploymentSettings) { exe.EnvironmentVariables[keyValuePair.Key] = keyValuePair.Value; diff --git a/Kudu.Core/Infrastructure/FileSystemHelpers.cs b/Kudu.Core/Infrastructure/FileSystemHelpers.cs index 4082a980..3d6132f9 100644 --- a/Kudu.Core/Infrastructure/FileSystemHelpers.cs +++ b/Kudu.Core/Infrastructure/FileSystemHelpers.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.IO; using System.IO.Abstractions; @@ -375,6 +376,28 @@ private static void DeleteDirectoryContentsSafe(DirectoryInfoBase directoryInfo, } } + public static void CreateRelativeSymlinks(string source, string destination, TimeSpan timeout) + { + string directory = FileSystemHelpers.GetDirectoryName(source); + string cmd = String.Format("cd {0}; timeout {1}s ln -s {2} {3}", directory, timeout.TotalSeconds, destination, source); + var escapedArgs = cmd.Replace("\"", "\\\""); + + var process = new Process + { + StartInfo = new ProcessStartInfo + { + RedirectStandardOutput = true, + UseShellExecute = false, + CreateNoWindow = true, + WindowStyle = ProcessWindowStyle.Hidden, + FileName = "/bin/bash", + Arguments = $"-c \"{escapedArgs}\"" + } + }; + process.Start(); + process.WaitForExit(); + } + private static void DeleteFileSystemInfo(FileSystemInfoBase fileSystemInfo, bool ignoreErrors) { if (!fileSystemInfo.Exists) diff --git a/Kudu.Core/Infrastructure/FunctionAppHelper.cs b/Kudu.Core/Infrastructure/FunctionAppHelper.cs index 67ee56ab..3a4fe65e 100644 --- a/Kudu.Core/Infrastructure/FunctionAppHelper.cs +++ b/Kudu.Core/Infrastructure/FunctionAppHelper.cs @@ -1,4 +1,5 @@ -using System; +using Kudu.Contracts.Settings; +using System; using System.Linq; namespace Kudu.Core.Infrastructure @@ -10,6 +11,11 @@ public static bool LooksLikeFunctionApp() return !string.IsNullOrEmpty(System.Environment.GetEnvironmentVariable(Constants.FunctionRunTimeVersion)); } + public static bool HasScmRunFromPackage() + { + return !string.IsNullOrEmpty(System.Environment.GetEnvironmentVariable(Constants.ScmRunFromPackage)); + } + public static bool IsCSharpFunctionFromProjectFile(string projectPath) { return VsHelper.IncludesAnyReferencePackage(projectPath, "Microsoft.NET.Sdk.Functions"); diff --git a/Kudu.Core/Infrastructure/IServerConfiguration.cs b/Kudu.Core/Infrastructure/IServerConfiguration.cs index ed473c10..fd7e5134 100644 --- a/Kudu.Core/Infrastructure/IServerConfiguration.cs +++ b/Kudu.Core/Infrastructure/IServerConfiguration.cs @@ -4,6 +4,6 @@ public interface IServerConfiguration { string ApplicationName { get; } - string GitServerRoot { get; } + string GitServerRoot { get; set; } } } diff --git a/Kudu.Core/Infrastructure/InstanceIdUtility.cs b/Kudu.Core/Infrastructure/InstanceIdUtility.cs index 9e73b016..5900100f 100644 --- a/Kudu.Core/Infrastructure/InstanceIdUtility.cs +++ b/Kudu.Core/Infrastructure/InstanceIdUtility.cs @@ -26,7 +26,7 @@ private static void EnsureInstanceId() return; } - string instanceId = System.Environment.GetEnvironmentVariable("WEBSITE_INSTANCE_ID"); + string instanceId = System.Environment.GetEnvironmentVariable(Constants.AzureWebsiteInstanceId); if (String.IsNullOrEmpty(instanceId)) { instanceId = System.Environment.MachineName; diff --git a/Kudu.Core/Infrastructure/ServerConfiguration.cs b/Kudu.Core/Infrastructure/ServerConfiguration.cs index ed995430..de64b2bb 100644 --- a/Kudu.Core/Infrastructure/ServerConfiguration.cs +++ b/Kudu.Core/Infrastructure/ServerConfiguration.cs @@ -6,6 +6,8 @@ public class ServerConfiguration : IServerConfiguration { private string _applicationName; + private string gitRoot = ""; + public string ApplicationName { get @@ -22,12 +24,19 @@ public string GitServerRoot { get { + if(!String.IsNullOrEmpty(gitRoot)) + { + return gitRoot; + } + if (String.IsNullOrEmpty(ApplicationName)) { return "git"; } return ApplicationName + ".git"; } + + set { gitRoot = value; } } public static string GetApplicationName() diff --git a/Kudu.Core/Infrastructure/ZipArchiveExtensions.cs b/Kudu.Core/Infrastructure/ZipArchiveExtensions.cs index 3b44c28a..031c577f 100644 --- a/Kudu.Core/Infrastructure/ZipArchiveExtensions.cs +++ b/Kudu.Core/Infrastructure/ZipArchiveExtensions.cs @@ -1,9 +1,12 @@ using System; +using System.Collections; +using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.IO.Abstractions; using System.IO.Compression; using Kudu.Contracts.Tracing; +using Kudu.Core.Helpers; using Kudu.Core.Tracing; namespace Kudu.Core.Infrastructure @@ -107,8 +110,10 @@ public static ZipArchiveEntry AddFile(this ZipArchive zip, string fileName, stri return entry; } - public static void Extract(this ZipArchive archive, string directoryName) + public static IDictionary Extract(this ZipArchive archive, string directoryName) { + IDictionary symLinks = new Dictionary(); + bool isSymLink = false; foreach (ZipArchiveEntry entry in archive.Entries) { string path = Path.Combine(directoryName, entry.FullName); @@ -125,16 +130,58 @@ public static void Extract(this ZipArchive archive, string directoryName) { fileInfo.Directory.Create(); } - using (Stream zipStream = entry.Open(), - fileStream = fileInfo.Open(FileMode.Create, FileAccess.Write)) + fileStream = fileInfo.Open(FileMode.Create, FileAccess.Write, FileShare.ReadWrite)) { zipStream.CopyTo(fileStream); } - fileInfo.LastWriteTimeUtc = entry.LastWriteTime.ToUniversalTime().DateTime; + isSymLink = false; + string originalFileName = string.Empty; + + if (!OSDetector.IsOnWindows()) + { + try + { + using (Stream fs = fileInfo.Open(FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + { + byte[] buffer = new byte[4]; + fs.Read(buffer, 0, buffer.Length); + fs.Close(); + + var str = System.Text.Encoding.Default.GetString(buffer); + if (str.StartsWith("../")) + { + using (StreamReader reader = fileInfo.OpenText()) + { + symLinks[entry.FullName] = reader.ReadToEnd(); + } + isSymLink = true; + } + } + } + catch (Exception ex) + { + throw new Exception("Could not identify symlinks in zip file : " + ex.ToString()); + } + } + + try + { + fileInfo.LastWriteTimeUtc = entry.LastWriteTime.ToUniversalTime().DateTime; + } + catch(Exception) + { + //best effort + } + + if(isSymLink) + { + fileInfo.Delete(); + } } } + return symLinks; } private static string EnsureTrailingSlash(string input) diff --git a/Kudu.Core/K8SE/BuildCtlArgumentsHelper.cs b/Kudu.Core/K8SE/BuildCtlArgumentsHelper.cs new file mode 100644 index 00000000..384e6974 --- /dev/null +++ b/Kudu.Core/K8SE/BuildCtlArgumentsHelper.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Kudu.Core.K8SE +{ + internal class BuildCtlArgumentsHelper + { + internal static void AddBuildCtlCommand(StringBuilder args, string verb) + { + args.AppendFormat("buildctl {0} ", verb); + } + + internal static void AddAppNameArgument(StringBuilder args, string appName) + { + args.AppendFormat(" -appName {0}", appName); + } + + internal static void AddAppPropertyArgument(StringBuilder args, string appProperty) + { + args.AppendFormat(" -appProperty {0}", appProperty); + } + + internal static void AddAppPropertyValueArgument(StringBuilder args, string appPropertyValue) + { + args.AppendFormat(" -appPropertyValue {0}", appPropertyValue); + } + + internal static void AddFunctionTriggersJsonToPatchValueArgument(StringBuilder args, string jsonToPatch) + { + args.AppendFormat(" -jsonToPatch {0}", jsonToPatch); + } + + } +} diff --git a/Kudu.Core/K8SE/K8SEDeploymentHelper.cs b/Kudu.Core/K8SE/K8SEDeploymentHelper.cs new file mode 100644 index 00000000..52451087 --- /dev/null +++ b/Kudu.Core/K8SE/K8SEDeploymentHelper.cs @@ -0,0 +1,252 @@ +using Kudu.Contracts.Deployment; +using Kudu.Contracts.Tracing; +using Kudu.Core.Deployment; +using Kudu.Core.Functions; +using Microsoft.AspNetCore.Http; +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Runtime.Caching; +using System.Text; +using Microsoft.Extensions.Primitives; + +namespace Kudu.Core.K8SE +{ + public static class K8SEDeploymentHelper + { + + public static ITracer _tracer; + public static ILogger _logger; + private static ObjectCache cache = MemoryCache.Default; + private static CacheItemPolicy instanceCachePolicy = new CacheItemPolicy + { + AbsoluteExpiration = DateTimeOffset.Now.AddSeconds(30.0), + + }; + + // K8SE_BUILD_SERVICE not null or empty + public static bool IsK8SEEnvironment() + { + return !String.IsNullOrEmpty(System.Environment.GetEnvironmentVariable(Constants.IsK8SEEnvironment)); + } + + /// + /// Calls into buildctl to retrieve BuildVersion of + /// the K8SE App + /// + /// + /// + public static string GetLinuxFxVersion(string appName) + { + var cmd = new StringBuilder(); + BuildCtlArgumentsHelper.AddBuildCtlCommand(cmd, "get"); + BuildCtlArgumentsHelper.AddAppNameArgument(cmd, appName); + BuildCtlArgumentsHelper.AddAppPropertyArgument(cmd, "linuxFxVersion"); + return RunBuildCtlCommand(cmd.ToString(), "Retrieving framework info..."); + } + + /// + /// Calls into buildctl to get a list of instaces for an app + /// + /// + /// + public static List GetInstances(string appName) + { + var cachedInstances = cache.Get(appName); + if (cachedInstances == null) + { + var cmd = new StringBuilder(); + BuildCtlArgumentsHelper.AddBuildCtlCommand(cmd, "get"); + BuildCtlArgumentsHelper.AddAppNameArgument(cmd, appName); + BuildCtlArgumentsHelper.AddAppPropertyArgument(cmd, "podInstances"); + var instList = RunBuildCtlCommand(cmd.ToString(), "Getting app instances..."); + byte[] data = Convert.FromBase64String(instList); + string json = Encoding.UTF8.GetString(data); + cachedInstances = JsonConvert.DeserializeObject>(json); + cache.Add(appName, cachedInstances, instanceCachePolicy); + } + + return (List)cachedInstances; + } + + /// + /// Calls into buildctl to update a BuildVersion of + /// the K8SE App + /// + /// + /// + public static void UpdateBuildNumber(string appName, BuildMetadata buildMetadata) + { + var cmd = new StringBuilder(); + BuildCtlArgumentsHelper.AddBuildCtlCommand(cmd, "update"); + BuildCtlArgumentsHelper.AddAppNameArgument(cmd, appName); + BuildCtlArgumentsHelper.AddAppPropertyArgument(cmd, "buildMetadata"); + BuildCtlArgumentsHelper.AddAppPropertyValueArgument(cmd, $"\\\"{GetBuildMetadataStr(buildMetadata)}\\\""); + RunBuildCtlCommand(cmd.ToString(), "Updating build version..."); + } + + /// + /// Updates the Image Tag of the K8SE custom container app + /// + /// + /// container image tag of the format registry/: + /// + public static void UpdateImageTag(string appName, string imageTag) + { + var cmd = new StringBuilder(); + BuildCtlArgumentsHelper.AddBuildCtlCommand(cmd, "update"); + BuildCtlArgumentsHelper.AddAppNameArgument(cmd, appName); + BuildCtlArgumentsHelper.AddAppPropertyArgument(cmd, "appImage"); + BuildCtlArgumentsHelper.AddAppPropertyValueArgument(cmd, imageTag); + RunBuildCtlCommand(cmd.ToString(), "Updating image tag..."); + } + + /// + /// Updates the triggers for the function apps + /// + /// The app name to update + /// The IEnumerable + /// Build number to update + public static void UpdateFunctionAppTriggers(string appName, IEnumerable functionTriggers, BuildMetadata buildMetadata) + { + var functionAppPatchJson = GetFunctionAppPatchJson(functionTriggers, buildMetadata); + if (string.IsNullOrEmpty(functionAppPatchJson)) + { + return; + } + + var cmd = new StringBuilder(); + BuildCtlArgumentsHelper.AddBuildCtlCommand(cmd, "updatejson"); + BuildCtlArgumentsHelper.AddAppNameArgument(cmd, appName); + BuildCtlArgumentsHelper.AddFunctionTriggersJsonToPatchValueArgument(cmd, functionAppPatchJson); + RunBuildCtlCommand(cmd.ToString(), "Updating function app triggers..."); + } + + private static string RunBuildCtlCommand(string args, string msg) + { + Console.WriteLine($"{msg} : {args}"); + var process = new Process() + { + StartInfo = new ProcessStartInfo + { + FileName = "/bin/bash", + Arguments = $"-c \"{args}\"", + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + } + }; + + process.Start(); + string output = process.StandardOutput.ReadToEnd(); + string error = process.StandardError.ReadToEnd(); + Console.WriteLine($"buildctl output:\n {output}"); + process.WaitForExit(); + + if (string.IsNullOrEmpty(error)) + { + return output; + } + else + { + throw new Exception(error); + } + } + + public static string GetAppName(HttpContext context) + { + var appName = context.Request.Headers["K8SE_APP_NAME"].ToString(); + + if (string.IsNullOrEmpty(appName)) + { + context.Response.StatusCode = 401; + // K8SE TODO: move this to resource map + throw new InvalidOperationException("Couldn't recognize AppName"); + } + return appName; + } + + public static string GetAppKind(HttpContext context) + { + var appKind = context.Request.Headers["K8SE_APP_KIND"].ToString(); + //K8SE_APP_KIND is only needed for the logic apps, for web apps and function apps, fallback to "kubeapp" + appKind = string.IsNullOrEmpty(appKind) ? "kubeapp" : appKind; + if (string.IsNullOrEmpty(appKind)) + { + context.Response.StatusCode = 401; + // K8SE TODO: move this to resource map + throw new InvalidOperationException("Couldn't recognize AppKind"); + } + + return appKind; + } + + public static string GetAppNamespace(HttpContext context) + { + var appNamepace = context.Request.Headers["K8SE_APP_NAMESPACE"].ToString(); + return appNamepace; + } + + public static void UpdateContextWithAppSettings(HttpContext context) + { + Dictionary appSettings = new Dictionary(StringComparer.InvariantCultureIgnoreCase); + var appSettingsPrefix = "appsetting_"; + var appSettingsWithHeader = context.Request.Headers + .Where(p => p.Key.StartsWith(appSettingsPrefix, StringComparison.OrdinalIgnoreCase)); + + foreach (var setting in appSettingsWithHeader) + { + var key = setting.Key.Substring(appSettingsPrefix.Length); + appSettings[key] = setting.Value; + } + + // Filter out App Settings headers + foreach (var key in appSettingsWithHeader.ToList()) + { + context.Request.Headers.Remove(key); + } + + context.Items.TryAdd("appSettings", appSettings); + } + + private static string GetFunctionAppPatchJson(IEnumerable functionTriggers, BuildMetadata buildMetadata) + { + if ((functionTriggers == null || !functionTriggers.Any()) && buildMetadata == null) + { + return null; + } + + var patchAppJson = new PatchAppJson { PatchSpec = new PatchSpec { } }; + if (functionTriggers?.Any() == true) + { + patchAppJson.PatchSpec.TriggerOptions = new TriggerOptions + { + Triggers = functionTriggers + }; + } + + if (buildMetadata != null) + { + patchAppJson.PatchSpec.Code = new CodeSpec + { + PackageRef = new PackageReference + { + BuildMetadata = GetBuildMetadataStr(buildMetadata), + } + }; + } + + var str= System.Convert.ToBase64String(ASCIIEncoding.ASCII.GetBytes(JsonConvert.SerializeObject(patchAppJson))); + Console.WriteLine("Test Str: " + str); + return str; + } + + private static string GetBuildMetadataStr(BuildMetadata buildMetadata) + { + return $"{buildMetadata.AppName}|{buildMetadata.BuildVersion}|{System.Convert.ToBase64String(ASCIIEncoding.ASCII.GetBytes(JsonConvert.SerializeObject(buildMetadata)))}"; + } + } +} diff --git a/Kudu.Core/K8SE/PodInstance.cs b/Kudu.Core/K8SE/PodInstance.cs new file mode 100644 index 00000000..690f5c82 --- /dev/null +++ b/Kudu.Core/K8SE/PodInstance.cs @@ -0,0 +1,19 @@ +using Newtonsoft.Json; + +namespace Kudu.Core.K8SE +{ + public class PodInstance + { + [JsonProperty(PropertyName = "name")] + public string Name { get; set; } + + [JsonProperty(PropertyName = "ipAddr")] + public string IpAddress { get; set; } + + [JsonProperty(PropertyName = "nodeName")] + public string NodeName { get; set; } + + [JsonProperty(PropertyName = "startTime")] + public string StartTime { get; set; } + } +} diff --git a/Kudu.Core/Kube/SecretProvider.cs b/Kudu.Core/Kube/SecretProvider.cs new file mode 100644 index 00000000..a3435a2c --- /dev/null +++ b/Kudu.Core/Kube/SecretProvider.cs @@ -0,0 +1,112 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Net.Http; +using System.Threading.Tasks; +using System.IO; +using System.Net.Security; +using System.Security.Cryptography.X509Certificates; + +namespace Kudu.Core.Kube +{ + + public class SecretProvider + { + private readonly string _secretKubeApiUrlPlaceHolder = "https://kubernetes.default.svc.cluster.local/api/v1/namespaces/{0}/secrets/{1}"; + private readonly string _rbacServiceActTokenFilePath = "/var/run/secrets/kubernetes.io/serviceaccount/token"; + private const string _caFilePath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"; + + public async Task GetSecretContent(string secretName, string secretNamespace) + { + var responseBodyContent = ""; + var secretKubeApiUrl = string.Format(_secretKubeApiUrlPlaceHolder, secretNamespace, secretName); + var accessToken = await GetAccessToken(); + var httpClient = CreateHttpClient(); + httpClient.DefaultRequestHeaders.Add("Authorization", "Bearer " + accessToken); + var responseMessage = await httpClient.GetAsync(secretKubeApiUrl); + + if (responseMessage.StatusCode == System.Net.HttpStatusCode.NotFound) + { + return responseBodyContent; + } + + using (var reader = new StreamReader(await responseMessage.Content.ReadAsStreamAsync())) + { + responseBodyContent = await reader.ReadToEndAsync(); + } + + return responseBodyContent; + } + + private async Task GetAccessToken() + { + var accessToken = ""; + using (var sr = File.OpenText(_rbacServiceActTokenFilePath)) + { + accessToken = await sr.ReadToEndAsync(); + } + + return accessToken; + } + + private static HttpClient CreateHttpClient() + { + var client = new HttpClient(new HttpClientHandler + { + ServerCertificateCustomValidationCallback = ServerCertificateValidationCallback + }); + + return client; + } + + private static bool ServerCertificateValidationCallback( + HttpRequestMessage request, + X509Certificate2 certificate, + X509Chain certChain, + SslPolicyErrors sslPolicyErrors) + { + if (sslPolicyErrors == SslPolicyErrors.None) + { + // certificate is already valid + return true; + } + else if (sslPolicyErrors == SslPolicyErrors.RemoteCertificateNameMismatch || + sslPolicyErrors == SslPolicyErrors.RemoteCertificateNotAvailable) + { + // api-server cert must exist and have the right subject + return false; + } + else if (sslPolicyErrors == SslPolicyErrors.RemoteCertificateChainErrors) + { + // only remaining error state is RemoteCertificateChainErrors + // check custom CA + var privateChain = new X509Chain(); + privateChain.ChainPolicy.RevocationMode = X509RevocationMode.NoCheck; + + var caCert = new X509Certificate2(_caFilePath); + // https://docs.microsoft.com/en-us/dotnet/api/system.security.cryptography.x509certificates.x509chainpolicy?view=netcore-2.2 + // Add CA cert to the chain store to include it in the chain check. + privateChain.ChainPolicy.ExtraStore.Add(caCert); + // Build the chain for `certificate` which should be the self-signed kubernetes api-server cert. + privateChain.Build(certificate); + + foreach (X509ChainStatus chainStatus in privateChain.ChainStatus) + { + if (chainStatus.Status != X509ChainStatusFlags.NoError && + // root CA cert is not always trusted. + chainStatus.Status != X509ChainStatusFlags.UntrustedRoot) + { + return false; + } + } + + return true; + } + else + { + // Unknown sslPolicyErrors + return false; + } + } + } +} diff --git a/Kudu.Core/Kube/SyncTriggerAuthenticator.cs b/Kudu.Core/Kube/SyncTriggerAuthenticator.cs new file mode 100644 index 00000000..450fbe29 --- /dev/null +++ b/Kudu.Core/Kube/SyncTriggerAuthenticator.cs @@ -0,0 +1,161 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Linq; +using System.Threading.Tasks; +using Newtonsoft.Json.Linq; +using System.Security.Cryptography; +using System.IO; + +namespace Kudu.Core.Kube +{ + public class SyncTriggerAuthenticator + { + private const string SiteTokenHeaderName = "x-ms-site-restricted-token"; + private const string FuncAppEncryptionKeyName = "WEBSITE_AUTH_ENCRYPTION_KEY"; + private const string FuncAppNameHeaderKey = "K8SE_APP_NAME"; + private const string FuncAppNamespaceHeaderKey = "K8SE_APP_NAMESPACE"; + public async static Task AuthenticateCaller(Dictionary> headers) + { + if (headers == null || !headers.Any()) + { + return false; + } + + //If there's no encryption key in the header return true + if (!headers.TryGetValue(SiteTokenHeaderName, out IEnumerable siteTokenHeaderValue)) + { + return false; + } + + //Auth header value is null or empty return false + var funcAppAuthToken = siteTokenHeaderValue.FirstOrDefault(); + if (string.IsNullOrEmpty(funcAppAuthToken)) + { + return false; + } + + //If there's no app name or app namespace in the header return false + if (!headers.TryGetValue(FuncAppNameHeaderKey, out IEnumerable funcAppNameHeaderValue) + || !headers.TryGetValue(FuncAppNamespaceHeaderKey, out IEnumerable funcAppNamespaceHeaderValue)) + { + return false; + } + + var funcAppName = funcAppNameHeaderValue.FirstOrDefault(); + var funcAppNamespace = funcAppNamespaceHeaderValue.FirstOrDefault(); + if (string.IsNullOrEmpty(funcAppName) || string.IsNullOrEmpty(funcAppNamespace)) + { + return false; + } + + //If the encryption key secret is null or empty in the Kubernetes - return false + var secretProvider = new SecretProvider(); + var encryptionKeySecretContent = await secretProvider.GetSecretContent(funcAppName + "-secrets".ToLower(), funcAppNamespace); + if (string.IsNullOrEmpty(encryptionKeySecretContent)) + { + return false; + } + + var encryptionSecretJObject = JObject.Parse(encryptionKeySecretContent); + var functionEncryptionKey = Base64Decode((string)encryptionSecretJObject["data"][FuncAppEncryptionKeyName]); + if (string.IsNullOrEmpty(functionEncryptionKey)) + { + return false; + } + + var decryptedToken = Decrypt(GetKeyBytes(functionEncryptionKey), funcAppAuthToken); + + return ValidateToken(decryptedToken); + } + + public static byte[] GetKeyBytes(string hexOrBase64) + { + // only support 32 bytes (256 bits) key length + if (hexOrBase64.Length == 64) + { + return Enumerable.Range(0, hexOrBase64.Length) + .Where(x => x % 2 == 0) + .Select(x => Convert.ToByte(hexOrBase64.Substring(x, 2), 16)) + .ToArray(); + } + + return Convert.FromBase64String(hexOrBase64); + } + + private static bool ValidateToken(string token) + { + if (string.IsNullOrEmpty(token)) + { + return false; + } + + var tokenValues = token.Split('='); + if (tokenValues.Length < 2) + { + return false; + } + + long ticksVal = 0; + if (!long.TryParse(tokenValues[1], out ticksVal)) + { + return false; + } + + //The token will be valid only for the next 5 more minutes after being generated + DateTime myDate = new DateTime(ticksVal); + if (myDate.AddMinutes(5) < DateTime.UtcNow) + { + return false; + } + + return true; + } + + private static string Base64Decode(string base64EncodedData) + { + var base64EncodedBytes = Convert.FromBase64String(base64EncodedData); + return Encoding.UTF8.GetString(base64EncodedBytes); + } + + private static string Decrypt(byte[] encryptionKey, string value) + { + var parts = value.Split(new[] { '.' }, StringSplitOptions.RemoveEmptyEntries); + if (parts.Length != 2 && parts.Length != 3) + { + throw new InvalidOperationException("Malformed token."); + } + + var iv = Convert.FromBase64String(parts[0]); + var data = Convert.FromBase64String(parts[1]); + var base64KeyHash = parts.Length == 3 ? parts[2] : null; + + if (!string.IsNullOrEmpty(base64KeyHash) && !string.Equals(GetSHA256Base64String(encryptionKey), base64KeyHash)) + { + throw new InvalidOperationException(string.Format("Key with hash {0} does not exist.", base64KeyHash)); + } + + using (var aes = new AesManaged { Key = encryptionKey }) + { + using (var ms = new MemoryStream()) + { + using (var cs = new CryptoStream(ms, aes.CreateDecryptor(aes.Key, iv), CryptoStreamMode.Write)) + using (var binaryWriter = new BinaryWriter(cs)) + { + binaryWriter.Write(data, 0, data.Length); + } + + return Encoding.UTF8.GetString(ms.ToArray()); + } + } + } + + private static string GetSHA256Base64String(byte[] key) + { + using (var sha256 = new SHA256Managed()) + { + return Convert.ToBase64String(sha256.ComputeHash(key)); + } + } + } +} diff --git a/Kudu.Core/Kudu.Core.csproj b/Kudu.Core/Kudu.Core.csproj index adaad2bb..9f1d8ee3 100644 --- a/Kudu.Core/Kudu.Core.csproj +++ b/Kudu.Core/Kudu.Core.csproj @@ -1,7 +1,7 @@  - netcoreapp2.2 + netcoreapp3.1 true @@ -24,18 +24,22 @@ + + - - + + - + + + diff --git a/Kudu.Core/LinuxConsumptionEnvironment.cs b/Kudu.Core/LinuxConsumptionEnvironment.cs new file mode 100644 index 00000000..51bdd7bd --- /dev/null +++ b/Kudu.Core/LinuxConsumptionEnvironment.cs @@ -0,0 +1,114 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Kudu.Contracts; +using Kudu.Contracts.Settings; + +namespace Kudu.Core +{ + public class LinuxConsumptionEnvironment : ILinuxConsumptionEnvironment + { + private readonly ReaderWriterLockSlim _delayLock = new ReaderWriterLockSlim(); + private TaskCompletionSource _delayTaskCompletionSource; + private bool? _standbyMode; + + public LinuxConsumptionEnvironment() + { + _delayTaskCompletionSource = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + _delayTaskCompletionSource.SetResult(null); + } + + public bool DelayRequestsEnabled => !DelayCompletionTask.IsCompleted; + + public Task DelayCompletionTask + { + get + { + _delayLock.EnterReadLock(); + try + { + return _delayTaskCompletionSource.Task; + } + finally + { + _delayLock.ExitReadLock(); + } + } + } + + public bool InStandbyMode + { + get + { + // once set, never reset + if (_standbyMode != null) + { + return _standbyMode.Value; + } + if (IsPlaceHolderModeEnabled()) + { + return true; + } + + // no longer standby mode + _standbyMode = false; + + return _standbyMode.Value; + } + } + + Task ILinuxConsumptionEnvironment.DelayCompletionTask => throw new NotImplementedException(); + + public void DelayRequests() + { + _delayLock.EnterUpgradeableReadLock(); + try + { + if (_delayTaskCompletionSource.Task.IsCompleted) + { + _delayLock.EnterWriteLock(); + try + { + _delayTaskCompletionSource = new TaskCompletionSource(); + } + finally + { + _delayLock.ExitWriteLock(); + } + } + } + finally + { + _delayLock.ExitUpgradeableReadLock(); + } + } + + public void ResumeRequests() + { + _delayLock.EnterReadLock(); + try + { + _delayTaskCompletionSource?.SetResult(null); + } + finally + { + _delayLock.ExitReadLock(); + } + } + + public void FlagAsSpecializedAndReady() + { + System.Environment.SetEnvironmentVariable(SettingsKeys.PlaceholderMode, "0"); + System.Environment.SetEnvironmentVariable(SettingsKeys.ContainerReady, "1"); + } + + private bool IsPlaceHolderModeEnabled() + { + // If WEBSITE_PLACEHOLDER_MODE is not set, we consider the container as a placeholder + string placeHolderMode = System.Environment.GetEnvironmentVariable(SettingsKeys.PlaceholderMode); + return placeHolderMode == "1" || string.IsNullOrEmpty(placeHolderMode); + } + } +} diff --git a/Kudu.Core/Properties/AssemblyInfo.cs b/Kudu.Core/Properties/AssemblyInfo.cs new file mode 100644 index 00000000..9e48b835 --- /dev/null +++ b/Kudu.Core/Properties/AssemblyInfo.cs @@ -0,0 +1,4 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Kudu.Tests")] +[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2")] \ No newline at end of file diff --git a/Kudu.Core/Resources.Designer.cs b/Kudu.Core/Resources.Designer.cs index 73507117..64d4d6da 100644 --- a/Kudu.Core/Resources.Designer.cs +++ b/Kudu.Core/Resources.Designer.cs @@ -19,7 +19,7 @@ namespace Kudu.Core { // class via a tool like ResGen or Visual Studio. // To add or remove a member, edit your .ResX file then rerun ResGen // with the /str option, or rebuild your VS project. - [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "15.0.0.0")] + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "16.0.0.0")] [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] internal class Resources { @@ -78,6 +78,15 @@ internal static string Deployment_UnknownValue { } } + /// + /// Looks up a localized string similar to Deployment Underway. + /// + internal static string DeploymentLockOccMsg { + get { + return ResourceManager.GetString("DeploymentLockOccMsg", resourceCulture); + } + } + /// /// Looks up a localized string similar to Unable to determine which project file to build. {0}.. /// @@ -356,7 +365,18 @@ internal static string Log_TriggeringContainerRestart { return ResourceManager.GetString("Log_TriggeringContainerRestart", resourceCulture); } } - + + /// + /// Looks up a localized string similar to K8SEApp will begin restart.. + /// + internal static string Log_TriggeringK8SERestart + { + get + { + return ResourceManager.GetString("Log_TriggeringK8SERestart", resourceCulture); + } + } + /// /// Looks up a localized string similar to An unknown error has occurred. Check the diagnostic log for details.. /// @@ -393,6 +413,15 @@ internal static string ReceivingChanges { } } + /// + /// Looks up a localized string similar to Scan Underway. + /// + internal static string ScanUnderwayMsg { + get { + return ResourceManager.GetString("ScanUnderwayMsg", resourceCulture); + } + } + /// /// Looks up a localized string similar to Building and Deploying '{0}'.. /// diff --git a/Kudu.Core/Resources.resx b/Kudu.Core/Resources.resx index 0dadf98b..0c4b8f25 100644 --- a/Kudu.Core/Resources.resx +++ b/Kudu.Core/Resources.resx @@ -120,6 +120,9 @@ Copied '{0}'. + + Deployment Underway + N/A @@ -228,6 +231,9 @@ Receiving changes. + + Scan Underway + Building and Deploying '{0}'. diff --git a/Kudu.Core/Scan/ScanManager.cs b/Kudu.Core/Scan/ScanManager.cs new file mode 100644 index 00000000..a1fe0bf4 --- /dev/null +++ b/Kudu.Core/Scan/ScanManager.cs @@ -0,0 +1,574 @@ +using Kudu.Contracts.Infrastructure; +using Kudu.Contracts.Scan; +using Kudu.Contracts.Tracing; +using Kudu.Core.Commands; +using Kudu.Core.Infrastructure; +using Kudu.Core.Tracing; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace Kudu.Core.Scan +{ + public class ScanManager : IScanManager + { + + private readonly ITracer _tracer; + private readonly AllSafeLinuxLock _scanLock; + private static readonly string DATE_TIME_FORMAT = "yyyy-MM-dd_HH-mm-ssZ"; + // private string tempScanFilePath = null; + + public ScanManager(ITracer tracer, IDictionary namedLocks) + { + _tracer = tracer; + _scanLock = (AllSafeLinuxLock) namedLocks["deployment"]; + } + + private static void UpdateScanStatus(String folderPath,ScanStatus status,String Id) + { + String filePath = Path.Combine(folderPath, Constants.ScanStatusFile); + ScanStatusResult obj = ReadScanStatusFile("", "", Constants.ScanStatusFile, folderPath); + + //Create new Scan Id if file is empty + //else get existing scan Id + if(obj == null || obj.Id == null) + { + obj = new ScanStatusResult(); + if(Id == null) + { + Id = DateTime.UtcNow.ToString(DATE_TIME_FORMAT); + } + obj.Id = Id; + } + + //Update status of the scan + obj.Status = status; + File.WriteAllText(filePath, JsonConvert.SerializeObject(obj)); + + } + + private Boolean CheckModifications(String mainScanDirPath) + { + //Create path of manifest file + string manifestFilePath = Path.Combine(mainScanDirPath, Constants.ScanManifest); + + //Check if manifest file exists + //This means atleast 1 previous scan has been done + if (FileSystemHelpers.FileExists(manifestFilePath)) + { + using (FileStream file = System.IO.File.OpenRead(manifestFilePath)) + { + using (StreamReader sr = new StreamReader(file)) + { + JsonSerializer serializer = new JsonSerializer(); + + //Read the manifest file into JSON object + JObject obj = (JObject)serializer.Deserialize(sr, typeof(JObject)); + + //Check for modifications + return IsFolderModified(obj, Constants.ScanDir); + } + + } + } + + //This is the first scan + //Return true + return true; + + } + + private Boolean IsFolderModified(JObject fileObj,string directoryPath) + { + //Fetch all files in this directory + string[] filePaths = FileSystemHelpers.GetFiles(directoryPath,"*"); + + if(filePaths != null) + { + foreach (string filePath in filePaths) + { + //If manifest does not contain an entry for this file + //It means the file was newly added + //We need to scan as this is a modification + if (!fileObj.ContainsKey(filePath)) + { + return true; + } + //Modified time in manifest + String lastModTime = (string)fileObj[filePath]; + //Current modified time + String currModTime = FileSystemHelpers.GetDirectoryLastWriteTimeUtc(filePath).ToString(); + + //If they are different + //It means file has been modified after last scan + if (!currModTime.Equals(lastModTime)) + return true; + } + } + + + //Fetch all the child directories of this directory + string[] direcPaths = FileSystemHelpers.GetDirectories(directoryPath); + + if(direcPaths != null) + { + //Do recursive comparison of all files in the child directories + foreach (string direcPath in direcPaths) + { + if (IsFolderModified(fileObj, direcPath)) + return true; + } + } + + //No modifications found + return false; + } + + private void ModifyManifestFile(JObject fileObj, string directoryPath) + { + //Get all files in this directory + string[] filePaths = FileSystemHelpers.GetFiles(directoryPath, "*"); + + foreach (string filePath in filePaths) + { + //Get last modified timestamp of this file + String timeString = FileSystemHelpers.GetDirectoryLastWriteTimeUtc(filePath).ToString(); + //Add it as an entry into the manifest + fileObj.Add(filePath, timeString); + } + + //Get all child directories of this directory + string[] direcPaths = FileSystemHelpers.GetDirectories(directoryPath); + //Do a recursive call to add files of child directories to manifest + foreach (string direcPath in direcPaths) + { + ModifyManifestFile(fileObj, direcPath); + } + + } + + public async Task StartScan(String timeout,String mainScanDirPath,String id, String host, Boolean checkModified) + { + using (_tracer.Step("Start scan in the background")) + { + String folderPath = Path.Combine(mainScanDirPath, Constants.ScanFolderName + id); + String filePath = Path.Combine(folderPath, Constants.ScanStatusFile); + Boolean hasFileModifcations = true; + + if (_scanLock.IsHeld) + { + return ScanRequestResult.ScanAlreadyInProgress; + } + + //Create unique scan folder and scan status file + _scanLock.LockOperation(() => + { + //This means user wants to start a scan without checking for file changes after previous scan + //Delete the manifest file containing last updated timestamps of files + //This will force a scan to start irrespective of changes made to files + if (!checkModified) + { + String manifestPath = Path.Combine(mainScanDirPath, Constants.ScanManifest); + if (FileSystemHelpers.FileExists(manifestPath)) + { + FileSystemHelpers.DeleteFileSafe(manifestPath); + } + } + + //Check if files are modified + if (CheckModifications(mainScanDirPath)) + { + //Create unique scan directory for current scan + FileSystemHelpers.CreateDirectory(folderPath); + _tracer.Trace("Unique scan directory created for scan {0}", id); + + //Create scan status file inside folder + FileSystemHelpers.CreateFile(filePath).Close(); + + //Create temp file to check if scan is still running + string tempScanFilePath = GetTempScanFilePath(mainScanDirPath); + tempScanFilePath = Path.Combine(mainScanDirPath, Constants.TempScanFile); + FileSystemHelpers.CreateFile(tempScanFilePath).Close(); + + UpdateScanStatus(folderPath, ScanStatus.Starting,id); + } + else + { + hasFileModifcations = false; + } + + + }, "Creating unique scan folder", TimeSpan.Zero); + + if (!hasFileModifcations) + { + return ScanRequestResult.NoFileModifications; + } + + //Start Backgorund Scan + using (var timeoutCancellationTokenSource = new CancellationTokenSource()) + { + var successfullyScanned = PerformBackgroundScan(_tracer, _scanLock, folderPath, timeoutCancellationTokenSource.Token,id,mainScanDirPath); + + //Wait till scan task completes or the timeout goes off + if (await Task.WhenAny(successfullyScanned, Task.Delay(Int32.Parse(timeout), timeoutCancellationTokenSource.Token)) == successfullyScanned) + { + //If scan task completes before timeout + //Delete excess scan folders, just keep the maximum number allowed + await DeletePastScans(mainScanDirPath, _tracer); + + //Create new Manifest file containing the modified timestamps + String manifestPath = Path.Combine(mainScanDirPath, Constants.ScanManifest); + if (FileSystemHelpers.FileExists(manifestPath)) + { + FileSystemHelpers.DeleteFileSafe(manifestPath); + } + JObject manifestObj = new JObject(); + + //Write to the manifest with new timestamps of the modified file + ModifyManifestFile(manifestObj, Constants.ScanDir); + File.WriteAllText(manifestPath, JsonConvert.SerializeObject(manifestObj)); + + //Path to common log file for azure monitor + String aggrLogPath = Path.Combine(mainScanDirPath, Constants.AggregrateScanResults); + + //This checks if result scan log is formed + //If yes, it will append necessary logs to the aggregrate log file + //Current appended logs will be "Scanned files","Infected files", and details of infected files + String currLogPath = Path.Combine(folderPath, Constants.ScanLogFile); + if (FileSystemHelpers.FileExists(currLogPath)) + { + StreamReader file = new StreamReader(currLogPath); + string line; + while ((line = file.ReadLine()) != null) + { + if (line.Contains("FOUND") || line.Contains("Infected files") || line.Contains("Scanned files")) + { + //logType "Infected" means this log line represents details of infected files + String logType = "Infected"; + if (line.Contains("Infected files") || line.Contains("Scanned files")) + { + //logType "Info" means this log line represents total number of scanned or infected files + logType = "Info"; + } + FileSystemHelpers.AppendAllTextToFile(aggrLogPath, DateTime.UtcNow.ToString(@"M/d/yyyy hh:mm:ss tt") + "," + id + "," + logType + "," + host + "," + line + '\n'); + } + } + } + + return successfullyScanned.Result + ? ScanRequestResult.RunningAynschronously + : ScanRequestResult.AsyncScanFailed; + } + else + { + //Timeout went off before scan task completion + //Cancel scan task + timeoutCancellationTokenSource.Cancel(); + + //Scan process will be cancelled + //wait till scan status file is appropriately updated + await successfullyScanned; + + //Delete excess scan folders, just keep the maximum number allowed + await DeletePastScans(mainScanDirPath, _tracer); + + return ScanRequestResult.AsyncScanFailed; + + } + } + + + } + + } + + public static async Task DeletePastScans(String mainDirectory, ITracer _tracer) + { + //Run task to delete unwanted previous scans + await Task.Run(() => + { + //Main scan directory where all scans are stored + DirectoryInfo info = new DirectoryInfo(mainDirectory); + //Get sub directories and sort them by time of creation + DirectoryInfo[] subDirs = info.GetDirectories().OrderByDescending(p => p.CreationTime).ToArray(); + //Get max number of scans to store as history + int maxCnt = Int32.Parse(Constants.MaxScans); + + + if (subDirs.Length > maxCnt) + { + int diff = subDirs.Length - maxCnt; + for (int i = subDirs.Length - 1; diff > 0; diff--, i--) + { + //Delete oldest directories till we only have max number and no more than that + subDirs[i].Delete(true); + _tracer.Trace("Deleted scan record folder {0}",subDirs[i].FullName); + } + } + }); + + } + + public async Task GetScanStatus(String scanId,String mainScanDirPath) + { + ScanStatusResult obj = null; + await Task.Run(() => + { + obj = ReadScanStatusFile(scanId, mainScanDirPath, Constants.ScanStatusFile,null); + }); + + return obj; + } + + public async Task GetScanResultFile(String scanId,String mainScanDirPath) + { + //JObject statusRes = await GetScanStatus(scanId, mainScanDirPath); + ScanReport report = null; + //Run task to read the result file + await Task.Run(() => + { + String report_path = Path.Combine(mainScanDirPath, Constants.ScanFolderName + scanId, Constants.ScanLogFile); + ScanStatusResult scr = ReadScanStatusFile(scanId, mainScanDirPath, Constants.ScanStatusFile, null); + + //Proceed only if this scan has actually been conducted + //Handling possibility of user entering invalid scanId and breaking the application + if(scr != null) + { + report = new ScanReport + { + Id = scr.Id, + Timestamp = DateTime.ParseExact(scr.Id, DATE_TIME_FORMAT, System.Globalization.CultureInfo.InvariantCulture).ToUniversalTime(), + Report = GetScanParsedLogs(report_path) + }; + } + }); + + //All the contents of the file and the timestamp + return report; + } + + private ScanDetail GetScanParsedLogs(string currLogPath) + { + ScanDetail result = null; + if (FileSystemHelpers.FileExists(currLogPath)) + { + result = new ScanDetail(); + StreamReader file = new StreamReader(currLogPath); + string line; + while ((line = file.ReadLine()) != null) + { + if (line.Contains("FOUND")) + { + List infectedList = result.InfectedFiles; + if(infectedList == null) + { + infectedList = new List(); + result.InfectedFiles = infectedList; + } + int separatorIndex = line.IndexOf(":"); + int endIndex = line.IndexOf("FOUND"); + string name = line.Substring(0, separatorIndex); + string infection = line.Substring(separatorIndex + 1, endIndex - separatorIndex - 1).Trim(); + + InfectedFileObject obj = new InfectedFileObject(name, infection); + infectedList.Add(obj); + } + else if(line.Contains("Infected files")) + { + result.TotalInfected = line.Substring(line.IndexOf(":") + 1).Trim(); + } + else if(line.Contains("Scanned files")) + { + result.TotalScanned = line.Substring(line.IndexOf(":") + 1).Trim(); + } + else if (line.Contains(": OK")) + { + List safeList = result.SafeFiles; + if (safeList == null) + { + safeList = new List(); + result.SafeFiles = safeList; + } + safeList.Add(line.Substring(0, line.IndexOf(":"))); + } + else if (line.Contains("Time")) + { + result.TimeTaken = line.Substring(line.IndexOf(":") + 1, line.IndexOf("sec") - line.IndexOf(":") - 1).Trim(); + } + } + } + + return result; + } + + private static ScanStatusResult ReadScanStatusFile(String scanId, String mainScanDirPath, String fileName,String folderName) + { + ScanStatusResult obj = null; + String readPath = Path.Combine(mainScanDirPath, Constants.ScanFolderName + scanId, fileName); + + //Give preference to folderName if given + if(folderName != null) + { + readPath = Path.Combine(folderName, fileName); + } + + //Check if scan status file has been formed + if (FileSystemHelpers.FileExists(readPath)) + { + //Read json file and deserialize into JObject + using (FileStream file = System.IO.File.OpenRead(readPath)) + { + using (StreamReader sr = new StreamReader(file)) + { + JsonSerializer serializer = new JsonSerializer(); + obj = (ScanStatusResult)serializer.Deserialize(sr, typeof(ScanStatusResult)); + } + + } + } + + return obj; + } + + public void StopScan(String mainScanDirPath) + { + string tempScanFilePath = GetTempScanFilePath(mainScanDirPath); + if (tempScanFilePath != null && FileSystemHelpers.FileExists(tempScanFilePath)) + { + FileSystemHelpers.DeleteFileSafe(tempScanFilePath); + _tracer.Trace("Scan is being stopped. Deleted temp scan file at {0}",tempScanFilePath); + } + } + + private string GetTempScanFilePath(String mainScanDirPath) + { + return Path.Combine(mainScanDirPath, Constants.TempScanFile); + } + + public async Task PerformBackgroundScan(ITracer _tracer, AllSafeLinuxLock _scanLock, String folderPath,CancellationToken token, String scanId, String mainScanDirPath) + { + + var successfulScan = true; + + await Task.Run(() => + { + + _scanLock.LockOperation(() => + { + _scanLock.SetLockMsg(Resources.ScanUnderwayMsg); + + String statusFilePath = Path.Combine(folderPath, Constants.ScanStatusFile); + + + String logFilePath = Path.Combine(folderPath, Constants.ScanLogFile); + _tracer.Trace("Starting Scan {0}, ScanCommand: {1}, LogFile: {2}", scanId, Constants.ScanCommand, logFilePath); + + UpdateScanStatus(folderPath, ScanStatus.Executing, null); + + var escapedArgs = Constants.ScanCommand + " " + logFilePath; + Process _executingProcess = new Process() + { + StartInfo = new ProcessStartInfo + { + FileName = "/bin/bash", + Arguments = "-c \"" + escapedArgs + "\"", + RedirectStandardOutput = true, + UseShellExecute = false, + CreateNoWindow = true, + } + }; + _executingProcess.Start(); + + string tempScanFilePath = GetTempScanFilePath(mainScanDirPath); + //Check if process is completing before timeout + while (!_executingProcess.HasExited) + { + //Process still running, but timeout is done + //Or Process is still running but scan has been stopped by user + if (token.IsCancellationRequested || (tempScanFilePath != null && !FileSystemHelpers.FileExists(tempScanFilePath))) + { + //Kill process + _executingProcess.Kill(true, _tracer); + //Wait for process to be completely killed + _executingProcess.WaitForExit(); + successfulScan = false; + if (token.IsCancellationRequested) + { + _tracer.Trace("Scan {0} has timed out at {1}", scanId, DateTime.UtcNow.ToString("yyy-MM-dd_HH-mm-ssZ")); + + //Update status file + UpdateScanStatus(folderPath, ScanStatus.TimeoutFailure, null); + } + else + { + _tracer.Trace("Scan {0} has been force stopped at {1}", scanId, DateTime.UtcNow.ToString("yyy-MM-dd_HH-mm-ssZ")); + + //Update status file + UpdateScanStatus(folderPath, ScanStatus.ForceStopped, null); + } + + break; + } + } + + //Clean up the temp file + StopScan(mainScanDirPath); + + //Update status file with success + if (successfulScan) + { + //Check if process terminated with errors + if (_executingProcess.ExitCode != 0) + { + UpdateScanStatus(folderPath, ScanStatus.Failed, null); + _tracer.Trace("Scan {0} has terminated with exit code {1}. More info found in {2}", scanId, _executingProcess.ExitCode, logFilePath); + } + else + { + UpdateScanStatus(folderPath, ScanStatus.Success, null); + _tracer.Trace("Scan {0} is Successful", scanId); + } + } + + + }, "Performing continuous scan", TimeSpan.Zero); + + _scanLock.SetLockMsg(""); + + }); + + return successfulScan; + + } + + public IEnumerable GetResults(String mainScanDir) + { + IEnumerable results = EnumerateResults(mainScanDir).OrderByDescending(t => t.Status.Id).ToList(); + return results; + } + + private IEnumerable EnumerateResults(String mainScanDir) + { + if (FileSystemHelpers.DirectoryExists(mainScanDir)) + { + + foreach (String scanFolderPath in FileSystemHelpers.GetDirectories(mainScanDir)) + { + ScanOverviewResult result = new ScanOverviewResult(); + ScanStatusResult scanStatus = ReadScanStatusFile("", "", Constants.ScanStatusFile, scanFolderPath); + result.Status = scanStatus; + + yield return result; + } + } + } + + } +} diff --git a/Kudu.Core/Settings/DeploymentSettingsManager.cs b/Kudu.Core/Settings/DeploymentSettingsManager.cs index cdb1a125..9889051c 100644 --- a/Kudu.Core/Settings/DeploymentSettingsManager.cs +++ b/Kudu.Core/Settings/DeploymentSettingsManager.cs @@ -50,7 +50,7 @@ public static IDeploymentSettingsManager BuildPerDeploymentSettingsManager(strin return new DeploymentSettingsManager(perSiteSettings, combinedSettingsProviders.ToArray()); } - public IEnumerable> GetValues() + public IEnumerable> GetValues(IDictionary injectedSettings) { Dictionary values = new Dictionary(StringComparer.OrdinalIgnoreCase); @@ -63,6 +63,15 @@ public IEnumerable> GetValues() } } + if(injectedSettings != null) + { + foreach (var setting in injectedSettings) + { + values[setting.Key] = setting.Value; + } + } + + return values; } diff --git a/Kudu.Core/SourceControl/Git/GitExeRepository.cs b/Kudu.Core/SourceControl/Git/GitExeRepository.cs index c34ae9b5..62980f97 100644 --- a/Kudu.Core/SourceControl/Git/GitExeRepository.cs +++ b/Kudu.Core/SourceControl/Git/GitExeRepository.cs @@ -185,7 +185,7 @@ public void Initialize() sb.AppendLine("#!/bin/sh"); sb.AppendLine("read i"); sb.AppendLine("echo $i > pushinfo"); - sb.AppendLine(KnownEnvironment.KUDUCOMMAND); + sb.AppendLine(KnownEnvironment.KUDUCOMMAND.Replace("$KUDU_APPPATH",_environment.RepositoryPath.Replace("/repository",""))); if (OSDetector.IsOnWindows()) { FileSystemHelpers.WriteAllText(PostReceiveHookPath, sb.ToString()); diff --git a/Kudu.Core/SourceControl/Git/KnownEnvironment.cs b/Kudu.Core/SourceControl/Git/KnownEnvironment.cs index 26b2a9af..67895f43 100644 --- a/Kudu.Core/SourceControl/Git/KnownEnvironment.cs +++ b/Kudu.Core/SourceControl/Git/KnownEnvironment.cs @@ -14,7 +14,7 @@ internal static class KnownEnvironment // Command to launch the post receive hook // CORE NOTE modified the script to run "dotnet," assuming EXEPATH points // to a framework-dependent Core app. - public static string KUDUCOMMAND = "dotnet \"$" + EXEPATH + "\" " + + public static string KUDUCOMMAND = "benv dotnet=3-lts dotnet \"$" + EXEPATH + "\" " + "\"$" + APPPATH + "\" " + "\"$" + MSBUILD + "\" " + "\"$" + DEPLOYER + "\""; diff --git a/Kudu.Core/SourceControl/Git/LibGit2SharpRepository.cs b/Kudu.Core/SourceControl/Git/LibGit2SharpRepository.cs index a7ac7599..c2a6e97a 100644 --- a/Kudu.Core/SourceControl/Git/LibGit2SharpRepository.cs +++ b/Kudu.Core/SourceControl/Git/LibGit2SharpRepository.cs @@ -108,7 +108,7 @@ public void Initialize() var content = @"#!/bin/sh read i echo $i > pushinfo -" + KnownEnvironment.KUDUCOMMAND + "\r\n"; +" + KnownEnvironment.KUDUCOMMAND.Replace("$KUDU_APPPATH", RepositoryPath.Replace("site/repository", "")) + "\r\n"; File.WriteAllText(PostReceiveHookPath, content); } diff --git a/Kudu.Core/SourceControl/NullRepository.cs b/Kudu.Core/SourceControl/NullRepository.cs index e21a1fe8..d7600f9e 100644 --- a/Kudu.Core/SourceControl/NullRepository.cs +++ b/Kudu.Core/SourceControl/NullRepository.cs @@ -21,7 +21,7 @@ public class NullRepository : IRepository private readonly string _path; private readonly ITraceFactory _traceFactory; - public NullRepository(string path, ITraceFactory traceFactory) + public NullRepository(string path, ITraceFactory traceFactory, string commitId = null) { _path = path; _traceFactory = traceFactory; diff --git a/Kudu.Core/Tracing/Analytics.cs b/Kudu.Core/Tracing/Analytics.cs index cd136568..4136e0b2 100644 --- a/Kudu.Core/Tracing/Analytics.cs +++ b/Kudu.Core/Tracing/Analytics.cs @@ -23,7 +23,7 @@ public Analytics(IDeploymentSettingsManager settings, IServerConfiguration serve public void ProjectDeployed(string projectType, string result, string error, long deploymentDurationInMilliseconds, string siteMode, string vsProjectId = "") { - KuduEventSource.Log.ProjectDeployed( + KuduEventGenerator.Log().ProjectDeployed( _serverConfiguration.ApplicationName, NullToEmptyString(projectType), NullToEmptyString(result), @@ -36,7 +36,7 @@ public void ProjectDeployed(string projectType, string result, string error, lon public void JobStarted(string jobName, string scriptExtension, string jobType, string siteMode, string error, string trigger) { - KuduEventSource.Log.WebJobStarted( + KuduEventGenerator.Log().WebJobStarted( _serverConfiguration.ApplicationName, NullToEmptyString(jobName), NullToEmptyString(scriptExtension), @@ -48,7 +48,7 @@ public void JobStarted(string jobName, string scriptExtension, string jobType, s public void JobEvent(string jobName, string message, string jobType, string error) { - KuduEventSource.Log.WebJobEvent( + KuduEventGenerator.Log().WebJobEvent( _serverConfiguration.ApplicationName, NullToEmptyString(jobName), NullToEmptyString(message), @@ -66,7 +66,7 @@ public void UnexpectedException(Exception exception, bool trace = true, string m return; } - KuduEventSource.Log.KuduException( + KuduEventGenerator.Log().KuduException( _serverConfiguration.ApplicationName, string.Empty, string.Empty, @@ -86,7 +86,7 @@ public void UnexpectedException(Exception ex, string method, string path, string return; } - KuduEventSource.Log.KuduException( + KuduEventGenerator.Log().KuduException( _serverConfiguration.ApplicationName, NullToEmptyString(method), NullToEmptyString(path), @@ -106,7 +106,7 @@ public void DeprecatedApiUsed(string route, string userAgent, string method, str return; } - KuduEventSource.Log.DeprecatedApiUsed( + KuduEventGenerator.Log().DeprecatedApiUsed( _serverConfiguration.ApplicationName, NullToEmptyString(route), NullToEmptyString(userAgent), @@ -118,7 +118,7 @@ public void DeprecatedApiUsed(string route, string userAgent, string method, str public void SiteExtensionEvent(string method, string path, string result, string deploymentDurationInMilliseconds, string message) { - KuduEventSource.Log.KuduSiteExtensionEvent( + KuduEventGenerator.Log().KuduSiteExtensionEvent( _serverConfiguration.ApplicationName, NullToEmptyString(method), NullToEmptyString(path), diff --git a/Kudu.Core/Tracing/ConsoleEventGenerator.cs b/Kudu.Core/Tracing/ConsoleEventGenerator.cs new file mode 100644 index 00000000..52856b75 --- /dev/null +++ b/Kudu.Core/Tracing/ConsoleEventGenerator.cs @@ -0,0 +1,158 @@ +using System; +using System.Diagnostics.Tracing; +using System.IO; +using Kudu.Core.Settings; + +namespace Kudu.Core.Tracing +{ + class ConsoleEventGenerator : IKuduEventGenerator + { + private readonly Action _writeEvent; + private readonly bool _consoleEnabled = true; + + public ConsoleEventGenerator() + { + _writeEvent = ConsoleWriter; + } + + public void ProjectDeployed(string siteName, string projectType, string result, string error, long deploymentDurationInMilliseconds, string siteMode, string scmType, string vsProjectId) + { + KuduEvent kuduEvent = new KuduEvent + { + siteName = siteName, + projectType = projectType, + result = result, + error = error, + deploymentDurationInMilliseconds = deploymentDurationInMilliseconds, + siteMode = siteMode, + scmType = scmType, + vsProjectId = vsProjectId + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void WebJobStarted(string siteName, string jobName, string scriptExtension, string jobType, string siteMode, string error, string trigger) + { + KuduEvent kuduEvent = new KuduEvent + { + siteName = siteName, + jobName = jobName, + scriptExtension = scriptExtension, + jobType = jobType, + siteMode = siteMode, + error = error, + trigger = trigger + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void KuduException(string siteName, string method, string path, string result, string Message, string exception) + { + KuduEvent kuduEvent = new KuduEvent + { + level = (int)EventLevel.Warning, + siteName = siteName, + method = method, + path = path, + result = result, + Message = Message, + exception = exception + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void DeprecatedApiUsed(string siteName, string route, string userAgent, string method, string path) + { + KuduEvent kuduEvent = new KuduEvent + { + level = (int)EventLevel.Warning, + siteName = siteName, + route = route, + userAgent = userAgent, + method = method, + path = path + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void KuduSiteExtensionEvent(string siteName, string method, string path, string result, string deploymentDurationInMilliseconds, string Message) + { + long duration = 0; + long.TryParse(deploymentDurationInMilliseconds, out duration); + KuduEvent kuduEvent = new KuduEvent + { + siteName = siteName, + method = method, + path = path, + result = result, + deploymentDurationInMilliseconds = duration, + Message = Message + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void WebJobEvent(string siteName, string jobName, string Message, string jobType, string error) + { + KuduEvent kuduEvent = new KuduEvent + { + siteName = siteName, + jobName = jobName, + Message = Message, + jobType = jobType, + error = error + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void GenericEvent(string siteName, string Message, string requestId, string scmType, string siteMode, string buildVersion) + { + KuduEvent kuduEvent = new KuduEvent + { + siteName = siteName, + Message = Message, + requestId = requestId, + scmType = scmType, + siteMode = siteMode, + buildVersion = buildVersion + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void ApiEvent(string siteName, string Message, string address, string verb, string requestId, int statusCode, long latencyInMilliseconds, string userAgent) + { + KuduEvent kuduEvent = new KuduEvent + { + siteName = siteName, + Message = Message, + address = address, + verb = verb, + requestId = requestId, + statusCode = statusCode, + latencyInMilliseconds = latencyInMilliseconds, + userAgent = userAgent + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void LogKuduTraceEvent(KuduEvent kuduEvent) + { + _writeEvent($"{Constants.LinuxLogEventStreamName} {kuduEvent.ToString()},{Environment.StampName},{Environment.TenantId},{Environment.ContainerName}"); + } + + private void ConsoleWriter(string evt) + { + if (_consoleEnabled) + { + Console.WriteLine(evt); + } + } + } +} diff --git a/Kudu.Core/Tracing/DefaultKuduEventGenerator.cs b/Kudu.Core/Tracing/DefaultKuduEventGenerator.cs new file mode 100644 index 00000000..5136cb63 --- /dev/null +++ b/Kudu.Core/Tracing/DefaultKuduEventGenerator.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Kudu.Core.Tracing +{ + class DefaultKuduEventGenerator : IKuduEventGenerator + { + public void ProjectDeployed(string siteName, string projectType, string result, string error, long deploymentDurationInMilliseconds, string siteMode, string scmType, string vsProjectId) + { + KuduEventSource.Log.ProjectDeployed(siteName, projectType, result, error, deploymentDurationInMilliseconds, siteMode, scmType, vsProjectId); + } + + public void WebJobStarted(string siteName, string jobName, string scriptExtension, string jobType, string siteMode, string error, string trigger) + { + KuduEventSource.Log.WebJobStarted(siteName, jobName, scriptExtension, jobType, siteMode, error, trigger); + } + + public void KuduException(string siteName, string method, string path, string result, string Message, string exception) + { + KuduEventSource.Log.KuduException(siteName, method, path, result, Message, exception); + } + + public void DeprecatedApiUsed(string siteName, string route, string userAgent, string method, string path) + { + KuduEventSource.Log.DeprecatedApiUsed(siteName, route, userAgent, method, path); + } + + public void KuduSiteExtensionEvent(string siteName, string method, string path, string result, string deploymentDurationInMilliseconds, string Message) + { + KuduEventSource.Log.KuduSiteExtensionEvent(siteName, method, path, result, deploymentDurationInMilliseconds, Message); + } + + public void WebJobEvent(string siteName, string jobName, string Message, string jobType, string error) + { + KuduEventSource.Log.WebJobEvent(siteName, jobName, Message, jobType, error); + } + + public void GenericEvent(string siteName, string Message, string requestId, string scmType, string siteMode, string buildVersion) + { + KuduEventSource.Log.GenericEvent(siteName, Message, requestId, scmType, siteMode, buildVersion); + } + + public void ApiEvent(string siteName, string Message, string address, string verb, string requestId, int statusCode, long latencyInMilliseconds, string userAgent) + { + KuduEventSource.Log.ApiEvent(siteName, Message, address, verb, requestId, statusCode, latencyInMilliseconds, userAgent); + } + } +} diff --git a/Kudu.Core/Tracing/ETWTracer.cs b/Kudu.Core/Tracing/ETWTracer.cs index 81b579b7..604de28c 100644 --- a/Kudu.Core/Tracing/ETWTracer.cs +++ b/Kudu.Core/Tracing/ETWTracer.cs @@ -76,7 +76,7 @@ public void Trace(string message, IDictionary attributes) strb.AppendFormat("{0}=\"{1}\" ", attrib.Key, attrib.Value); } - KuduEventSource.Log.GenericEvent(ServerConfiguration.GetApplicationName(), + KuduEventGenerator.Log().GenericEvent(ServerConfiguration.GetApplicationName(), strb.ToString(), _requestId, string.Empty, diff --git a/Kudu.Core/Tracing/IKuduEventGenerator.cs b/Kudu.Core/Tracing/IKuduEventGenerator.cs new file mode 100644 index 00000000..371b12d0 --- /dev/null +++ b/Kudu.Core/Tracing/IKuduEventGenerator.cs @@ -0,0 +1,24 @@ +using System.Diagnostics.CodeAnalysis; +using System.Diagnostics.Tracing; + +namespace Kudu.Core.Tracing +{ + public interface IKuduEventGenerator + { + void ProjectDeployed(string siteName, string projectType, string result, string error, long deploymentDurationInMilliseconds, string siteMode, string scmType, string vsProjectId); + + void WebJobStarted(string siteName, string jobName, string scriptExtension, string jobType, string siteMode, string error, string trigger); + + void KuduException(string siteName, string method, string path, string result, string Message, string exception); + + void DeprecatedApiUsed(string siteName, string route, string userAgent, string method, string path); + + void KuduSiteExtensionEvent(string siteName, string method, string path, string result, string deploymentDurationInMilliseconds, string Message); + + void WebJobEvent(string siteName, string jobName, string Message, string jobType, string error); + + void GenericEvent(string siteName, string Message, string requestId, string scmType, string siteMode, string buildVersion); + + void ApiEvent(string siteName, string Message, string address, string verb, string requestId, int statusCode, long latencyInMilliseconds, string userAgent); + } +} diff --git a/Kudu.Core/Tracing/KuduEvent.cs b/Kudu.Core/Tracing/KuduEvent.cs new file mode 100644 index 00000000..69863425 --- /dev/null +++ b/Kudu.Core/Tracing/KuduEvent.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Tracing; +using System.Text; + +namespace Kudu.Core.Tracing +{ + class KuduEvent + { + public int level = (int)EventLevel.Informational; + public string siteName = string.Empty; + public string projectType = string.Empty; + public string result = string.Empty; + public string error = string.Empty; + public long deploymentDurationInMilliseconds = 0; + public string siteMode = string.Empty; + public string scmType = string.Empty; + public string vsProjectId = string.Empty; + public string jobName = string.Empty; + public string scriptExtension = string.Empty; + public string jobType = string.Empty; + public string trigger = string.Empty; + public string method = string.Empty; + public string path = string.Empty; + public string Message = string.Empty; + public string exception = string.Empty; + public string route = string.Empty; + public string userAgent = string.Empty; + public string requestId = string.Empty; + public string buildVersion = Constants.KuduBuild; + public string address = string.Empty; + public string verb = string.Empty; + public int statusCode = 0; + public long latencyInMilliseconds = 0; + + public override string ToString() + { + return $"{level},{siteName},{projectType},{result},{NormalizeString(error)},{deploymentDurationInMilliseconds},{siteMode},{scmType},{vsProjectId}," + + $"{jobName},{scriptExtension},{jobType},{trigger},{method},{path},{NormalizeString(Message)},{NormalizeString(exception)}," + + $"{route},{NormalizeString(userAgent)},{requestId},{buildVersion},{address},{verb},{statusCode},{latencyInMilliseconds}"; + } + + private string NormalizeString(string value) + { + // need to remove newlines for csv output + value = value.Replace(System.Environment.NewLine, " "); + value = value.Replace("\"", " "); + + // Wrap string literals in enclosing quotes + // For string columns that may contain quotes and/or + // our delimiter ',', before writing the value we + // enclose in quotes. This allows us to define matching + // groups based on quotes for these values. + return $"\"{value}\""; + } + } +} diff --git a/Kudu.Core/Tracing/KuduEventGenerator.cs b/Kudu.Core/Tracing/KuduEventGenerator.cs new file mode 100644 index 00000000..f141e563 --- /dev/null +++ b/Kudu.Core/Tracing/KuduEventGenerator.cs @@ -0,0 +1,38 @@ +using Kudu.Core.Helpers; + +namespace Kudu.Core.Tracing +{ + public class KuduEventGenerator + { + private static IKuduEventGenerator _eventGenerator = null; + + public static IKuduEventGenerator Log() + { + // Linux Consumptions only + bool isLinuxContainer = !string.IsNullOrEmpty(Environment.ContainerName); + if (isLinuxContainer) + { + if (_eventGenerator == null) + { + _eventGenerator = new LinuxContainerEventGenerator(); + } + } + else + { + if (_eventGenerator == null) + { + // Generate ETW events when running on windows + if (OSDetector.IsOnWindows()) + { + _eventGenerator = new DefaultKuduEventGenerator(); + } + else + { + _eventGenerator = new Log4NetEventGenerator(); + } + } + } + return _eventGenerator; + } + } +} diff --git a/Kudu.Core/Tracing/LinuxContainerEventGenerator.cs b/Kudu.Core/Tracing/LinuxContainerEventGenerator.cs new file mode 100644 index 00000000..235c8eef --- /dev/null +++ b/Kudu.Core/Tracing/LinuxContainerEventGenerator.cs @@ -0,0 +1,158 @@ +using System; +using System.Diagnostics.Tracing; +using System.IO; +using Kudu.Core.Settings; + +namespace Kudu.Core.Tracing +{ + class LinuxContainerEventGenerator : IKuduEventGenerator + { + private readonly Action _writeEvent; + private readonly bool _consoleEnabled = true; + + public LinuxContainerEventGenerator() + { + _writeEvent = ConsoleWriter; + } + + public void ProjectDeployed(string siteName, string projectType, string result, string error, long deploymentDurationInMilliseconds, string siteMode, string scmType, string vsProjectId) + { + KuduEvent kuduEvent = new KuduEvent + { + siteName = siteName, + projectType = projectType, + result = result, + error = error, + deploymentDurationInMilliseconds = deploymentDurationInMilliseconds, + siteMode = siteMode, + scmType = scmType, + vsProjectId = vsProjectId + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void WebJobStarted(string siteName, string jobName, string scriptExtension, string jobType, string siteMode, string error, string trigger) + { + KuduEvent kuduEvent = new KuduEvent + { + siteName = siteName, + jobName = jobName, + scriptExtension = scriptExtension, + jobType = jobType, + siteMode = siteMode, + error = error, + trigger = trigger + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void KuduException(string siteName, string method, string path, string result, string Message, string exception) + { + KuduEvent kuduEvent = new KuduEvent + { + level = (int)EventLevel.Warning, + siteName = siteName, + method = method, + path = path, + result = result, + Message = Message, + exception = exception + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void DeprecatedApiUsed(string siteName, string route, string userAgent, string method, string path) + { + KuduEvent kuduEvent = new KuduEvent + { + level = (int)EventLevel.Warning, + siteName = siteName, + route = route, + userAgent = userAgent, + method = method, + path = path + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void KuduSiteExtensionEvent(string siteName, string method, string path, string result, string deploymentDurationInMilliseconds, string Message) + { + long duration = 0; + long.TryParse(deploymentDurationInMilliseconds, out duration); + KuduEvent kuduEvent = new KuduEvent + { + siteName = siteName, + method = method, + path = path, + result = result, + deploymentDurationInMilliseconds = duration, + Message = Message + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void WebJobEvent(string siteName, string jobName, string Message, string jobType, string error) + { + KuduEvent kuduEvent = new KuduEvent + { + siteName = siteName, + jobName = jobName, + Message = Message, + jobType = jobType, + error = error + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void GenericEvent(string siteName, string Message, string requestId, string scmType, string siteMode, string buildVersion) + { + KuduEvent kuduEvent = new KuduEvent + { + siteName = siteName, + Message = Message, + requestId = requestId, + scmType = scmType, + siteMode = siteMode, + buildVersion = buildVersion + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void ApiEvent(string siteName, string Message, string address, string verb, string requestId, int statusCode, long latencyInMilliseconds, string userAgent) + { + KuduEvent kuduEvent = new KuduEvent + { + siteName = siteName, + Message = Message, + address = address, + verb = verb, + requestId = requestId, + statusCode = statusCode, + latencyInMilliseconds = latencyInMilliseconds, + userAgent = userAgent + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void LogKuduTraceEvent(KuduEvent kuduEvent) + { + _writeEvent($"{Constants.LinuxLogEventStreamName} {kuduEvent.ToString()},{Environment.StampName},{Environment.TenantId},{Environment.ContainerName}"); + } + + private void ConsoleWriter(string evt) + { + if (_consoleEnabled) + { + Console.WriteLine(evt); + } + } + } +} diff --git a/Kudu.Core/Tracing/Log4NetEventGenerator.cs b/Kudu.Core/Tracing/Log4NetEventGenerator.cs new file mode 100644 index 00000000..530a0c6e --- /dev/null +++ b/Kudu.Core/Tracing/Log4NetEventGenerator.cs @@ -0,0 +1,157 @@ +using System; +using System.Diagnostics.Tracing; +using System.IO; +using Kudu.Core.Settings; +using NuGet.Protocol.Core.Types; + +namespace Kudu.Core.Tracing +{ + class Log4NetEventGenerator : IKuduEventGenerator + { + private readonly Action _writeEvent; + + public static readonly log4net.ILog log = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); + public Log4NetEventGenerator() + { + _writeEvent = Log4NetWriter; + } + + public void ProjectDeployed(string siteName, string projectType, string result, string error, long deploymentDurationInMilliseconds, string siteMode, string scmType, string vsProjectId) + { + KuduEvent kuduEvent = new KuduEvent + { + siteName = siteName, + projectType = projectType, + result = result, + error = error, + deploymentDurationInMilliseconds = deploymentDurationInMilliseconds, + siteMode = siteMode, + scmType = scmType, + vsProjectId = vsProjectId + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void WebJobStarted(string siteName, string jobName, string scriptExtension, string jobType, string siteMode, string error, string trigger) + { + KuduEvent kuduEvent = new KuduEvent + { + siteName = siteName, + jobName = jobName, + scriptExtension = scriptExtension, + jobType = jobType, + siteMode = siteMode, + error = error, + trigger = trigger + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void KuduException(string siteName, string method, string path, string result, string Message, string exception) + { + KuduEvent kuduEvent = new KuduEvent + { + level = (int)EventLevel.Warning, + siteName = siteName, + method = method, + path = path, + result = result, + Message = Message, + exception = exception + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void DeprecatedApiUsed(string siteName, string route, string userAgent, string method, string path) + { + KuduEvent kuduEvent = new KuduEvent + { + level = (int)EventLevel.Warning, + siteName = siteName, + route = route, + userAgent = userAgent, + method = method, + path = path + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void KuduSiteExtensionEvent(string siteName, string method, string path, string result, string deploymentDurationInMilliseconds, string Message) + { + long duration = 0; + long.TryParse(deploymentDurationInMilliseconds, out duration); + KuduEvent kuduEvent = new KuduEvent + { + siteName = siteName, + method = method, + path = path, + result = result, + deploymentDurationInMilliseconds = duration, + Message = Message + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void WebJobEvent(string siteName, string jobName, string Message, string jobType, string error) + { + KuduEvent kuduEvent = new KuduEvent + { + siteName = siteName, + jobName = jobName, + Message = Message, + jobType = jobType, + error = error + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void GenericEvent(string siteName, string Message, string requestId, string scmType, string siteMode, string buildVersion) + { + KuduEvent kuduEvent = new KuduEvent + { + siteName = siteName, + Message = Message, + requestId = requestId, + scmType = scmType, + siteMode = siteMode, + buildVersion = buildVersion + }; + + LogKuduTraceEvent(kuduEvent); + } + + public void ApiEvent(string siteName, string Message, string address, string verb, string requestId, int statusCode, long latencyInMilliseconds, string userAgent) + { + KuduEvent kuduEvent = new KuduEvent + { + siteName = siteName, + Message = Message, + address = address, + verb = verb, + requestId = requestId, + statusCode = statusCode, + latencyInMilliseconds = latencyInMilliseconds, + userAgent = userAgent + }; + + LogKuduTraceEvent(kuduEvent); + } + + + public void LogKuduTraceEvent(KuduEvent kuduEvent) + { + _writeEvent($"{Constants.LinuxLogEventStreamName} {kuduEvent.ToString()},{Environment.StampName},{Environment.TenantId},{Environment.ContainerName}"); + } + + private void Log4NetWriter(string evt) + { + log.Debug(evt); + } + } +} diff --git a/Kudu.Core/Tracing/TextLogger.cs b/Kudu.Core/Tracing/TextLogger.cs index 347b88ba..fd9dfba0 100644 --- a/Kudu.Core/Tracing/TextLogger.cs +++ b/Kudu.Core/Tracing/TextLogger.cs @@ -43,10 +43,15 @@ public void WriteLine(string value, int depth) strb.Append(DateTime.UtcNow.ToString("s")); strb.Append(GetIndentation(depth + 1)); strb.Append(value); - - using (StreamWriter writer = new StreamWriter(FileSystemHelpers.OpenFile(_logFile, FileMode.Append, FileAccess.Write, FileShare.ReadWrite))) + try + { + using (StreamWriter writer = new StreamWriter(FileSystemHelpers.OpenFile(_logFile, FileMode.Append, FileAccess.Write, FileShare.ReadWrite))) + { + writer.WriteLine(strb.ToString()); + } + } catch(Exception) { - writer.WriteLine(strb.ToString()); + } } diff --git a/Kudu.Core/Tracing/XmlTracer.cs b/Kudu.Core/Tracing/XmlTracer.cs index 3b24df57..431e0c5e 100644 --- a/Kudu.Core/Tracing/XmlTracer.cs +++ b/Kudu.Core/Tracing/XmlTracer.cs @@ -32,7 +32,6 @@ public class XmlTracer : ITracer private static long _salt = 0; private static DateTime _lastCleanup = DateTime.MinValue; - public static readonly log4net.ILog log = log4net.LogManager.GetLogger(System.Reflection.MethodBase.GetCurrentMethod().DeclaringType); private bool startTagAdded = false; private readonly static TimeSpan ElapsedThreshold = TimeSpan.FromSeconds(10); @@ -99,10 +98,7 @@ private IDisposable WriteStartElement(string title, IDictionary // generate trace file name base on attribs _file = GenerateFileName(info); if (!startTagAdded) - { - log.Debug("@@@StartTrace@@@" + DateTime.UtcNow.ToString("yyyy-MM-dd HH:mm:ss.fff", - CultureInfo.InvariantCulture) + "," + ServerConfiguration.GetApplicationName() + - ","); + { startTagAdded = true; } } @@ -133,7 +129,6 @@ private IDisposable WriteStartElement(string title, IDictionary } FileSystemHelpers.AppendAllTextToFile(_file, strb.ToString()); - log.Debug(Regex.Replace(strb.ToString(), @"\t|\n|\r", "")); _infos.Push(info); _isStartElement = true; @@ -174,7 +169,6 @@ private void WriteEndTrace() strb.AppendLine(String.Format("", elapsed.TotalMilliseconds)); FileSystemHelpers.AppendAllTextToFile(_file, strb.ToString()); - log.Debug(Regex.Replace(strb.ToString(), @"\t|\n|\r", "")); _isStartElement = false; // adjust filename with statusCode @@ -222,7 +216,6 @@ private void WriteEndTrace() if (isOutgoingResponse&&zeroInfos) { startTagAdded = false; - log.Debug("@@@EndTrace@@@\n\n\n"); } } } diff --git a/Kudu.Services.Web/Infrastructure/RouteBuilderExtensions.cs b/Kudu.Services.Web/Infrastructure/RouteBuilderExtensions.cs index 71ad6cf0..b6b0e498 100644 --- a/Kudu.Services.Web/Infrastructure/RouteBuilderExtensions.cs +++ b/Kudu.Services.Web/Infrastructure/RouteBuilderExtensions.cs @@ -1,12 +1,11 @@ using System; -using Microsoft.AspNetCore.Routing; using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Routing; namespace Kudu.Services.Web.Infrastructure { - // CORE NOTE This was renamed/reworked from RouteCollectionExtensions - + /// /// /// @@ -16,34 +15,42 @@ public static class RouteBuilderExtensions { public const string DeprecatedKey = "deprecated"; - public static void MapHttpWebJobsRoute(this IRouteBuilder routes, string name, string jobType, string routeTemplate, object defaults, object constraints = null) + public static void MapHttpWebJobsRoute(this IRouteBuilder routes, string name, string jobType, + string routeTemplate, object defaults, object constraints = null) { // e.g. api/continuouswebjobs/foo - routes.MapHttpRoute(name, String.Format("api/{0}webjobs{1}", jobType, routeTemplate), defaults, constraints, deprecated: false); + routes.MapHttpRoute(name, String.Format("api/{0}webjobs{1}", jobType, routeTemplate), defaults, constraints, + deprecated: false); // e.g. api/triggeredjobs/foo/history/17 - routes.MapHttpRoute(name + "-dep", String.Format("api/{0}jobs{1}", jobType, routeTemplate), defaults, constraints, deprecated: true); + routes.MapHttpRoute(name + "-dep", String.Format("api/{0}jobs{1}", jobType, routeTemplate), defaults, + constraints, deprecated: true); // e.g. jobs/triggered/foo/history/17 and api/jobs/triggered/foo/history/17 - routes.MapHttpRouteDual(name + "-old", String.Format("jobs/{0}{1}", jobType, routeTemplate), defaults, constraints, bothDeprecated: true); + routes.MapHttpRouteDual(name + "-old", String.Format("jobs/{0}{1}", jobType, routeTemplate), defaults, + constraints, bothDeprecated: true); } - public static void MapHttpProcessesRoute(this IRouteBuilder routes, string name, string routeTemplate, object defaults, object constraints = null) + public static void MapHttpProcessesRoute(this IRouteBuilder routes, string name, string routeTemplate, + object defaults, object constraints = null) { // e.g. api/processes/3958/dump - routes.MapHttpRoute(name + "-direct", "api/processes" + routeTemplate, defaults, constraints, deprecated: false); + routes.MapHttpRoute(name + "-direct", "api/processes" + routeTemplate, defaults, constraints, + deprecated: false); // e.g. api/diagnostics/processes/4845 routes.MapHttpRouteDual(name, "diagnostics/processes" + routeTemplate, defaults, constraints); } - public static void MapHttpRouteDual(this IRouteBuilder routes, string name, string routeTemplate, object defaults, object constraints = null, bool bothDeprecated = false) + public static void MapHttpRouteDual(this IRouteBuilder routes, string name, string routeTemplate, + object defaults, object constraints = null, bool bothDeprecated = false) { routes.MapHttpRoute(name + "-dep", routeTemplate, defaults, constraints, deprecated: true); routes.MapHttpRoute(name, "api/" + routeTemplate, defaults, constraints, deprecated: bothDeprecated); } - public static void MapHttpRoute(this IRouteBuilder routes, string name, string routeTemplate, object defaults, object constraints, bool deprecated) + public static void MapHttpRoute(this IRouteBuilder routes, string name, string routeTemplate, object defaults, + object constraints, bool deprecated) { // CORE TODO Note that the only place that the deprecated datatoken is used, it only checks to see // if the key is *there*, not the bool value of it, so this is a little awkward looking due to the way @@ -51,7 +58,7 @@ public static void MapHttpRoute(this IRouteBuilder routes, string name, string r if (deprecated) { name += "-dep"; - routes.MapRoute(name, routeTemplate, defaults, constraints, new { DeprecatedKey = true }); + routes.MapRoute(name, routeTemplate, defaults, constraints, new {DeprecatedKey = true}); } else { @@ -79,4 +86,4 @@ public static void MapHandler(this IRouteBuilder routes, IKernel kerne } */ } -} +} \ No newline at end of file diff --git a/Kudu.Services.Web/InstanceMiddleware.cs b/Kudu.Services.Web/InstanceMiddleware.cs new file mode 100644 index 00000000..82bd6569 --- /dev/null +++ b/Kudu.Services.Web/InstanceMiddleware.cs @@ -0,0 +1,187 @@ +using Kudu.Core.K8SE; +using Microsoft.AspNetCore.Http; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace Kudu.Services.Web +{ + public class InstanceMiddleware + { + private readonly RequestDelegate _next; + private static readonly HttpClient _httpClient = new HttpClient(new HttpClientHandler() + { + AllowAutoRedirect = false, + MaxConnectionsPerServer = int.MaxValue, + UseCookies = false, + }); + + private const string CDN_HEADER_NAME = "Cache-Control"; + private static readonly string[] NotForwardedHttpHeaders = new[] { "Connection", "Host" }; + + static Regex rx = new Regex(@"^(\/instances\/)([A-Z0-9a-z\-]*)(\/)(.*)$", + RegexOptions.Compiled | RegexOptions.IgnoreCase); + public InstanceMiddleware(RequestDelegate next) + { + _next = next; + } + + public async Task Invoke(HttpContext context) + { + Console.WriteLine($"Instance Middleware"); + + if (context.Request.Path.Value.StartsWith("/instances/", StringComparison.OrdinalIgnoreCase) + && context.Request.Path.Value.IndexOf("/webssh") < 0) + { + Console.WriteLine($"Getting target URI"); + var targetUri = await RewriteInstanceUri(context); + Console.WriteLine($"Got target URI"); + + if (targetUri != null) + { + var requestMessage = GenerateProxifiedRequest(context, targetUri); + await SendAsync(context, requestMessage); + + return; + } + } + + await _next(context); + } + + private async Task SendAsync(HttpContext context, HttpRequestMessage requestMessage) + { + using (var responseMessage = await _httpClient.SendAsync(requestMessage, HttpCompletionOption.ResponseHeadersRead, context.RequestAborted)) + { + context.Response.StatusCode = (int)responseMessage.StatusCode; + + foreach (var header in responseMessage.Headers) + { + context.Response.Headers[header.Key] = header.Value.ToArray(); + } + + foreach (var header in responseMessage.Content.Headers) + { + context.Response.Headers[header.Key] = header.Value.ToArray(); + } + + context.Response.Headers.Remove("transfer-encoding"); + + if (!context.Response.Headers.ContainsKey(CDN_HEADER_NAME)) + { + context.Response.Headers.Add(CDN_HEADER_NAME, "no-cache, no-store"); + } + + await responseMessage.Content.CopyToAsync(context.Response.Body); + } + } + + private static HttpRequestMessage GenerateProxifiedRequest(HttpContext context, Uri targetUri) + { + var requestMessage = new HttpRequestMessage(); + CopyRequestContentAndHeaders(context, requestMessage); + + requestMessage.RequestUri = targetUri; + requestMessage.Headers.Host = targetUri.Host; + requestMessage.Method = GetMethod(context.Request.Method); + + + return requestMessage; + } + + private static void CopyRequestContentAndHeaders(HttpContext context, HttpRequestMessage requestMessage) + { + var requestMethod = context.Request.Method; + if (!HttpMethods.IsGet(requestMethod) && + !HttpMethods.IsHead(requestMethod) && + !HttpMethods.IsDelete(requestMethod) && + !HttpMethods.IsTrace(requestMethod)) + { + var streamContent = new StreamContent(context.Request.Body); + requestMessage.Content = streamContent; + } + + foreach (var header in context.Request.Headers) + { + if (!NotForwardedHttpHeaders.Contains(header.Key)) + { + if (header.Key != "User-Agent") + { + if (!requestMessage.Headers.TryAddWithoutValidation(header.Key, header.Value.ToArray()) && requestMessage.Content != null) + { + requestMessage.Content?.Headers.TryAddWithoutValidation(header.Key, header.Value.ToArray()); + } + } + else + { + string userAgent = header.Value.Count > 0 ? (header.Value[0] + " " + context.TraceIdentifier) : string.Empty; + + if (!requestMessage.Headers.TryAddWithoutValidation(header.Key, userAgent) && requestMessage.Content != null) + { + requestMessage.Content?.Headers.TryAddWithoutValidation(header.Key, userAgent); + } + } + + } + } + } + + private static HttpMethod GetMethod(string method) + { + if (HttpMethods.IsDelete(method)) return HttpMethod.Delete; + if (HttpMethods.IsGet(method)) return HttpMethod.Get; + if (HttpMethods.IsHead(method)) return HttpMethod.Head; + if (HttpMethods.IsOptions(method)) return HttpMethod.Options; + if (HttpMethods.IsPost(method)) return HttpMethod.Post; + if (HttpMethods.IsPut(method)) return HttpMethod.Put; + if (HttpMethods.IsTrace(method)) return HttpMethod.Trace; + return new HttpMethod(method); + } + + public Task RewriteInstanceUri(HttpContext context) + { + Match m = rx.Match(context.Request.Path); + if (m.Success && m.Groups.Count >= 4) + { + // Find matches. + + var instanceId = m.Groups[2].Value; + var remainingPath = m.Groups[4].Value; + Console.WriteLine(instanceId); + Console.WriteLine(remainingPath); + List instances = K8SEDeploymentHelper.GetInstances(K8SEDeploymentHelper.GetAppName(context)); + PodInstance instance = instances.Where(i => i.Name.Equals(instanceId, System.StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); ; + + // handle null, 0 instances + if(string.IsNullOrEmpty(remainingPath)) + { + return null; + } + + if(instances == null || instances.Count == 0) + { + throw new ArgumentOutOfRangeException($"Instance '{instanceId}' not found"); + } + + if (instances.Count > 0 && instanceId.Equals("any", System.StringComparison.OrdinalIgnoreCase)) + { + instance = instances[0]; + } + + var newUri = $"http://{instance.IpAddress}:1601/{remainingPath}{context.Request.QueryString}"; + Console.WriteLine($"URI: http://{instance.IpAddress}:1601/{remainingPath}{context.Request.QueryString}"); + var targetUri = new Uri(newUri); + return Task.FromResult(targetUri); + } + else + { + Console.WriteLine($"Different count {m.Groups.Count} , Success? {m.Success}"); + } + + return Task.FromResult((Uri)null); + } + } +} diff --git a/Kudu.Services.Web/KubeMiddleware.cs b/Kudu.Services.Web/KubeMiddleware.cs new file mode 100644 index 00000000..a6d65c39 --- /dev/null +++ b/Kudu.Services.Web/KubeMiddleware.cs @@ -0,0 +1,212 @@ +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Http; +using System; +using System.Collections.Generic; +using Kudu.Core.Infrastructure; +using Kudu.Contracts.Settings; +using Kudu.Services.Infrastructure; +using Kudu.Core; +using System.Text; +using Kudu.Core.Helpers; +using System.IO; +using System.Linq; +using Kudu.Core.K8SE; + +namespace Kudu.Services.Web +{ + /// + /// Middleware to modify Kudu Context when running on an K8 Cluster + /// + public class KubeMiddleware + { + private const string KuduConsoleFilename = "kudu.dll"; + private const string KuduConsoleRelativePath = "KuduConsole"; + private readonly RequestDelegate _next; + + /// + /// Filter out unnecessary routes for Linux Consumption + /// + /// The next request middleware to be passed in + public KubeMiddleware(RequestDelegate next) + { + _next = next; + } + + /// + /// Detect if a route matches any of whitelisted prefixes + /// + /// Http request context + /// Authorization service for each request + /// Response be set to 404 if the route is not whitelisted + public async Task Invoke(HttpContext context, IEnvironment environment, IServerConfiguration serverConfig) + { + string appName = K8SEDeploymentHelper.GetAppName(context); + string appNamenamespace = K8SEDeploymentHelper.GetAppNamespace(context); + string appType = K8SEDeploymentHelper.GetAppKind(context); + + string homeDir = ""; + string siteRepoDir = ""; + if (OSDetector.IsOnWindows()) + { + // K8SE TODO : Move to constants + homeDir = "C:\\repos\\apps\\"; + siteRepoDir = "\\site\\repository"; + } + else + { + // K8SE TODO : Move to constants + homeDir = "/home/apps/"; + siteRepoDir = "/site/repository"; + } + + // Cache the App Environment for this request + context.Items.TryAdd("environment", GetEnvironment(homeDir, appName, null, null, appNamenamespace, appType)); + + // Cache the appName for this request + context.Items.TryAdd("appName", appName); + + // Add All AppSettings to the context. + K8SEDeploymentHelper.UpdateContextWithAppSettings(context); + + PodInstance instance = null; + + if (context.Request.Path.Value.StartsWith("/instances/", StringComparison.OrdinalIgnoreCase) + && context.Request.Path.Value.IndexOf("/webssh") > 0) + { + List instances = K8SEDeploymentHelper.GetInstances(appName); + + int idx = context.Request.Path.Value.IndexOf("/webssh"); + string instanceId = context.Request.Path.Value.Substring(0, idx).Replace("/instances/", ""); + Console.WriteLine($"\n\n\n\n inst id {instanceId}"); + if (instances.Count > 0) + { + instance = instances.Where(i => i.Name.Equals(instanceId, System.StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); + } + + if (instances.Count > 0 && instanceId.Equals("any", System.StringComparison.OrdinalIgnoreCase)) + { + instance = instances[0]; + } + + if (instance == null) + { + context.Response.StatusCode = StatusCodes.Status404NotFound; + await context.Response.WriteAsync("Instance not found"); + return; + } + + int idx2 = context.Request.Path.Value.IndexOf("/webssh"); + context.Request.Path = context.Request.Path.Value.Substring(idx2); + if (!context.Request.Headers.ContainsKey("WEBSITE_SSH_USER")) + { + context.Request.Headers.Add("WEBSITE_SSH_USER", "root"); + } + if (!context.Request.Headers.ContainsKey("WEBSITE_SSH_PASSWORD")) + { + context.Request.Headers.Add("WEBSITE_SSH_PASSWORD", "Docker!"); + } + if (!context.Request.Headers.ContainsKey("WEBSITE_SSH_IP")) + { + context.Request.Headers.Add("WEBSITE_SSH_IP", instance.IpAddress); + } + } + + // Cache the appNamenamespace for this request if it's not empty or null + if (!string.IsNullOrEmpty(appNamenamespace)) + { + context.Items.TryAdd("appNamespace", appNamenamespace); + } + + string[] pathParts = context.Request.Path.ToString().Split("/"); + + if (pathParts != null && pathParts.Length >= 1 && IsGitRoute(context.Request.Path)) + { + appName = pathParts[1]; + appName = appName.Trim().Replace(".git", ""); + if (!FileSystemHelpers.DirectoryExists(homeDir + appName)) + { + context.Response.StatusCode = 404; + await context.Response.WriteAsync("The repository does not exist", Encoding.UTF8); + return; + } + } + + serverConfig.GitServerRoot = appName + ".git"; + // TODO: Use Path.Combine + environment.RepositoryPath = $"{homeDir}{appName}{siteRepoDir}"; + await _next.Invoke(context); + } + + private bool IsGitRoute(PathString routePath) + { + string[] pathParts = routePath.ToString().Split("/"); + if (pathParts != null && pathParts.Length >= 1) + { + return pathParts[1].EndsWith(".git"); + } + return false; + } + + /// + /// Returns a specified environment configuration as the current webapp's + /// default configuration during the runtime. + /// + private static IEnvironment GetEnvironment( + string home, + string appName, + IDeploymentSettingsManager settings = null, + HttpContext httpContext = null, + string appNamespace = null, + string appType = null) + { + var root = KubeMiddleware.ResolveRootPath(home, appName); + var siteRoot = Path.Combine(root, Constants.SiteFolder); + var repositoryPath = Path.Combine(siteRoot, + settings == null ? Constants.RepositoryPath : settings.GetRepositoryPath()); + var binPath = AppContext.BaseDirectory; + var requestId = httpContext != null ? httpContext.Request.GetRequestId() : null; + var kuduConsoleFullPath = + Path.Combine(AppContext.BaseDirectory, KuduConsoleRelativePath, KuduConsoleFilename); + return new Core.Environment(root, EnvironmentHelper.NormalizeBinPath(binPath), repositoryPath, requestId, + kuduConsoleFullPath, null, appName, appNamespace, appType); + } + + /// + /// Resolves the root path for the app being served by + /// Multitenant Kudu + /// + /// + /// + /// + public static string ResolveRootPath(string home, string appName) + { + // The HOME path should always be set correctly + //var path = System.Environment.ExpandEnvironmentVariables(@"%HOME%"); + var path = $"{home}{appName}"; + + FileSystemHelpers.EnsureDirectory(path); + FileSystemHelpers.EnsureDirectory($"{path}/site/artifacts/hostingstart"); + // For users running Windows Azure Pack 2 (WAP2), %HOME% actually points to the site folder, + // which we don't want here. So yank that segment if we detect it. + if (Path.GetFileName(path).Equals(Constants.SiteFolder, StringComparison.OrdinalIgnoreCase)) + { + path = Path.GetDirectoryName(path); + } + + return path; + + } + } + + /// + /// Extension wrapper for using Kube Middleware + /// + public static class KubeMiddlewareExtension + { + public static IApplicationBuilder UseKubeMiddleware(this IApplicationBuilder builder) + { + return builder.UseMiddleware(); + } + } +} diff --git a/Kudu.Services.Web/Kudu.Services.Web.csproj b/Kudu.Services.Web/Kudu.Services.Web.csproj index 2e86f400..62d982ca 100644 --- a/Kudu.Services.Web/Kudu.Services.Web.csproj +++ b/Kudu.Services.Web/Kudu.Services.Web.csproj @@ -1,6 +1,6 @@  - netcoreapp2.2 + netcoreapp3.1 true @@ -8,6 +8,9 @@ $(NoWarn);NU1608 $(NoWarn);NU1701 $(NoWarn);NU1604 + $(NoWarn);NU1591 + $(NoWarn);NU1702 + $(NoWarn);CS1591 true @@ -20,13 +23,16 @@ + + - - - + + + + @@ -37,14 +43,11 @@ - - - diff --git a/Kudu.Services.Web/KuduWebUtil.cs b/Kudu.Services.Web/KuduWebUtil.cs new file mode 100644 index 00000000..427a16df --- /dev/null +++ b/Kudu.Services.Web/KuduWebUtil.cs @@ -0,0 +1,556 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; +using Kudu.Contracts.Infrastructure; +using Kudu.Contracts.Settings; +using Kudu.Contracts.Tracing; +using Kudu.Core; +using Kudu.Core.Deployment; +using Kudu.Core.Helpers; +using Kudu.Core.Infrastructure; +using Kudu.Core.Settings; +using Kudu.Core.Tracing; +using Kudu.Services.Infrastructure; +using Kudu.Services.Infrastructure.Authorization; +using Kudu.Services.Infrastructure.Authentication; +using Kudu.Services.Web.Tracing; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.StaticFiles; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.FileProviders; +using Microsoft.AspNetCore.Authorization; +using Environment = Kudu.Core.Environment; +using Org.BouncyCastle.Asn1.Ocsp; + +namespace Kudu.Services.Web +{ + internal static class KuduWebUtil + { + private const string KuduConsoleFilename = "kudu.dll"; + private const string KuduConsoleRelativePath = "KuduConsole"; + + private static Dictionary _namedLocks; + + private static DeploymentLockFile _deploymentLock; + + // + // This method initializes status,ssh,hooks & deployment locks used by Kudu to ensure + // synchronized operations. This method creates a dictionary of locks which is injected + // into various controllers to resolve the locks they need. + // + // + // Locks used by Kudu: + // + // + // Status Lock + // Used by DeploymentStatusManager + // + // + // SSH Lock + // Used by SSHKeyController + // + // + // Hooks Lock + // Used by WebHooksManager + // + // + // Deployment Lock + // + // Used by DeploymentController, DeploymentManager, SettingsController, + // FetchDeploymentManager, LiveScmController, ReceivePackHandlerMiddleware + // + // + // + // + // + // Uses File watcher. + // This originally used Ninject's "WhenInjectedInto" in .Net project for specific instances. IServiceCollection + // doesn't support this concept, or anything similar like named instances. There are a few possibilities, + // but the hack solution for now is just injecting a dictionary of locks and letting each dependent resolve + // the one it needs. + // + private static void SetupLocks(ITraceFactory traceFactory, IEnvironment environment) + { + var lockPath = Path.Combine(environment.SiteRootPath, Constants.LockPath); + var deploymentLockPath = Path.Combine(lockPath, Constants.DeploymentLockFile); + var statusLockPath = Path.Combine(lockPath, Constants.StatusLockFile); + var sshKeyLockPath = Path.Combine(lockPath, Constants.SSHKeyLockFile); + var hooksLockPath = Path.Combine(lockPath, Constants.HooksLockFile); + _deploymentLock = DeploymentLockFile.GetInstance(deploymentLockPath, traceFactory); + _deploymentLock.InitializeAsyncLocks(); + + var statusLock = new LockFile(statusLockPath, traceFactory); + statusLock.InitializeAsyncLocks(); + var sshKeyLock = new LockFile(sshKeyLockPath, traceFactory); + sshKeyLock.InitializeAsyncLocks(); + var hooksLock = new LockFile(hooksLockPath, traceFactory); + hooksLock.InitializeAsyncLocks(); + + _namedLocks = new Dictionary + { + {"status", statusLock}, + {"ssh", sshKeyLock}, + {"hooks", hooksLock}, + {"deployment", _deploymentLock} + }; + } + + + internal static void SetupFileServer(IApplicationBuilder app, string fileDirectoryPath, string requestPath) + { + + // Create deployment logs directory if it doesn't exist + FileSystemHelpers.CreateDirectory(fileDirectoryPath); + + // Set up custom content types - associating file extension to MIME type + var provider = new FileExtensionContentTypeProvider + { + Mappings = + { + [".py"] = "text/html", + [".env"] = "text/html", + [".cshtml"] = "text/html", + [".log"] = "text/html", + [".image"] = "image/png" + } + }; + + app.UseFileServer(new FileServerOptions + { + FileProvider = new PhysicalFileProvider( + fileDirectoryPath), + RequestPath = requestPath, + EnableDirectoryBrowsing = true, + StaticFileOptions = + { + ServeUnknownFileTypes = true, + DefaultContentType = "text/plain", + ContentTypeProvider = provider + } + }); + } + + /// + /// Returns absolute URL for a request including host, path and the query string + /// + /// + /// Object encapsulating all HTTP-specific information about an individual HTTP request. + /// + /// + internal static Uri GetAbsoluteUri(HttpContext httpContext) + { + var request = httpContext.Request; + var uriBuilder = new UriBuilder + { + Scheme = request.Scheme, + Host = request.Host.Host, + Path = request.Path.ToString(), + Query = request.QueryString.ToString() + }; + return uriBuilder.Uri; + } + + /// + /// Returns the tracer objects for request tracing + /// + internal static ITracer GetTracer(IServiceProvider serviceProvider) + { + var environment = serviceProvider.GetRequiredService(); + var level = serviceProvider.GetRequiredService().GetTraceLevel(); + var contextAccessor = serviceProvider.GetRequiredService(); + var httpContext = contextAccessor.HttpContext; + var requestTraceFile = TraceServices.GetRequestTraceFile(httpContext); + if (level <= TraceLevel.Off || requestTraceFile == null) return NullTracer.Instance; + var textPath = Path.Combine(environment.TracePath, requestTraceFile); + return new CascadeTracer(new XmlTracer(environment.TracePath, level), + new TextTracer(textPath, level)); + } + + /// + /// Returns the name of the trace file. Each request is associated with a Tracing GUID, returns this guid. + /// + internal static string GetRequestTraceFile(IServiceProvider serviceProvider) + { + var traceLevel = serviceProvider.GetRequiredService().GetTraceLevel(); + if (traceLevel <= TraceLevel.Off) return null; + var contextAccessor = serviceProvider.GetRequiredService(); + var httpContext = contextAccessor.HttpContext; + return TraceServices.GetRequestTraceFile(httpContext); + } + + + /// + /// Ensures smooth transition between mono based Kudu and KuduLite. + /// + /// + /// + /// POST Receive GitHook File:This file was previously hard coded with mono path to launch kudu console. + /// + /// + /// We will would use the OryxBuild in future for deployments, as a safety measure we clear + /// the deployment script. + /// + /// + /// + /// + /// + internal static void MigrateToNetCorePatch(IEnvironment environment) + { + // Get the repository path: + // Use value in the settings.xml file if it is present. + string repositoryPath = environment.RepositoryPath; + IDeploymentSettingsManager settings = GetDeploymentSettingsManager(environment); + if (settings != null) + { + var settingsRepoPath = DeploymentSettingsExtension.GetRepositoryPath(settings); + repositoryPath = Path.Combine(environment.SiteRootPath, settingsRepoPath); + } + + var gitPostReceiveHookFile = Path.Combine(repositoryPath, ".git", "hooks", "post-receive"); + if (FileSystemHelpers.FileExists(gitPostReceiveHookFile)) + { + var fileText = FileSystemHelpers.ReadAllText(gitPostReceiveHookFile); + var isRunningOnAzure = System.Environment.GetEnvironmentVariable("WEBSITE_INSTANCE_ID") != null; + if (fileText.Contains("/usr/bin/mono")) + { + if(isRunningOnAzure) + { + FileSystemHelpers.WriteAllText(gitPostReceiveHookFile, fileText.Replace("/usr/bin/mono", "benv dotnet=3-lts dotnet")); + } + } + else if(!fileText.Contains("benv") && fileText.Contains("dotnet") && isRunningOnAzure) + { + FileSystemHelpers.WriteAllText(gitPostReceiveHookFile, fileText.Replace("dotnet", "benv dotnet=3-lts dotnet")); + } + + } + + if (FileSystemHelpers.DirectoryExists(Path.Combine(environment.RootPath, ".mono")) + && FileSystemHelpers.FileExists(Path.Combine(environment.DeploymentToolsPath, "deploy.sh"))) + { + FileSystemHelpers.DeleteFileSafe(Path.Combine(environment.DeploymentToolsPath, "deploy.sh")); + } + } + + internal static void TraceShutdown(IEnvironment environment, IDeploymentSettingsManager settings) + { + ITracer tracer = GetTracerWithoutContext(environment, settings); + var attribs = new Dictionary(); + + // Add an attribute containing the process, AppDomain and Thread ids to help debugging + attribs.Add("pid", String.Format("{0},{1},{2}", + Process.GetCurrentProcess().Id, + AppDomain.CurrentDomain.Id.ToString(), + Thread.CurrentThread.ManagedThreadId)); + + attribs.Add("uptime", TraceMiddleware.UpTime.ToString()); + + attribs.Add("lastrequesttime", TraceMiddleware.LastRequestTime.ToString()); + + tracer.Trace(XmlTracer.ProcessShutdownTrace, attribs); + + OperationManager.SafeExecute(() => + { + KuduEventSource.Log.GenericEvent( + ServerConfiguration.GetApplicationName(), + string.Format("Shutdown pid:{0}, domain:{1}", Process.GetCurrentProcess().Id, + AppDomain.CurrentDomain.Id), + string.Empty, + string.Empty, + string.Empty, + string.Empty); + }); + } + + internal static ITracer GetTracerWithoutContext(IEnvironment environment, IDeploymentSettingsManager settings) + { + // when file system has issue, this can throw (environment.TracePath calls EnsureDirectory). + // prefer no-op tracer over outage. + return OperationManager.SafeExecute(() => + { + var traceLevel = settings.GetTraceLevel(); + return traceLevel > TraceLevel.Off + ? new XmlTracer(environment.TracePath, traceLevel) + : NullTracer.Instance; + }) ?? NullTracer.Instance; + } + + /// + /// Returns the ILogger object to log deployments + /// + internal static ILogger GetDeploymentLogger(IServiceProvider serviceProvider) + { + var environment = serviceProvider.GetRequiredService(); + var level = serviceProvider.GetRequiredService().GetTraceLevel(); + var contextAccessor = serviceProvider.GetRequiredService(); + var httpContext = contextAccessor.HttpContext; + var requestTraceFile = TraceServices.GetRequestTraceFile(httpContext); + if (level <= TraceLevel.Off || requestTraceFile == null) return NullLogger.Instance; + var textPath = Path.Combine(environment.DeploymentTracePath, requestTraceFile); + return new TextLogger(textPath); + } + + /// + /// Returns a specified environment configuration as the current webapp's + /// default configuration during the runtime. + /// + internal static IEnvironment GetEnvironment(IWebHostEnvironment hostingEnvironment, + IDeploymentSettingsManager settings = null, + HttpContext httpContext = null) + { + var root = PathResolver.ResolveRootPath(); + var siteRoot = Path.Combine(root, Constants.SiteFolder); + var repositoryPath = Path.Combine(siteRoot, + settings == null ? Constants.RepositoryPath : settings.GetRepositoryPath()); + var binPath = AppContext.BaseDirectory; + var requestId = httpContext != null ? httpContext.Request.GetRequestId() : null; + var kuduConsoleFullPath = + Path.Combine(AppContext.BaseDirectory, KuduConsoleRelativePath, KuduConsoleFilename); + return new Environment(root, EnvironmentHelper.NormalizeBinPath(binPath), repositoryPath, requestId, + kuduConsoleFullPath, null); + } + + /// + /// Ensures %HOME% is correctly set + /// + internal static void EnsureHomeEnvironmentVariable() + { + // PlatformServices.Default and the injected IHostingEnvironment have at runtime. + if (Directory.Exists(System.Environment.ExpandEnvironmentVariables(@"%HOME%"))) return; + + //For Debug + System.Environment.SetEnvironmentVariable("HOME", OSDetector.IsOnWindows() ? @"F:\kudu-debug" : "/home"); + + /* + // If MapPath("/_app") returns a valid folder, set %HOME% to that, regardless of + // it current value. This is the non-Azure code path. + string path = HostingEnvironment.MapPath(Constants.MappedSite); + if (Directory.Exists(path)) + { + path = Path.GetFullPath(path); + System.Environment.SetEnvironmentVariable("HOME", path); + } + */ + } + + /// + /// Ensures valid /home/site/deployments/settings.xml is loaded, deletes + /// corrupt settings.xml file + /// + /// + /// IEnvironment object that maintains paths used by kudu + /// + internal static void EnsureValidDeploymentXmlSettings(IEnvironment environment) + { + var path = GetSettingsPath(environment); + if (!FileSystemHelpers.FileExists(path)) return; + try + { + var settings = new DeploymentSettingsManager(new XmlSettings.Settings(path)); + settings.GetValue(SettingsKeys.TraceLevel); + } + catch (Exception ex) + { + DateTime lastWriteTimeUtc = DateTime.MinValue; + OperationManager.SafeExecute(() => lastWriteTimeUtc = File.GetLastWriteTimeUtc(path)); + // trace initialization error + KuduEventSource.Log.KuduException( + ServerConfiguration.GetApplicationName(), + "Startup.cs", + string.Empty, + string.Empty, + string.Format("Invalid '{0}' is detected and deleted. Last updated time was {1}.", path, + lastWriteTimeUtc), + ex.ToString()); + File.Delete(path); + } + } + + /// + /// Get Deploployment settings + /// + /// + private static IDeploymentSettingsManager GetDeploymentSettingsManager(IEnvironment environment) + { + var path = GetSettingsPath(environment); + if (!FileSystemHelpers.FileExists(path)) + { + return null; + } + + IDeploymentSettingsManager result = null; + + try + { + var settings = new DeploymentSettingsManager(new XmlSettings.Settings(path)); + settings.GetValue(SettingsKeys.TraceLevel); + + result = settings; + } + catch (Exception ex) + { + DateTime lastWriteTimeUtc = DateTime.MinValue; + OperationManager.SafeExecute(() => lastWriteTimeUtc = File.GetLastWriteTimeUtc(path)); + // trace initialization error + KuduEventSource.Log.KuduException( + ServerConfiguration.GetApplicationName(), + "Startup.cs", + string.Empty, + string.Empty, + string.Format("Invalid '{0}' is detected and deleted. Last updated time was {1}.", path, + lastWriteTimeUtc), + ex.ToString()); + File.Delete(path); + } + + return result; + } + + internal static void PrependFoldersToPath(IEnvironment environment) + { + var folders = PathUtilityFactory.Instance.GetPathFolders(environment); + + var path = System.Environment.GetEnvironmentVariable("PATH"); + // Ignore any folder that doesn't actually exist + var additionalPaths = + string.Join(Path.PathSeparator.ToString(), folders.Where(Directory.Exists)); + + // Make sure we haven't already added them. This can happen if the Kudu appdomain restart (since it's still same process) + if (path != null && path.Contains(additionalPaths)) return; + path = additionalPaths + Path.PathSeparator + path; + + // PHP 7 was mistakenly added to the path unconditionally on Azure. To work around, if we detect + // some PHP v5.x anywhere on the path, we yank the unwanted PHP 7 + // TODO: remove once the issue is fixed on Azure + if (path.Contains(@"PHP\v5")) + { + path = path.Replace(@"D:\Program Files (x86)\PHP\v7.0" + Path.PathSeparator, String.Empty); + } + + System.Environment.SetEnvironmentVariable("PATH", path); + } + + /// + /// Sets the environment variables for Net Core CLI. + /// + /// + /// This method previously included environment variables to optimize net core runtime to run in a container. + /// But they have been moved to the Kudu Dockerfile + /// > + /// + /// IEnvironment object that maintains references to al the paths used by Kudu during runtime. + /// + internal static void EnsureDotNetCoreEnvironmentVariable(IEnvironment environment) + { + if (Environment.IsAzureEnvironment()) + { + // On Azure, restore nuget packages to d:\home\.nuget so they're persistent. It also helps + // work around https://github.com/projectkudu/kudu/issues/2056. + // Note that this only applies to project.json scenarios (not packages.config) + SetEnvironmentVariableIfNotYetSet("NUGET_PACKAGES", Path.Combine(environment.RootPath, ".nuget")); + + // Set the telemetry environment variable + SetEnvironmentVariableIfNotYetSet("DOTNET_CLI_TELEMETRY_PROFILE", "AzureKudu"); + } + else + { + // Set it slightly differently if outside of Azure to differentiate + SetEnvironmentVariableIfNotYetSet("DOTNET_CLI_TELEMETRY_PROFILE", "Kudu"); + } + } + + /// + /// Returns the singleton Deployment Lock Object + /// + /// + /// + /// + internal static DeploymentLockFile GetDeploymentLock(ITraceFactory traceFactory, IEnvironment environment) + { + if (_namedLocks == null || _deploymentLock == null) + { + GetNamedLocks(traceFactory, environment); + } + + return _deploymentLock; + } + + /// + /// Returns a dictionary containing references to all the singleton lock objects. Initialises these locks + /// if they have not been initialised + /// + /// + /// + /// + internal static Dictionary GetNamedLocks(ITraceFactory traceFactory, + IEnvironment environment) + { + if (_namedLocks == null) + { + SetupLocks(traceFactory, environment); + } + + return _namedLocks; + } + + // + // Adds an environment variable for site bitness. Can be "AMD64" or "x86"; + // + internal static void EnsureSiteBitnessEnvironmentVariable() + { + SetEnvironmentVariableIfNotYetSet("SITE_BITNESS", + System.Environment.Is64BitProcess ? Constants.X64Bit : Constants.X86Bit); + } + + private static void SetEnvironmentVariableIfNotYetSet(string name, string value) + { + if (System.Environment.GetEnvironmentVariable(name) == null) + { + System.Environment.SetEnvironmentVariable(name, value); + } + } + + /// + /// Returns the path to the settings.xml file in the deployments directory + /// + internal static string GetSettingsPath(IEnvironment environment) + { + return Path.Combine(environment.DeploymentsPath, Constants.DeploySettingsPath); + } + + internal static IServiceCollection AddLinuxConsumptionAuthentication(this IServiceCollection services) + { + services.AddAuthentication() + .AddArmToken(); + + return services; + } + + /// + /// In Linux consumption, we are running the KuduLite instance in a Service Fabric Mesh container. + /// We want to introduce AdminAuthLevel policy to restrict instance admin endpoint access. + /// + /// Dependency injection to application service + /// Service + internal static IServiceCollection AddLinuxConsumptionAuthorization(this IServiceCollection services, IEnvironment environment) + { + services.AddAuthorization(o => + { + o.AddInstanceAdminPolicies(environment); + }); + + services.AddSingleton(); + return services; + } + + internal static string GetWebSSHProxyPort() + { + return System.Environment.GetEnvironmentVariable(Constants.WebSiteSwapSlotName) ?? Constants.WebSSHReverseProxyDefaultPort; + } + } +} diff --git a/Kudu.Services.Web/Pages/DebugConsole/DebugConsoleController.cs b/Kudu.Services.Web/Pages/DebugConsole/DebugConsoleController.cs deleted file mode 100644 index b4e85b5b..00000000 --- a/Kudu.Services.Web/Pages/DebugConsole/DebugConsoleController.cs +++ /dev/null @@ -1,27 +0,0 @@ -using Microsoft.AspNetCore.Mvc; -using Kudu.Core.Helpers; - -namespace Kudu.Services.Web.Pages.DebugConsole -{ - // CORE NOTE This is a new shim to get the right console to load; couldn't do it the way it was originally done - // due to the differences in the way the new razor pages work - public class DebugConsoleController : Controller - { - public ActionResult Index() - { - var os = OSDetector.IsOnWindows() ? "Windows" : "Linux"; - return View($"~/Pages/DebugConsole/{os}Console.cshtml"); - } - - public ActionResult LinuxConsole() - { - return View($"~/Pages/DebugConsole/LinuxConsole.cshtml"); - } - - public ActionResult WindowsConsole() - { - return View($"~/Pages/DebugConsole/WindowsConsole.cshtml"); - } - - } -} \ No newline at end of file diff --git a/Kudu.Services.Web/Pages/DebugConsole/LinuxConsole.cshtml b/Kudu.Services.Web/Pages/DebugConsole/LinuxConsole.cshtml deleted file mode 100644 index 80a340e0..00000000 --- a/Kudu.Services.Web/Pages/DebugConsole/LinuxConsole.cshtml +++ /dev/null @@ -1,23 +0,0 @@ -@{ - ViewData["Title"] = "Diagnostic Console"; - Layout = "~/Pages/_Layout.cshtml"; -} - - - - - - - -
-
-
-
-
- - - - - \ No newline at end of file diff --git a/Kudu.Services.Web/Pages/DebugConsole/WindowsConsole.cshtml b/Kudu.Services.Web/Pages/DebugConsole/WindowsConsole.cshtml deleted file mode 100644 index 098e1c74..00000000 --- a/Kudu.Services.Web/Pages/DebugConsole/WindowsConsole.cshtml +++ /dev/null @@ -1,192 +0,0 @@ -@{ - ViewData["Title"] = "Diagnostic Console"; - Layout = "~/Pages/_Layout.cshtml"; -} - - - - - - - - - -
- -
-
-
-
- - ... / - - / - - - - - | - 0 items - - | - - - - - - - - - - -
-
-
-
Drag here to upload and unzip
-
- - - - - - - - - - - - - - - - - -
NameModifiedSize
-
- - - - - - - - - - - - -
- 		- -   - - - - - - - - -
-
-
-
-

- - -

-
- Use old console -
- - -
- -
- - - - - - - - - diff --git a/Kudu.Services.Web/Pages/Detectors/index.cshtml b/Kudu.Services.Web/Pages/Detectors/index.cshtml new file mode 100644 index 00000000..f569cc3f --- /dev/null +++ b/Kudu.Services.Web/Pages/Detectors/index.cshtml @@ -0,0 +1,73 @@ +@page +@model Kudu.Services.Web.Pages.Detectors.IndexModel +@{ + var ownerName = Environment.GetEnvironmentVariable("WEBSITE_OWNER_NAME") ?? ""; + var subscriptionId = ownerName; + var resourceGroup = Environment.GetEnvironmentVariable("WEBSITE_RESOURCE_GROUP") ?? ""; + var siteName = Environment.GetEnvironmentVariable("WEBSITE_SITE_NAME") ?? ""; + var hostName = Environment.GetEnvironmentVariable("HTTP_HOST") ?? ""; + var slotName = ""; + var isFunctionApp = Environment.GetEnvironmentVariable("FUNCTIONS_WORKER_RUNTIME") != null; + + var index = ownerName.IndexOf('+'); + if (index >= 0) + { + subscriptionId = ownerName.Substring(0, index); + } + + string detectorPath; + string troubleshootExt = ""; + if (isFunctionApp) + { + troubleshootExt = "appsvc.troubleshoot%3Dtrue%7C"; // "appsvc.troubleshoot=true|" + detectorPath = "diagnostics%2Ffunctionappdownanderrors"; // "diagnostics/functionappdownanderrors" + } + else if (Kudu.Core.Helpers.OSDetector.IsOnWindows()) + { + detectorPath = "diagnostics%2Favailability%2Fanalysis"; // "diagnostics/availability/analysis" + } + else + { + detectorPath = "detectors%2FLinuxAppDown"; // "detectors/LinuxAppDown" + } + + var hostNameIndex = hostName.IndexOf('.'); + if (hostNameIndex >= 0) + { + hostName = hostName.Substring(0, hostNameIndex); + } + + var runtimeSuffxIndex = siteName.IndexOf("__"); + if (runtimeSuffxIndex >= 0) + { + siteName = siteName.Substring(0, runtimeSuffxIndex); + } + + // Get the slot name + if (!hostName.Equals(siteName, StringComparison.CurrentCultureIgnoreCase)) + { + var slotNameIndex = siteName.Length; + if (hostName.Length > slotNameIndex && hostName[slotNameIndex] == '-') + { + // Fix up hostName by removing "-SLOTNAME" + slotName = hostName.Substring(slotNameIndex + 1); + hostName = hostName.Substring(0, slotNameIndex); + } + } + + var isSlot = !String.IsNullOrWhiteSpace(slotName) && !slotName.Equals("production", StringComparison.CurrentCultureIgnoreCase); + + var detectorDeepLink = "https://portal.azure.com/?websitesextension_ext=" + + troubleshootExt + + "asd.featurePath%3D" + + detectorPath + + "#resource/subscriptions/" + subscriptionId + + "/resourceGroups/" + resourceGroup + + "/providers/Microsoft.Web/sites/" + + hostName + + (isSlot ? "/slots/" + slotName : "") + + "/troubleshoot"; + + Response.Redirect(detectorDeepLink); +} + diff --git a/Kudu.Services.Web/Pages/Detectors/index.cshtml.cs b/Kudu.Services.Web/Pages/Detectors/index.cshtml.cs new file mode 100644 index 00000000..42bf45c2 --- /dev/null +++ b/Kudu.Services.Web/Pages/Detectors/index.cshtml.cs @@ -0,0 +1,11 @@ +using Microsoft.AspNetCore.Mvc.RazorPages; + +namespace Kudu.Services.Web.Pages.Detectors +{ + public class IndexModel : PageModel + { + public void OnGet() + { + } + } +} diff --git a/Kudu.Services.Web/Pages/Env.cshtml b/Kudu.Services.Web/Pages/Env.cshtml index 7e23bb7e..e81fe20c 100644 --- a/Kudu.Services.Web/Pages/Env.cshtml +++ b/Kudu.Services.Web/Pages/Env.cshtml @@ -63,7 +63,7 @@ } - @foreach (KeyValuePair kv in _settingsManager.GetValues()) + @foreach (KeyValuePair kv in _settingsManager.GetValues((IDictionary)HttpContext.Items["appSettings"])) { if (kv.Value != null) diff --git a/Kudu.Services.Web/Pages/Index.cshtml b/Kudu.Services.Web/Pages/Index.cshtml index d7b1a1b7..74a0ef52 100644 --- a/Kudu.Services.Web/Pages/Index.cshtml +++ b/Kudu.Services.Web/Pages/Index.cshtml @@ -30,7 +30,6 @@ Assembly assembly; try { - //return "test"; assembly = Assembly.Load("Microsoft.Web.Hosting, Version=7.1.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35"); var fileVersionInfo = System.Diagnostics.FileVersionInfo.GetVersionInfo(assembly.Location); return fileVersionInfo.ProductVersion; @@ -48,10 +47,10 @@
@{ - //string commitFile = Server.MapPath("~/commit.txt"); string commitFile = System.IO.Path.Combine(hostingEnvironment.WebRootPath, "commit.txt"); string sha = System.IO.File.Exists(commitFile) ? System.IO.File.ReadAllText(commitFile).Trim() : null; - var version = typeof(Kudu.Services.Web.Tracing.TraceMiddleware).Assembly.GetName().Version; + //var version = typeof(Kudu.Services.Web.Tracing.TraceMiddleware).Assembly.GetName().Version; + var version = Constants.KuduBuild; }

Environment

@@ -69,24 +68,12 @@ }
- @if (appServiceVersion != null) - { -
-
- Azure App Service -
-
- @GetAppServiceVersion() -
-
- }
Site up time
@Kudu.Services.Web.Tracing.TraceMiddleware.UpTime.ToString(@"dd\.hh\:mm\:ss") - 12345
@@ -119,9 +106,6 @@
  • Files
    • -
  • - Current Docker logs (Download as zip) -

    • More information about Kudu can be found on the wiki.

    diff --git a/Kudu.Services.Web/Pages/NewUI/DebugConsole2/DebugConsole2Controller.cs b/Kudu.Services.Web/Pages/NewUI/DebugConsole2/DebugConsole2Controller.cs deleted file mode 100644 index d801917b..00000000 --- a/Kudu.Services.Web/Pages/NewUI/DebugConsole2/DebugConsole2Controller.cs +++ /dev/null @@ -1,27 +0,0 @@ -using Microsoft.AspNetCore.Mvc; -using Kudu.Core.Helpers; - -namespace Kudu.Services.Web.Pages.NewUI.DebugConsole2 -{ - // CORE NOTE This is a new shim to get the right console to load; couldn't do it the way it was originally done - // due to the differences in the way the new razor pages work - public class DebugConsole2Controller : Controller - { - public ActionResult Index() - { - var os = OSDetector.IsOnWindows() ? "Windows" : "Linux"; - return View($"~/Pages/NewUI/DebugConsole2/{os}Console2.cshtml"); - } - - public ActionResult LinuxConsole() - { - return View($"~/Pages/NewUI/DebugConsole/LinuxConsole2.cshtml"); - } - - public ActionResult WindowsConsole() - { - return View($"~/Pages/NewUI/DebugConsole2/WindowsConsole2.cshtml"); - } - - } -} \ No newline at end of file diff --git a/Kudu.Services.Web/Pages/NewUI/DebugConsole2/LinuxConsole2.cshtml b/Kudu.Services.Web/Pages/NewUI/DebugConsole2/LinuxConsole2.cshtml deleted file mode 100644 index 9bf943b7..00000000 --- a/Kudu.Services.Web/Pages/NewUI/DebugConsole2/LinuxConsole2.cshtml +++ /dev/null @@ -1,23 +0,0 @@ -@{ - ViewData["Title"] = "Diagnostic Console"; - Layout = "~/Pages/NewUI/_Layout.cshtml"; -} - - - - - - - -
    -
    -
    -
    -
    - - - - - \ No newline at end of file diff --git a/Kudu.Services.Web/Pages/NewUI/DebugConsole2/WindowsConsole2.cshtml b/Kudu.Services.Web/Pages/NewUI/DebugConsole2/WindowsConsole2.cshtml deleted file mode 100644 index d4707d3c..00000000 --- a/Kudu.Services.Web/Pages/NewUI/DebugConsole2/WindowsConsole2.cshtml +++ /dev/null @@ -1,192 +0,0 @@ -@{ - ViewData["Title"] = "Diagnostic Console"; - Layout = "~/Pages/NewUI/_Layout.cshtml"; -} - - - - - - - - - -
    - -
    -
    -
    -
    - - ... / - - / - - - - - | - 0 items - - | - - - - - - - - - - -
    -
    -
    -
    Drag here to upload and unzip
    -
    - - - - - - - - - - - - - - - - - -
    NameModifiedSize
    -
    - - - - - - - - - - - - -
    -     		- -   - - - - - - - - -
    -
    -
    -
    -

    - - -

    -
    - Use old console -
    - - -
    - -
    - - - - - - - - - diff --git a/Kudu.Services.Web/Pages/NewUI/Env.cshtml b/Kudu.Services.Web/Pages/NewUI/Env.cshtml index 7e23bb7e..35287327 100644 --- a/Kudu.Services.Web/Pages/NewUI/Env.cshtml +++ b/Kudu.Services.Web/Pages/NewUI/Env.cshtml @@ -62,8 +62,8 @@ @name = @System.Configuration.ConfigurationManager.AppSettings[name] } - - @foreach (KeyValuePair kv in _settingsManager.GetValues()) + + @foreach (KeyValuePair kv in _settingsManager.GetValues((IDictionary)HttpContext.Items["appSettings"])) { if (kv.Value != null) diff --git a/Kudu.Services.Web/Pages/NewUI/Index.cshtml b/Kudu.Services.Web/Pages/NewUI/Index.cshtml index e8cc224e..f347ff79 100644 --- a/Kudu.Services.Web/Pages/NewUI/Index.cshtml +++ b/Kudu.Services.Web/Pages/NewUI/Index.cshtml @@ -1,9 +1,12 @@ @page @using Kudu.Core.Deployment + +@using Kudu.Core @using Microsoft.AspNetCore.Hosting @inject IHostingEnvironment hostingEnvironment @inject IDeploymentManager deploymentManager +@inject Microsoft.AspNetCore.Http.IHttpContextAccessor accessor @{ ViewBag.Title = "title"; @@ -11,7 +14,9 @@ } @{ - //string commitFile = Server.MapPath("~/commit.txt"); - var commitFile = System.IO.Path.Combine(hostingEnvironment.WebRootPath, "commit.txt"); - var sha = System.IO.File.Exists(commitFile) ? System.IO.File.ReadAllText(commitFile).Trim() : null; - var workerIdHash = System.Environment.GetEnvironmentVariable("WEBSITE_INSTANCE_ID"); + var workerIdHash = System.Environment.GetEnvironmentVariable(Constants.AzureWebsiteInstanceId); var workerIdShortHash = workerIdHash?.Substring(0, 12); const string version = Constants.KuduBuild; var showLearningBanner = !deploymentManager.GetResults().Any(); @@ -48,19 +50,31 @@
    -

    KuduLite

    @version
    - Site UpTime: @Tracing.TraceMiddleware.UpTime.ToString(@"dd\.hh\:mm\:ss") | Site Folder: @PathResolver.ResolveRootPath() | Temp Folder: @System.IO.Path.GetTempPath() -
    Kudu Attached to Instance:   @workerIdShortHash | Switch Instance +

    KuduLite

    @version +
    + Site UpTime: @Tracing.TraceMiddleware.UpTime.ToString(@"dd\.hh\:mm\:ss") | Site Folder: @(((IEnvironment)accessor.HttpContext.Items["environment"]).SiteRootPath) | Temp Folder: @System.IO.Path.GetTempPath() +
    + Kudu Attached to Instance:   @workerIdShortHash | + +
    + Instance 1 + Instance 2 + Instance 3 +
    +     +