Program.cs

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using Microsoft.Data.DataView;
using Microsoft.ML;
using Microsoft.ML.Calibrators;
using Microsoft.ML.Data;
using Microsoft.ML.LightGBM;
using static Microsoft.ML.LightGBM.Options;
using static Microsoft.ML.TrainCatalogBase;
using NumSharp.Core;

namespace KaggleMicrosoftMalwarePrediction
{
    class Program
    {
        static readonly string trainDataFile = @"C:\KaggleMalwarePrediction\Dataset\train.csv";
        static readonly string testDataFile = @"C:\KaggleMalwarePrediction\Dataset\test.csv";
        static readonly string submissionFile = @"C:\KaggleMalwarePrediction\Submissions\submission1.csv";
        static TextLoader reader;
        const int number_of_rows = 100000;
        const int seed = 1234;
        const int number_of_folds = 5;

        static void Main(string[] args)
        {
            // Set MLContext
            MLContext mlContext = new MLContext(seed: seed);
            
            // Method 1: Data Loading
            /* 
            reader = mlContext.Data.CreateTextLoader(new TextLoader.Options()
                            {
                                Separators = new[] { ',' },
                                HasHeader = true,
                                Columns = new[]
                                {
                                    // Specify all different Columns
                                    // https://github.com/dotnet/machinelearning/blob/master/src/Microsoft.ML.Core/Data/DataKind.cs
                                    new TextLoader.Column("MachineIdentifier", DataKind.String, 0),
                                    new TextLoader.Column("ProductName", DataKind.String, 1),
                                    new TextLoader.Column("EngineVersion", DataKind.String, 2),
                                    new TextLoader.Column("AppVersion", DataKind.String, 3),
                                    new TextLoader.Column("AvSigVersion", DataKind.String, 4),
                                    new TextLoader.Column("IsBeta", DataKind.Boolean, 5),
                                    new TextLoader.Column("RtpStateBitfield", DataKind.Single, 6),
                                    new TextLoader.Column("IsSxsPassiveMode", DataKind.Boolean, 7),                                    
                                    new TextLoader.Column("DefaultBrowsersIdentifier", DataKind.String, 8),
                                    new TextLoader.Column("AVProductStatesIdentifier", DataKind.Single, 9),
                                    new TextLoader.Column("AVProductsInstalled", DataKind.Single, 10),
                                    new TextLoader.Column("AVProductsEnabled", DataKind.Single, 11),
                                    new TextLoader.Column("HasTpm", DataKind.Boolean, 12),
                                    new TextLoader.Column("CountryIdentifier", DataKind.Single, 13),
                                    new TextLoader.Column("CityIdentifier", DataKind.Single, 14),
                                    new TextLoader.Column("OrganizationIdentifier", DataKind.Single, 15),
                                    new TextLoader.Column("GeoNameIdentifier", DataKind.Single, 16),
                                    new TextLoader.Column("LocaleEnglishNameIdentifier", DataKind.Single, 17),                                  
                                    new TextLoader.Column("Platform", DataKind.String, 18), 
                                    new TextLoader.Column("Processor", DataKind.String, 19),
                                    new TextLoader.Column("OsVer", DataKind.String, 20),
                                    new TextLoader.Column("OsBuild", DataKind.Single, 21),
                                    new TextLoader.Column("OsSuite", DataKind.Single, 22), 
                                    new TextLoader.Column("OsPlatformSubRelease", DataKind.String, 23),
                                    new TextLoader.Column("OsBuildLab", DataKind.String, 24),
                                    new TextLoader.Column("SkuEdition", DataKind.String, 25),
                                    new TextLoader.Column("IsProtected", DataKind.Boolean, 26),
                                    new TextLoader.Column("AutoSampleOptIn", DataKind.Boolean, 27),
                                    new TextLoader.Column("PuaMode", DataKind.Boolean, 28),
                                    new TextLoader.Column("SMode", DataKind.Boolean, 29),
                                    new TextLoader.Column("IeVerIdentifier", DataKind.Single, 30),
                                    new TextLoader.Column("SmartScreen", DataKind.String, 31),
                                    new TextLoader.Column("Firewall", DataKind.Boolean, 32),
                                    new TextLoader.Column("UacLuaenable", DataKind.Boolean, 33),
                                    new TextLoader.Column("Census_MDC2FormFactor", DataKind.String, 34),
                                    new TextLoader.Column("Census_DeviceFamily", DataKind.String, 35),
                                    new TextLoader.Column("Census_OEMNameIdentifier", DataKind.Single, 36),
                                    new TextLoader.Column("Census_OEMModelIdentifier", DataKind.Single, 37),
                                    new TextLoader.Column("Census_ProcessorCoreCount", DataKind.Single, 38),
                                    new TextLoader.Column("Census_ProcessorManufacturerIdentifier", DataKind.UInt16, 39),
                                    new TextLoader.Column("Census_ProcessorModelIdentifier", DataKind.Single, 40),
                                    new TextLoader.Column("Census_ProcessorClass", DataKind.String, 41),
                                    new TextLoader.Column("Census_PrimaryDiskTotalCapacity", DataKind.Single, 42),
                                    new TextLoader.Column("Census_PrimaryDiskTypeName", DataKind.String, 43),
                                    new TextLoader.Column("Census_SystemVolumeTotalCapacity", DataKind.Single, 44),
                                    new TextLoader.Column("Census_HasOpticalDiskDrive", DataKind.Boolean, 45),
                                    new TextLoader.Column("Census_TotalPhysicalRAM", DataKind.Single, 46),
                                    new TextLoader.Column("Census_ChassisTypeName", DataKind.String, 47),
                                    new TextLoader.Column("Census_InternalPrimaryDiagonalDisplaySizeInInches", DataKind.Single, 48),
                                    new TextLoader.Column("Census_InternalPrimaryDisplayResolutionHorizontal", DataKind.Single, 49),
                                    new TextLoader.Column("Census_InternalPrimaryDisplayResolutionVertical", DataKind.Single, 50),   
                                    new TextLoader.Column("Census_PowerPlatformRoleName", DataKind.String, 51),
                                    new TextLoader.Column("Census_InternalBatteryType", DataKind.String, 52),
                                    new TextLoader.Column("Census_InternalBatteryNumberOfCharges", DataKind.Single, 53),
                                    new TextLoader.Column("Census_OSVersion", DataKind.String, 54),
                                    new TextLoader.Column("Census_OSArchitecture", DataKind.String, 55),
                                    new TextLoader.Column("Census_OSBranch", DataKind.String, 56),
                                    new TextLoader.Column("Census_OSBuildNumber", DataKind.Single, 57),     
                                    new TextLoader.Column("Census_OSBuildRevision", DataKind.Single, 58),                              
                                    new TextLoader.Column("Census_OSEdition", DataKind.String, 59),
                                    new TextLoader.Column("Census_OSSkuName", DataKind.String, 60),
                                    new TextLoader.Column("Census_OSInstallTypeName", DataKind.String, 61),
                                    new TextLoader.Column("Census_OSInstallLanguageIdentifier", DataKind.Single, 62), 
                                    new TextLoader.Column("Census_OSUILocaleIdentifier", DataKind.String, 63),
                                    new TextLoader.Column("Census_OSWUAutoUpdateOptionsName", DataKind.String, 64),
                                    new TextLoader.Column("Census_IsPortableOperatingSystem", DataKind.Boolean, 65),
                                    new TextLoader.Column("Census_GenuineStateName", DataKind.String, 66),
                                    new TextLoader.Column("Census_ActivationChannel", DataKind.String, 67),
                                    new TextLoader.Column("Census_IsFlightingInternal", DataKind.Boolean, 68),
                                    new TextLoader.Column("Census_IsFlightsDisabled", DataKind.Boolean, 69),
                                    new TextLoader.Column("Census_FlightRing", DataKind.String, 70),
                                    new TextLoader.Column("Census_ThresholdOptIn", DataKind.Boolean, 71),
                                    new TextLoader.Column("Census_FirmwareManufacturerIdentifier", DataKind.Single, 72),
                                    new TextLoader.Column("Census_FirmwareVersionIdentifier", DataKind.Single, 73),
                                    new TextLoader.Column("Census_IsSecureBootEnabled", DataKind.Boolean, 74),
                                    new TextLoader.Column("Census_IsWIMBootEnabled", DataKind.Boolean, 75),
                                    new TextLoader.Column("Census_IsVirtualDevice", DataKind.Boolean, 76),
                                    new TextLoader.Column("Census_IsTouchEnabled", DataKind.Boolean, 77),
                                    new TextLoader.Column("Census_IsPenCapable", DataKind.Boolean, 78),                                    
                                    new TextLoader.Column("Census_IsAlwaysOnAlwaysConnectedCapable", DataKind.Boolean, 79),
                                    new TextLoader.Column("Wdft_IsGamer", DataKind.Boolean, 80),                                    
                                    new TextLoader.Column("Wdft_RegionIdentifier", DataKind.Single, 81),
                                    new TextLoader.Column("HasDetections", DataKind.Boolean, 82)                                           
                                }, 
                                // If you specify MaxRows than that amount applies to train and test data
                                // Remove this option to use the full train and test data.
                                MaxRows = number_of_rows 
                            });
            IDataView train = reader.Load(trainDataFile);
            */  

            // Method 2: Data Loading with type
            reader = mlContext.Data.CreateTextLoader<DataRow>(hasHeader: true, separatorChar: ',');
            IDataView train = reader.Load(trainDataFile);
           
            // Preview of Train Data
            var trainPreview = train.Preview(maxRows: 5);
            foreach (var column in trainPreview.ColumnView)
            {
                Console.WriteLine($"{column.Column.Name} (Type: {column.Column.Type}):");
                foreach(var value in column.Values)
                {
                    Console.WriteLine($"\t{value}");
                }
            }
 
            #region TrainTestSplit
            
            /* 
            // Train model with 80/20 Train Test Split
            var dataSplit = mlContext.BinaryClassification.TrainTestSplit(train, testFraction: 0.2);
            var model = Train(mlContext, dataSplit.TrainSet);

            // Evaluate the Model
            Evaluate(mlContext, model, dataSplit.TestSet);            
            
            // Create Submission on Test data
            CreateSubmission(mlContext, model); 
            */

            #endregion

            #region CrossValidation

            // Train model with CrossValidation
            var modelCV = TrainCV(mlContext, train);
            
            // Evaluate the Model
            EvaluateCV(modelCV);
 
            // Create Submission on Test Data
            CreateSubmissionCV(modelCV, mlContext); 
            
            #endregion
            
            // Finished
            Console.WriteLine("=== Done...");
        }

        private static EstimatorChain<BinaryPredictionTransformer<CalibratedModelParametersBase<LightGbmBinaryModelParameters, PlattCalibrator>>> CreatePipeline(MLContext mlContext)
        {
            var pipeline = mlContext.Transforms.Categorical.OneHotHashEncoding("AppVersion")
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("ProductName"))
                            .Append(mlContext.Transforms.Categorical.OneHotHashEncoding("EngineVersion"))
                            .Append(mlContext.Transforms.Categorical.OneHotHashEncoding("AvSigVersion"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("SmartScreen"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("SMode"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("Wdft_IsGamer"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("OsVer"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("OsBuildLab"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("Census_OSVersion"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("Census_OSBranch"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("Census_OSEdition"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("Census_OSInstallTypeName"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("Census_OSSkuName"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("Census_OSUILocaleIdentifier"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("Census_PowerPlatformRoleName"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("OsPlatformSubRelease"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("Census_MDC2FormFactor"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("Census_ChassisTypeName"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("Processor"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("Census_OSArchitecture"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("Census_OSWUAutoUpdateOptionsName"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("Census_IsAlwaysOnAlwaysConnectedCapable"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("IsProtected"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("Census_ActivationChannel"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("Census_IsTouchEnabled"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("SkuEdition"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("Census_PrimaryDiskTypeName"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("Census_InternalBatteryType"))
                            .Append(mlContext.Transforms.Categorical.OneHotEncoding("DefaultBrowsersIdentifier"))
                            .Append(mlContext.Transforms.Concatenate("CategoricalFeatures", "ProductName", 
                                                                                            "AppVersion", 
                                                                                            "EngineVersion", 
                                                                                            "AvSigVersion", 
                                                                                            "SmartScreen",
                                                                                            "SMode",
                                                                                            "Wdft_IsGamer",
                                                                                            "OsVer",
                                                                                            "OsBuildLab",
                                                                                            "Census_OSVersion",
                                                                                            "Census_OSBranch",
                                                                                            "Census_OSEdition",
                                                                                            "Census_OSInstallTypeName",
                                                                                            "Census_OSSkuName",
                                                                                            "Census_OSUILocaleIdentifier",
                                                                                            "Census_PowerPlatformRoleName",
                                                                                            "OsPlatformSubRelease",
                                                                                            "Census_MDC2FormFactor",
                                                                                            "Census_ChassisTypeName",
                                                                                            "Processor",
                                                                                            "Census_OSArchitecture",
                                                                                            "Census_OSWUAutoUpdateOptionsName",
                                                                                            "Census_IsAlwaysOnAlwaysConnectedCapable",
                                                                                            "IsProtected",
                                                                                            "Census_ActivationChannel",
                                                                                            "Census_IsTouchEnabled",
                                                                                            "SkuEdition",
                                                                                            "Census_PrimaryDiskTypeName",
                                                                                            "Census_InternalBatteryType",
                                                                                            "DefaultBrowsersIdentifier" ))
                            .Append(mlContext.Transforms.FeatureSelection.SelectFeaturesBasedOnCount("CategoricalFeatures", count: 2500))
                            .Append(mlContext.Transforms.Concatenate("Features", "CategoricalFeatures", 
                                                                                 "Census_SystemVolumeTotalCapacity", 
                                                                                 "Census_PrimaryDiskTotalCapacity",
                                                                                 "Wdft_RegionIdentifier",
                                                                                 "GeoNameIdentifier",
                                                                                 "OrganizationIdentifier",
                                                                                 "CityIdentifier",
                                                                                 "CountryIdentifier",
                                                                                 "LocaleEnglishNameIdentifier",
                                                                                 "AVProductStatesIdentifier",
                                                                                 "AVProductsInstalled",
                                                                                 "AVProductsEnabled",
                                                                                 "OsBuild",
                                                                                 "OsSuite",
                                                                                 "Census_InternalPrimaryDiagonalDisplaySizeInInches", 
                                                                                 "Census_InternalPrimaryDisplayResolutionHorizontal", 
                                                                                 "Census_InternalPrimaryDisplayResolutionVertical",
                                                                                 "Census_ProcessorModelIdentifier",
                                                                                 "Census_TotalPhysicalRAM",
                                                                                 "Census_OSBuildRevision",
                                                                                 "Census_OEMNameIdentifier",
                                                                                 "Census_ProcessorCoreCount",
                                                                                 "Census_OEMModelIdentifier",
                                                                                 "IeVerIdentifier",
                                                                                 "Census_FirmwareManufacturerIdentifier",
                                                                                 "Census_FirmwareVersionIdentifier",
                                                                                 "Census_OSBuildNumber",
                                                                                 "Census_OSInstallLanguageIdentifier",
                                                                                 "Census_InternalBatteryNumberOfCharges",
                                                                                 "RtpStateBitfield" ))
                            .AppendCacheCheckpoint(mlContext)
                            .Append(mlContext.BinaryClassification.Trainers.LightGbm(new Options
                            {
                                NumLeaves = 512,
                                NThread = null,     // Specify Integer to limit to certain amount of threads
                                MinDataPerLeaf = 256, 
                                NumBoostRound = 2500,
                                LearningRate = 0.05,
                                LabelColumn =  "HasDetections",
                                VerboseEval = true,
                                UseCat = false,
                                Booster = new TreeBooster.Options()
                                        { 
                                            MaxDepth = 15,                                             
                                            FeatureFraction = 0.4,
                                            MinChildWeight = 1,
                                            MinSplitGain = 0.0001,
                                            RegAlpha = 0.1,
                                            RegLambda = 0.1,
                                            Subsample = 0.8,
                                            SubsampleFreq = 10
                                        }
                            }));

            return pipeline;
        }

        public static ITransformer Train(MLContext mlContext, IDataView dataView)
        {                                              
            Console.WriteLine("=== Model Train Start");

            // Create Pipeline
            var pipeline = CreatePipeline(mlContext);

            // Train Model
            var model = pipeline.Fit(dataView);
            
            return model;
        }
                
        public static CrossValidationResult<CalibratedBinaryClassificationMetrics>[] TrainCV(MLContext mlContext, IDataView dataView)
        {                                              
            Console.WriteLine("=== Model Cross Validation Train Start");

            // Create Pipeline
            var pipeline = CreatePipeline(mlContext);
            
            // Perform Cross Validation 
            var modelCV = mlContext.BinaryClassification.CrossValidate(dataView, 
                                                                        pipeline, 
                                                                        numFolds: number_of_folds, 
                                                                        labelColumn: "HasDetections",
                                                                        seed: seed);
            
            // Return CrossValidation Results
            return modelCV;
        }
        
        public static void Evaluate(MLContext mlContext, ITransformer model, IDataView dataView)
        {
            Console.WriteLine("=== Model Evaluation");

            var predictions = model.Transform(dataView);
            var metrics = mlContext.BinaryClassification.Evaluate(predictions, "HasDetections");

            Console.WriteLine("===---------------------------");
            Console.WriteLine($"Accuracy: {metrics.Accuracy:P2}");
            Console.WriteLine($"Auc: {metrics.Auc}");
            Console.WriteLine($"F1 Score: {metrics.F1Score}");
            Console.WriteLine("===---------------------------");
        }
        
        public static void EvaluateCV(CrossValidationResult<CalibratedBinaryClassificationMetrics>[] modelCV)
        {
            Console.WriteLine("=== Model CV Evaluation");
            
            var cvAccuracy = modelCV.Select(r => r.Metrics.Accuracy);
            var cvAuc = modelCV.Select(r => r.Metrics.Auc);
            var cvF1score = modelCV.Select(r => r.Metrics.F1Score);

            Console.WriteLine("===---------------------------");
            Console.WriteLine($"Average Accuracy: {cvAccuracy.Average():P2}");
            Console.WriteLine($"Average Auc: {cvAuc.Average()}");
            Console.WriteLine($"Average F1 Score: {cvF1score.Average()}");
            Console.WriteLine("===---------------------------");
        }

        public static void CreateSubmission(MLContext mlContext, ITransformer model)
        {
            Console.WriteLine("=== Generating Submission File");

            // Read Test Data
            IDataView test = reader.Load(testDataFile);

            // Predict on Test Data
            var predictions = model.Transform(test);

            // Convert predictions in ML.NET format to native C# class.
            var nativePredictions = mlContext.Data.CreateEnumerable<Prediction>(predictions, false).ToList();

            // Data
            List<string> machineId = nativePredictions.Select(x => x.MachineIdentifier).ToList();
            List<string> hasDetections = nativePredictions.Select(x => x.Probability.ToString()).ToList();
            
            // Write Submission file
            using (StreamWriter submission_file = new StreamWriter(submissionFile, false))
            {
                // Write Header
                submission_file.WriteLine("MachineIdentifier,HasDetections");

                for (var index = 0; index < machineId.Count(); index++)
                {
                    // Write Predictions
                    submission_file.WriteLine(machineId[index] + "," + hasDetections[index]);
                }                
            }

            Console.WriteLine("=== Submission File Generated");
        }
            
        public static void CreateSubmissionCV(CrossValidationResult<CalibratedBinaryClassificationMetrics>[] modelCV, MLContext mlContext)
        {
            Console.WriteLine("=== Generating Submission File");

            // Read Test Data
            IDataView test = reader.Load(testDataFile);
            
            // Get Model for each fold
            var models = modelCV.Select(r => r.Model);

            // Placeholder for Submission
            NDArray subPreds = np.zeros(7853253);
            List<string> machineIds = new List<string>();

            // Loop through models
            foreach(var model in models)
            {
                var predictions = model.Transform(test);

                // Convert predictions in ML.NET format to native C# class.
                var foldPredictions = mlContext.Data.CreateEnumerable<Prediction>(predictions, false).ToList();

                // Submission info
                if (!machineIds.Any()) machineIds = foldPredictions.Select(x => x.MachineIdentifier).ToList();
                var foldProbabilities = foldPredictions.Select(p => p.Probability).ToArray();
                NDArray foldPreds = np.array(foldProbabilities, typeof(float),  number_of_rows);
                subPreds += foldPreds / number_of_folds;
            }
              
            // Write Submission file
            using (StreamWriter submission_file = new StreamWriter(submissionFile, false))
            {
                // Write Header
                submission_file.WriteLine("MachineIdentifier,HasDetections");

                for (var index = 0; index < machineIds.Count(); index++)
                {
                    // Write Predictions
                    submission_file.WriteLine(machineIds[index] + "," + subPreds[index]);
                }                
            }

            Console.WriteLine("=== Submission File Generated");
        }
    }
}