From 55b241605be91cd1dbb73e7f981bb3386dc474ae Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 14 Jan 2025 11:21:44 +0530
Subject: [PATCH 01/90] init jobserveR

---
 api/utils/log.v2.js                       |   0
 jobServer/Job.js                          |  94 +++++++++
 jobServer/JobManager.js                   | 106 ++++++++++
 jobServer/JobScanner.js                   | 246 ++++++++++++++++++++++
 jobServer/JobServer.js                    | 189 +++++++++++++++++
 jobServer/JobUtils.js                     | 238 +++++++++++++++++++++
 jobServer/README.md                       |   1 +
 jobServer/config.js                       |  24 +++
 jobServer/index.js                        |  64 ++++++
 jobServer/jobRunner/IJobRunner.js         |  84 ++++++++
 jobServer/jobRunner/JobRunnerBullImpl.js  |  95 +++++++++
 jobServer/jobRunner/JobRunnerPulseImpl.js | 114 ++++++++++
 jobServer/jobRunner/index.js              |  39 ++++
 13 files changed, 1294 insertions(+)
 create mode 100644 api/utils/log.v2.js
 create mode 100644 jobServer/Job.js
 create mode 100644 jobServer/JobManager.js
 create mode 100644 jobServer/JobScanner.js
 create mode 100644 jobServer/JobServer.js
 create mode 100644 jobServer/JobUtils.js
 create mode 100644 jobServer/README.md
 create mode 100644 jobServer/config.js
 create mode 100644 jobServer/index.js
 create mode 100644 jobServer/jobRunner/IJobRunner.js
 create mode 100644 jobServer/jobRunner/JobRunnerBullImpl.js
 create mode 100644 jobServer/jobRunner/JobRunnerPulseImpl.js
 create mode 100644 jobServer/jobRunner/index.js

diff --git a/api/utils/log.v2.js b/api/utils/log.v2.js
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/jobServer/Job.js b/jobServer/Job.js
new file mode 100644
index 00000000000..56b618874d3
--- /dev/null
+++ b/jobServer/Job.js
@@ -0,0 +1,94 @@
+/**
+ * Represents a job.
+ */
+class Job {
+
+    jobName = Job.name;
+
+    logger = {
+        d: console.debug,
+        w: console.warn,
+        e: console.error,
+        i: console.info
+    };
+
+    /**
+     * Creates an instance of Job.
+     */
+    constructor() {
+        this.doneCallback = this.doneCallback.bind(this);
+    }
+
+    /**
+     * Sets the name of the job.
+     * @param {String} name The name of the job
+     * @returns {void} The name of the job
+     */
+    setJobName(name) {
+        this.jobName = name;
+    }
+
+    /**
+     * Sets the logger
+     * @param {Logger} logger The logger
+     */
+    setLogger(logger) {
+        this.logger = logger;
+    }
+
+    /**
+     * Runs the job.
+     * @param {Object} db The database connection
+     * @param {Function} done The callback function to be called when the job is done
+     * @throws {Error} If the method is not overridden
+     * @abstract
+     */
+    async run(/*db, done*/) {
+        throw new Error('Job must be overridden');
+    }
+
+    /**
+     * Get the schedule for the job in cron format.
+     * @returns {string} The cron schedule
+     * @throws {Error} If the method is not overridden
+     * @abstract
+     */
+    schedule() {
+        throw new Error('schedule must be overridden');
+    }
+
+    /**
+     * Callback function to be called when the job is done.
+     * @param {Error} error The error that occurred
+     * @param {Object} result The job that was run
+     * @returns {Error} The error to be returned to the job run by pulseScheduler
+     */
+    doneCallback(error, result) {
+        if (error) {
+            this.logger.e('Job failed with error:', error);
+            throw error;
+        }
+        else {
+            this.logger.i('Job completed successfully:', result ? result : '');
+            return result ? result : null;
+        }
+    }
+
+    /**
+     * Runs the job.
+     * @param { Object } db The database
+     * @private
+     */
+    _run(db) {
+        this.logger.d(`Job "${this.jobName}" is starting with database:`, db?._cly_debug?.db);
+        try {
+            this.run(db, this.doneCallback);
+        }
+        catch (error) {
+            this.logger.e(`Job "${this.jobName}" encountered an error during execution:`, error);
+            this.doneCallback(error, this);
+        }
+    }
+}
+
+module.exports = Job;
\ No newline at end of file
diff --git a/jobServer/JobManager.js b/jobServer/JobManager.js
new file mode 100644
index 00000000000..abf51d104ce
--- /dev/null
+++ b/jobServer/JobManager.js
@@ -0,0 +1,106 @@
+const {RUNNER_TYPES, createJobRunner} = require('./JobRunner');
+const config = require("./config");
+
+/**
+ * Manages job configurations and initialization.
+ */
+class JobManager {
+
+    /**
+     * The logger instance
+     * @private
+     * @type {import('../api/utils/log.js').Logger}
+     * */
+    #log;
+
+    /**
+     * The database connection
+     * @type {import('mongodb').Db | null}
+     */
+    #db = null;
+
+    /**
+     * The job runner instance
+     * @type {IJobRunner | JobRunnerPulseImpl |null}
+     */
+    #jobRunner = null;
+
+    /**
+     * Creates a new JobManager instance
+     * @param {Object} db Database connection
+     * @param {function} Logger - Logger constructor
+     */
+    constructor(db, Logger) {
+        this.Logger = Logger;
+        this.#db = db;
+        this.#log = Logger('jobs:manager');
+        this.#log.d('Creating JobManager');
+
+        const runnerType = RUNNER_TYPES.PULSE;
+        const pulseConfig = config.PULSE;
+
+        this.#jobRunner = createJobRunner(this.#db, runnerType, pulseConfig, Logger);
+    }
+
+    /**
+     * Starts the job manager by loading and running jobs.
+     * @param {Object.<string, Function>} jobClasses Object containing job classes keyed by job name
+     * @returns {Promise<void>} A promise that resolves once the jobs are started.
+     */
+    async start(jobClasses) {
+        if (!jobClasses || Object.keys(jobClasses).length === 0) {
+            throw new Error('No job classes provided');
+        }
+        if (!this.#jobRunner) {
+            throw new Error('Job runner not initialized');
+        }
+
+        await this.#loadJobs(jobClasses);
+        await this.#jobRunner.start();
+        await this.#scheduleJobs(jobClasses);
+        this.#log.d('JobManager started successfully');
+    }
+
+    /**
+     * Loads the job classes into the job runner
+     * @param {Object.<string, Function>} jobClasses Object containing job classes keyed by job name
+     * @returns {Promise<void>} A promise that resolves once the jobs are loaded
+     */
+    #loadJobs(jobClasses) {
+        return Promise.all(
+            Object.entries(jobClasses)
+                .map(([name, JobClass]) => {
+                    this.#jobRunner.createJob(name, JobClass);
+                })
+        );
+    }
+
+    /**
+     * Schedules the jobs to run at their respective intervals
+     * @param { Object.<string, Function> } jobClasses Object containing job classes keyed by job name
+     * @returns {Promise<Awaited<void>[]>} A promise that resolves once the jobs are scheduled
+     */
+    #scheduleJobs(jobClasses) {
+        return Promise.all(
+            Object.entries(jobClasses)
+                .map(([name, JobClass]) => {
+                    let instance = new JobClass(name);
+                    let schedule = instance.schedule();
+                    if (schedule) {
+                        this.#jobRunner.schedule(name, schedule);
+                    }
+                })
+        );
+    }
+
+    /**
+     * Closes the JobManager and cleans up resources
+     * @returns {Promise<void>} A promise that resolves once cleanup is complete
+     */
+    async close() {
+        this.#log.d('JobManager closed successfully');
+        await this.#jobRunner.close();
+    }
+}
+
+module.exports = JobManager;
\ No newline at end of file
diff --git a/jobServer/JobScanner.js b/jobServer/JobScanner.js
new file mode 100644
index 00000000000..374172438fa
--- /dev/null
+++ b/jobServer/JobScanner.js
@@ -0,0 +1,246 @@
+/**
+ * @fileoverview Job Scanner module - Scans for and loads job files from both core API and plugins
+ * @module jobs/scanner
+ */
+
+const path = require('path');
+const fs = require('fs').promises;
+const JobUtils = require('./JobUtils');
+
+/**
+ * @typedef {Object} JobConfig
+ * @property {string} category - Category of the job (e.g., 'api' or plugin name)
+ * @property {string} dir - Directory path containing job files
+ */
+
+/**
+ * @typedef {Object} JobDescriptor
+ * @property {string} category - Category of the job
+ * @property {string} name - Name of the job file (without extension)
+ * @property {string} file - Full path to the job file
+ */
+
+/**
+ * @typedef {Object} ScanResult
+ * @property {Object.<string, string>} files - Object storing job file paths, keyed by job name
+ * @property {Object.<string, Function>} classes - Object storing job class implementations, keyed by job name
+ */
+
+/**
+ * Class responsible for scanning and loading job files from both core API and plugins
+ */
+class JobScanner {
+    /**
+     * @private
+     * @type {import('mongodb').Db}
+     * */
+    #db;
+
+    /**
+     * @private
+     * @type {import('../api/utils/log.js').Logger}
+     * */
+    #log;
+
+    /**
+     * @private
+     * @type {import('../plugins/pluginManager.js')}
+     */
+    #pluginManager;
+
+    /**
+     * @private
+     * @type {string}
+     */
+    #baseJobsPath;
+
+    /**
+     * @private
+     * @type {string}
+     */
+    #pluginsPath;
+
+    /**
+     * @private
+     * @type {Object.<string, string>}
+     */
+    #currentFiles = {};
+
+    /**
+     * @private
+     * @type {Object.<string, Function>}
+     */
+    #currentClasses = {};
+
+    /**
+     * Creates a new JobScanner instance
+     * @param {Object} db - Database connection object
+     * @param {function} Logger - Logging function
+     * @param {pluginManager} pluginManager - Plugin manager instance
+     */
+    constructor(db, Logger, pluginManager) {
+        this.#pluginManager = pluginManager;
+        this.#db = db;
+        this.#log = Logger('jobs:scanner');
+        this.#baseJobsPath = path.join(__dirname, '../api/jobs');
+        this.#pluginsPath = path.join(__dirname, '../plugins');
+    }
+
+    /**
+     * Safely reads a directory and filters for job files
+     * @private
+     * @param {JobConfig} jobConfig - Configuration for the job directory to scan
+     * @returns {Promise<JobDescriptor[]>} Array of job descriptors found in directory
+     */
+    async #scanJobDirectory(jobConfig) {
+        try {
+            const files = await fs.readdir(jobConfig.dir);
+            const jobFiles = [];
+
+            for (const file of files) {
+                const fullPath = path.join(jobConfig.dir, file);
+                try {
+                    const stats = await fs.stat(fullPath);
+                    if (stats.isFile()) {
+                        jobFiles.push({
+                            category: jobConfig.category,
+                            name: path.basename(file, '.js'),
+                            file: fullPath
+                        });
+                    }
+                }
+                catch (err) {
+                    this.#log.w(`Failed to stat file ${fullPath}: ${err.message}`);
+                }
+            }
+
+            return jobFiles;
+        }
+        catch (err) {
+            this.#log.w(`Failed to read directory ${jobConfig.dir}: ${err.message}`);
+            return [];
+        }
+    }
+
+    /**
+     * Loads a single job file and returns its information
+     * @private
+     * @param {JobDescriptor} job - Descriptor for the job to load
+     * @returns {{name: string, file: string, implementation: Function}} Job information
+     */
+    #loadJobFile(job) {
+        const jobName = `${job.category}:${job.name}`;
+        try {
+            // Clear require cache to ensure fresh load
+            // delete require.cache[require.resolve(job.file)];
+            const implementation = require(job.file);
+            this.#log.d(`Loaded job ${jobName} from ${job.file}`);
+            JobUtils.validateJobClass(implementation);
+
+            return {
+                name: jobName,
+                file: job.file,
+                implementation
+            };
+        }
+        catch (err) {
+            this.#log.e(`Failed to load job ${job.file}: ${err.message}`, err);
+            return null;
+        }
+    }
+
+    /**
+     * Initializes plugin configurations
+     * @private
+     * @returns {Promise<void>} A promise that resolves once the plugin configurations are loaded
+     */
+    async #initializeConfig() {
+        return new Promise((resolve) => {
+            this.#pluginManager.loadConfigs(this.#db, () => resolve());
+        });
+    }
+
+    /**
+     * Gets the list of directories to scan for jobs
+     * @private
+     * @param {string[]} plugins - List of plugin names
+     * @returns {JobConfig[]} Array of job directory configurations
+     */
+    #getJobDirectories(plugins) {
+        return [
+            {
+                category: 'api',
+                dir: this.#baseJobsPath
+            },
+            ...plugins.map(plugin => ({
+                category: plugin,
+                dir: path.join(this.#pluginsPath, plugin, 'api/jobs')
+            }))
+        ];
+    }
+
+    /**
+     * @private
+     * Stores a loaded job in the internal collections
+     * @param {{name: string, file: string, implementation: Function}} loaded - Loaded job information
+     */
+    #storeLoadedJob(loaded) {
+        if (loaded) {
+            this.#currentFiles[loaded.name] = loaded.file;
+            this.#currentClasses[loaded.name] = loaded.implementation;
+        }
+    }
+
+    /**
+     * Scans for job files and returns the loaded jobs
+     * @returns {Promise<ScanResult>} Object containing job files and implementations
+     * @throws {Error} If plugin configuration is invalid or missing
+     */
+    async scan() {
+        // Initialize plugin manager
+        await this.#initializeConfig();
+
+        const plugins = this.#pluginManager.getPlugins(true);
+        if (!plugins?.length) {
+            throw new Error('No valid plugins.json configuration found');
+        }
+
+        this.#log.i('Scanning plugins:', plugins);
+
+        // Reset current collections
+        this.#currentFiles = {};
+        this.#currentClasses = {};
+
+        // Build list of directories to scan
+        const jobDirs = this.#getJobDirectories(plugins);
+
+        // Scan all directories concurrently
+        const jobFiles = await Promise.all(
+            jobDirs.map(dir => this.#scanJobDirectory(dir))
+        );
+
+        // Load all discovered jobs and collect results
+        jobFiles.flat()
+            .filter(Boolean)
+            .forEach(job => {
+                const loaded = this.#loadJobFile(job);
+                this.#storeLoadedJob(loaded);
+            });
+
+        return {
+            files: this.#currentFiles,
+            classes: this.#currentClasses
+        };
+    }
+
+    /**
+     * Gets the current number of loaded jobs
+     * @public
+     * @returns {number} Number of loaded jobs
+     */
+    get loadedJobCount() {
+        return Object.keys(this.#currentFiles).length;
+    }
+}
+
+module.exports = JobScanner;
\ No newline at end of file
diff --git a/jobServer/JobServer.js b/jobServer/JobServer.js
new file mode 100644
index 00000000000..a2441896944
--- /dev/null
+++ b/jobServer/JobServer.js
@@ -0,0 +1,189 @@
+const JobManager = require('./JobManager');
+const JobScanner = require('./JobScanner');
+
+/**
+ * Class representing a job process.
+ */
+class JobServer {
+
+    /**
+     * The logger instance
+     * @private
+     * @type {import('../api/utils/log.js').Logger}
+     * */
+    #log;
+
+    /**
+     * The plugin manager instance
+     * @private
+     * @type {import('../plugins/pluginManager.js')}
+     */
+    #pluginManager;
+
+    /**
+     * The Countly common object
+     * @private
+     * @type {import('../api/utils/common.js')}
+     */
+    #common;
+
+    /**
+     * The job manager instance
+     * @private
+     * @type {JobManager}
+     */
+    #jobManager;
+
+    /**
+     * The job scanner instance
+     * @private
+     * @type {JobScanner}
+     */
+    #jobScanner;
+
+    /**
+     * Flag indicating whether the job process is running
+     * @private
+     */
+    #isRunning = false;
+
+    /**
+     * The database connection
+     * @type {import('mongodb').Db | null}
+     */
+    #db = null;
+
+    /**
+     * Creates a new JobProcess instance.
+     * @param {Object} common Countly common
+     * @param {function} Logger - Logger constructor
+     * @param {pluginManager} pluginManager - Plugin manager instance
+     * @returns {Promise<JobServer>} A promise that resolves to a new JobProcess instance.
+     */
+    static async create(common, Logger, pluginManager) {
+        const process = new JobServer(common, Logger, pluginManager);
+        await process.init();
+        return process;
+    }
+
+    /**
+     * Creates a new JobServer instance
+     * @param {Object} common Countly common
+     * @param {function} Logger - Logger constructor
+     * @param {pluginManager} pluginManager - Plugin manager instance
+     */
+    constructor(common, Logger, pluginManager) {
+        this.#common = common;
+        this.Logger = Logger;
+        this.#log = Logger('jobs:server');
+        this.#pluginManager = pluginManager;
+    }
+
+    /**
+     * init the job process.
+     * @returns {Promise<void>} A promise that resolves once the job process is initialized.
+     */
+    async init() {
+        try {
+            await this.#connectToDb();
+
+            this.#jobManager = new JobManager(this.#db, this.Logger);
+            this.#jobScanner = new JobScanner(this.#db, this.Logger, this.#pluginManager);
+
+            this.#setupSignalHandlers();
+            this.#log.i('Job process init successfully');
+        }
+        catch (error) {
+            this.#log.e('Failed to initialize job process:', error);
+            throw error;
+        }
+    }
+
+    /**
+     * Starts the job process.
+     * @returns {Promise<void>} A promise that resolves once the job process is started.
+     */
+    async start() {
+        if (this.#isRunning) {
+            this.#log.w('Process is already running');
+            return;
+        }
+
+        try {
+            this.#isRunning = true;
+            this.#log.i('Starting job process');
+
+            // Load job classes
+            const { classes } = await this.#jobScanner.scan();
+            // Start job manager
+
+            await this.#jobManager.start(classes);
+            this.#log.i('Job process is running');
+        }
+        catch (error) {
+            this.#log.e('Error starting job process:', error);
+            await this.#shutdown(1);
+        }
+    }
+
+    /**
+     * Connects to the mongo database.
+     * @returns {Promise<void>} A promise that resolves once the connection is established.
+     */
+    async #connectToDb() {
+        try {
+            this.#db = await this.#pluginManager.dbConnection('countly');
+        }
+        catch (e) {
+            this.#log.e('Failed to connect to database:', e);
+            throw e;
+        }
+    }
+
+    /**
+     * Sets up signal handlers for graceful shutdown and uncaught exceptions.
+     */
+    #setupSignalHandlers() {
+        // Handle graceful shutdown
+        process.on('SIGTERM', () => this.#shutdown());
+        process.on('SIGINT', () => this.#shutdown());
+
+        // Handle uncaught errors
+        process.on('uncaughtException', (error) => {
+            this.#log.e('Uncaught exception:', error);
+            this.#shutdown(1);
+        });
+    }
+
+    /**
+     * Shuts down the job process.
+     * @param {number} [exitCode=0] - The exit code to use when shutting down the process.
+     * @returns {Promise<void>} A promise that resolves once the job process is shut down.
+     */
+    async #shutdown(exitCode = 0) {
+        if (!this.#isRunning) {
+            this.#log.w('Shutdown called but process is not running');
+            process.exit(exitCode);
+            return;
+        }
+
+        this.#log.i('Shutting down job process...');
+        this.#isRunning = false;
+
+        try {
+            if (this.#jobManager) {
+                await this.#jobManager.close();
+            }
+            this.#log.i('Job process shutdown complete');
+        }
+        catch (error) {
+            this.#log.e('Error during shutdown:', error);
+            exitCode = 1;
+        }
+        finally {
+            process.exit(exitCode);
+        }
+    }
+}
+
+module.exports = JobServer;
\ No newline at end of file
diff --git a/jobServer/JobUtils.js b/jobServer/JobUtils.js
new file mode 100644
index 00000000000..2b69927ae7b
--- /dev/null
+++ b/jobServer/JobUtils.js
@@ -0,0 +1,238 @@
+const Job = require('./Job');
+const {isValidCron} = require('cron-validator');
+const later = require('@breejs/later');
+
+/**
+ * Class responsible for validating job classes.
+ */
+class JobUtils {
+
+    /**
+     * Validates if a given job class is valid.
+     * @param {Function} JobClass - The job class to validate.
+     * @param {Function} [BaseClass=Job] - The base class that the job class should extend.
+     * @returns {boolean} True if the job class is valid.
+     * @throws {Error} If the job class is not a constructor or does not extend the base class.
+     */
+    static validateJobClass(JobClass, BaseClass = Job) {
+        // Check if it's a class/constructor
+        if (typeof JobClass !== 'function') {
+            throw new Error('Job must be a class constructor');
+        }
+
+        // Check if it inherits from the base class
+        if (!(JobClass.prototype instanceof BaseClass)) {
+            throw new Error(`Job class must extend ${BaseClass.name}`);
+        }
+
+        // Additional checks can be added here
+        return true;
+    }
+
+    /**
+     * Converts a later.js schedule to a cron string.
+     * @param {String} laterString - The later.js schedule string.
+     * @constructor
+     * @retuns {String} The cron string.
+     *
+     * @note
+     * TESTS NEEDED
+     *
+     * "every 5 minutes"
+     * "at 01:01 am every 1 day"
+     * "every seconds"
+     * "at 3:00 am every 7 days"
+     * "every 1 hour on the 1st min"
+     * "every 1 day"
+     * "every 1 day"
+     * "every 1 hour starting on the 0 min"
+     * "once in 2 hours"
+     * "0 0 * * *"
+     * "every 5 minutes"
+     * "at 10:15 am every weekday"
+     * "at 00:01 am every 1 day"
+     * "every 1 day"
+     * "every 1 day"
+     * "every 1 day"
+     * "every 1 day"
+     * "every 5 minutes"
+     * "every 10 minutes"
+     * "every 1 minute"
+     * "every 1 day"
+     * "every 10 minutes"
+     * "every 30 minutes"
+     * "every 1 minutes"
+     * "every 2 minutes"
+     * "every 5 minutes"
+     * "every 5 minute"
+     * "every 1 hour"
+     * "at 00:30 am every 1 day"
+     * "every 5 minutes"
+     * "every 12 hours"
+     * "every 5 minutes"
+     * "every 1 year"
+     * "every 1 day"
+     * "every 1 hour"
+     * "at 03:25 am every 1 day"
+     * "every 1 hour"
+     * "every 5 minutes"
+     */
+    static LaterToCron(laterString) {
+        // Handle direct cron expressions
+        if (isValidCron(laterString)) {
+            return laterString;
+        }
+
+        // Parse the schedule
+        const schedule = later.parse.text(laterString);
+        if (schedule.error !== -1) {
+            throw new Error(`Invalid schedule string: ${laterString}`);
+        }
+
+
+        /**
+         * Get the values of a component from the schedule
+         * @param {Object} component - The component to extract values from.
+         * @returns {any[]|*[]} The values of the component.
+         */
+        function getValues(component) {
+            if (!schedule.schedules || schedule.schedules.length === 0) {
+                return [];
+            }
+            const values = new Set();
+            schedule.schedules.forEach(s => {
+                if (s[component]) {
+                    s[component].forEach(val => values.add(val));
+                }
+            });
+            return Array.from(values).sort((a, b) => a - b);
+        }
+
+        // Extract schedule components
+        const minutes = getValues('m');
+        const hours = getValues('h');
+        const daysOfMonth = getValues('D');
+        const months = getValues('M');
+        const daysOfWeek = getValues('d');
+
+        // Common pattern detection
+        const isEveryMinute = laterString.match(/every\s+(\d+)\s*minute/);
+        const isEveryHour = laterString.match(/every\s+(\d+)\s*hour/);
+        const isSpecificTime = laterString.match(/at\s+(\d{1,2}):(\d{2})\s*(am|pm)?/i);
+        const isEveryDay = laterString.includes('every 1 day') || laterString.includes('every day');
+        const isEveryYear = laterString.includes('every 1 year') || laterString.includes('yearly');
+        const isWeekday = laterString.toLowerCase().includes('weekday');
+        const isEveryNDays = laterString.match(/every\s+(\d+)\s+days?/);
+        const isLastDayOfMonth = laterString.includes("last day of the month");
+        const isOnceInHours = laterString.match(/once in (\d+) hours?/);
+
+        // Handle specific patterns
+        if (isEveryYear) {
+            return '0 0 1 1 *'; // Midnight on January 1st
+        }
+
+        if (isEveryMinute) {
+            const interval = parseInt(isEveryMinute[1]);
+            return `*/${interval} * * * *`;
+        }
+
+        if (isEveryHour) {
+            const interval = parseInt(isEveryHour[1]);
+            return laterString.includes('on the 1st min')
+                ? `1 */${interval} * * *`
+                : `0 */${interval} * * *`;
+        }
+
+        if (isSpecificTime && isEveryDay) {
+            let [, hour, minute, meridiem] = isSpecificTime;
+            hour = parseInt(hour);
+            minute = parseInt(minute);
+            if (meridiem) {
+                hour = meridiem.toLowerCase() === 'pm' && hour < 12 ? hour + 12 : hour;
+                hour = meridiem.toLowerCase() === 'am' && hour === 12 ? 0 : hour;
+            }
+            return `${minute} ${hour} * * *`;
+        }
+
+        if (isWeekday) {
+            if (isSpecificTime) {
+                let [, hour, minute, meridiem] = isSpecificTime;
+                hour = parseInt(hour);
+                minute = parseInt(minute);
+                if (meridiem) {
+                    hour = meridiem.toLowerCase() === 'pm' && hour < 12 ? hour + 12 : hour;
+                    hour = meridiem.toLowerCase() === 'am' && hour === 12 ? 0 : hour;
+                }
+                return `${minute} ${hour} * * 1-5`;
+            }
+            return '0 0 * * 1-5';
+        }
+
+        if (isEveryNDays) {
+            const interval = parseInt(isEveryNDays[1]);
+            if (isSpecificTime) {
+                let [, hour, minute, meridiem] = isSpecificTime;
+                hour = parseInt(hour);
+                minute = parseInt(minute);
+                if (meridiem) {
+                    hour = meridiem.toLowerCase() === 'pm' && hour < 12 ? hour + 12 : hour;
+                    hour = meridiem.toLowerCase() === 'am' && hour === 12 ? 0 : hour;
+                }
+                return `${minute} ${hour} */${interval} * *`;
+            }
+            return `0 0 */${interval} * *`;
+        }
+
+        if (isLastDayOfMonth) {
+            if (isSpecificTime) {
+                let [, hour, minute, meridiem] = isSpecificTime;
+                hour = parseInt(hour);
+                minute = parseInt(minute);
+                if (meridiem) {
+                    hour = meridiem.toLowerCase() === 'pm' && hour < 12 ? hour + 12 : hour;
+                    hour = meridiem.toLowerCase() === 'am' && hour === 12 ? 0 : hour;
+                }
+                return `${minute} ${hour} L * *`;
+            }
+            return `0 0 L * *`;
+        }
+
+        if (isOnceInHours) {
+            const interval = parseInt(isOnceInHours[1]);
+            return `0 */${interval} * * *`;
+        }
+
+        /**
+         * Formats a component of the cron string.
+         * @param {String} values - The values of the component.
+         * @param {Number} total - The total number of possible
+         * @returns {string|*} The formatted component.
+         */
+        function formatComponent(values, total) {
+            if (values.length === 0) {
+                return '*';
+            }
+            if (values.length === 1) {
+                return values[0];
+            }
+            if (values.length === total) {
+                return '*';
+            }
+
+            const interval = values.length > 1 && values[1] - values[0];
+            const isInterval = values.every((v, i) => i === 0 || v - values[i - 1] === interval);
+            return isInterval ? `*/${interval}` : values.join(',');
+        }
+
+        const minutePart = formatComponent(minutes, 60);
+        const hourPart = formatComponent(hours, 24);
+        const dayPart = formatComponent(daysOfMonth, 31);
+        const monthPart = formatComponent(months, 12);
+        const dayOfWeekPart = formatComponent(daysOfWeek, 7);
+
+        return `${minutePart} ${hourPart} ${dayPart} ${monthPart} ${dayOfWeekPart}`;
+    }
+
+}
+
+module.exports = JobUtils;
\ No newline at end of file
diff --git a/jobServer/README.md b/jobServer/README.md
new file mode 100644
index 00000000000..c858d63ada6
--- /dev/null
+++ b/jobServer/README.md
@@ -0,0 +1 @@
+# Job Server
\ No newline at end of file
diff --git a/jobServer/config.js b/jobServer/config.js
new file mode 100644
index 00000000000..acb82171fbc
--- /dev/null
+++ b/jobServer/config.js
@@ -0,0 +1,24 @@
+/**
+ * Default configuration for Pulse jobs
+ * @type {import('@pulsecron/pulse').PulseConfig}
+ */
+const DEFAULT_PULSE_CONFIG = {
+    name: 'core', // Name of the Pulse instance
+    processEvery: '10 seconds', // Frequency to check for new jobs
+    maxConcurrency: 5, // Maximum number of jobs that can run concurrently
+    defaultConcurrency: 3, // Default number of jobs that can run concurrently
+    lockLimit: 3, // Maximum number of jobs that can be locked at the same time
+    defaultLockLimit: 2, // Default number of jobs that can be locked at the same time
+    defaultLockLifetime: 55 * 60 * 1000, // 55 minutes, time in milliseconds for how long a job should be locked
+    sort: { nextRunAt: 1, priority: -1 }, // Sorting order for job execution
+    disableAutoIndex: false, // Whether to disable automatic index creation
+    resumeOnRestart: true, // Whether to resume jobs on restart
+    db: {
+        collection: 'pulseJobs', // MongoDB collection to store jobs
+    }
+};
+
+module.exports = {
+    PULSE: DEFAULT_PULSE_CONFIG,
+    BULL: {}
+};
\ No newline at end of file
diff --git a/jobServer/index.js b/jobServer/index.js
new file mode 100644
index 00000000000..c551753bed4
--- /dev/null
+++ b/jobServer/index.js
@@ -0,0 +1,64 @@
+/**
+ * @module jobServer
+ * @version 2.0
+ * @author Countly
+ *
+ * @note
+ *  Dependencies like common utilities and plugin manager should only be imported in this entry file
+ *  and injected into the respective modules via their constructors or create methods.
+ *
+ * @description
+ * This module provides the job management system for countly.
+ * It handles job scheduling, execution, and management through a flexible API.
+ *
+ *
+ * @property {typeof import('./Job')} Job - Class for creating and managing individual jobs
+ * @property {typeof import('./JobServer')} JobServer - Class for running jobs in a separate process
+ *
+ * @throws {Error} When database connection fails during initialization
+ * @throws {Error} When job definition is invalid
+ *
+ * @requires './Job'
+ * @requires './JobServer'
+ *
+ * @external Common
+ * @see {@import ../api/utils/common.js|common}
+ *
+ * @external PluginManager
+ * @see {@import ../plugins/pluginManager.js|PluginManager}
+ *
+ * @execution
+ * This module can be run directly as a standalone process:
+ *
+ * ```bash
+ * node index.js
+ * ```
+ * When run directly, it will:
+ * 1. Create a new JobServer instance
+ * 2. Initialize it with the common utilities and plugin manager
+ * 3. Start the job processing
+ * 4. Handle process signals (SIGTERM, SIGINT) for graceful shutdown
+ *
+ */
+
+const JobServer = require('./JobServer');
+const Job = require('./Job');
+
+// Start the process if this file is run directly
+if (require.main === module) {
+
+    const common = require('../api/utils/common.js');
+    const pluginManager = require('../plugins/pluginManager.js');
+    const Logger = common.log;
+
+    JobServer.create(common, Logger, pluginManager)
+        .then(process => process.start())
+        .catch(error => {
+            console.error('Failed to start job process:', error);
+            process.exit(1);
+        });
+}
+
+module.exports = {
+    Job: Job
+};
\ No newline at end of file
diff --git a/jobServer/jobRunner/IJobRunner.js b/jobServer/jobRunner/IJobRunner.js
new file mode 100644
index 00000000000..86fd08daf59
--- /dev/null
+++ b/jobServer/jobRunner/IJobRunner.js
@@ -0,0 +1,84 @@
+/**
+ * Interface for job runner implementations
+ */
+class IJobRunner {
+
+    db;
+
+    config;
+
+    log;
+
+    /**
+     *@param {Object} db Database connection
+     * @param {Object} config Configuration
+     * @param {function} Logger - Logger constructor
+     */
+    constructor(db, config, Logger) {
+        this.log = Logger('jobs:runner');
+        this.db = db;
+        this.config = config;
+    }
+
+    /**
+     * Defines a new job
+     * @param {String} jobName The name of the job
+     * @param {function} jobRunner The job runner function
+     * @param {Object} [jobOptions=null] The job options
+     * @returns {Promise<void>} A promise that resolves once the job is defined
+     */
+    async createJob(/*jobName, jobRunner, jobOptions=null*/) {
+        throw new Error('Method not implemented');
+    }
+
+    /**
+     * Schedules jobs
+     * @param {String} name The name of the job
+     * @param {String} schedule Cron string for the job schedule
+     * @param {Object} data Data to pass to the job
+     * @returns {Promise<void>} A promise that resolves once the job is scheduled
+     */
+    async schedule(/*name, schedule, data*/) {
+        throw new Error('Method not implemented');
+    }
+
+    /**
+     * Runs a job once
+     * @param {String} name The name of the job
+     * @param {Date} date The schedule
+     * @param {Object} data Data to pass to the job
+     * @returns {Promise<void>} A promise that resolves once the job is scheduled
+     */
+    async once(/*name, date, data*/) {
+        throw new Error('Method not implemented');
+    }
+
+    /**
+     * Runs a job now
+     * @param {String} name The name of the job
+     * @param {Object} data Data to pass to the job
+     * @returns {Promise<void>} A promise that resolves once the job is run
+     */
+    async now(/*name, data*/) {
+        throw new Error('Method not implemented');
+    }
+
+    /**
+     * Starts the job runner
+     * @param {Object.<string, Function>} jobClasses Object containing job classes keyed by job name
+     * @returns {Promise<void>} A promise that resolves once the runner is started
+     */
+    async start(/*jobClasses*/) {
+        throw new Error('Method not implemented');
+    }
+
+    /**
+     * Closes the job runner and cleans up resources
+     * @returns {Promise<void>} A promise that resolves once the runner is closed
+     */
+    async close() {
+        throw new Error('Method not implemented');
+    }
+}
+
+module.exports = IJobRunner;
\ No newline at end of file
diff --git a/jobServer/jobRunner/JobRunnerBullImpl.js b/jobServer/jobRunner/JobRunnerBullImpl.js
new file mode 100644
index 00000000000..826e0ae7497
--- /dev/null
+++ b/jobServer/jobRunner/JobRunnerBullImpl.js
@@ -0,0 +1,95 @@
+const IJobRunner = require('./IJobRunner');
+const { Queue, Worker } = require('bullmq');
+
+/**
+ * BullMQ implementation of the job runner
+ */
+class JobRunnerBullImpl extends IJobRunner {
+    /**
+     * Map of queue names to BullMQ Queue instances
+     * @type {Map<string, import('bullmq').Queue>}
+     */
+    #queues = new Map();
+
+    /**
+     * Map of queue names to BullMQ Worker instances
+     * @type {Map<string, import('bullmq').Worker>}
+     */
+    #workers = new Map();
+
+    #bullConfig;
+
+    #redisConnection;
+
+    #Queue;
+
+    #Worker;
+
+    /**
+     * Creates a new BullMQ job runner
+     * @param {Object} db Database connection
+     * @param {Object} config Configuration object
+     */
+    constructor(db, config) {
+        super(db, config);
+        this.#Queue = Queue;
+        this.#Worker = Worker;
+        this.#redisConnection = config.redis;
+        this.#bullConfig = config.bullConfig;
+    }
+
+    /**
+     * @param {Object.<string, Function>} jobClasses Object containing job classes keyed by job name
+     */
+    async start(jobClasses) {
+
+        // Create queues and workers for each job
+        for (const [name, JobClass] of Object.entries(jobClasses)) {
+            // Create queue
+            const queue = new this.#Queue(name, {
+                connection: this.#redisConnection,
+                ...this.#bullConfig
+            });
+            this.#queues.set(name, queue);
+
+            // Create worker
+            const worker = new this.#Worker(name, async(job) => {
+                const instance = new JobClass(name);
+                await instance.run(job);
+            }, {
+                connection: this.#redisConnection,
+                ...this.#bullConfig
+            });
+            this.#workers.set(name, worker);
+
+            // Handle worker events
+            worker.on('completed', (job) => {
+                console.log(`Job ${job.id} completed`);
+            });
+
+            worker.on('failed', (job, err) => {
+                console.error(`Job ${job.id} failed:`, err);
+            });
+        }
+    }
+
+    /**
+     * Closes all queues and workers
+     * @returns {Promise<void>} A promise that resolves once all queues and workers are closed
+     */
+    async close() {
+        // Close all workers
+        for (const worker of this.#workers.values()) {
+            await worker.close();
+        }
+        this.#workers.clear();
+
+        // Close all queues
+        for (const queue of this.#queues.values()) {
+            await queue.close();
+        }
+        this.#queues.clear();
+    }
+}
+
+module.exports = JobRunnerBullImpl;
\ No newline at end of file
diff --git a/jobServer/jobRunner/JobRunnerPulseImpl.js b/jobServer/jobRunner/JobRunnerPulseImpl.js
new file mode 100644
index 00000000000..c68b6afedd2
--- /dev/null
+++ b/jobServer/jobRunner/JobRunnerPulseImpl.js
@@ -0,0 +1,114 @@
+const IJobRunner = require('./IJobRunner');
+const { Pulse, JobPriority } = require('@pulsecron/pulse');
+const {isValidCron} = require('cron-validator');
+
+/**
+ * Pulse implementation of the job runner
+ */
+class JobRunnerPulseImpl extends IJobRunner {
+    /**
+     * The Pulse runner instance
+     * @type {import('@pulsecron/pulse').Pulse}
+     */
+    #pulseRunner;
+
+    /**
+     * Creates a new Pulse job runner
+     * @param {Object} db Database connection
+     * @param {Object} config Configuration object
+     * @param {function} Logger - Logger constructor
+     */
+    constructor(db, config, Logger) {
+        super(db, config, Logger);
+        this.log = Logger('jobs:runner:pulse');
+        this.#pulseRunner = new Pulse({
+            ...config,
+            mongo: this.db,
+        });
+    }
+
+    /**
+     * Starts the Pulse runner
+     */
+    async start() {
+        if (!this.#pulseRunner) {
+            throw new Error('Pulse runner not initialized');
+        }
+        await this.#pulseRunner.start();
+    }
+
+    /**
+     * Creates a new job with the given class
+     * @param {string} jobName The name of the job
+     * @param {Function} JobClass The job class to create
+     * @returns {Promise<void>} A promise that resolves once the job is created
+     */
+    async createJob(jobName, JobClass) {
+        const instance = new JobClass(jobName);
+        instance.setLogger(this.log);
+        // const schedule = instance.schedule();
+
+        this.#pulseRunner.define(
+            jobName,
+            instance._run.bind(instance, this.db),
+            {
+                priority: JobPriority.normal,
+                concurrency: 1,
+                lockLifetime: 10000,
+                shouldSaveResult: true,
+                attempts: 3,
+                backoff: {
+                    type: 'exponential',
+                    delay: 2000
+                }
+            }
+        );
+        //
+        // const job = new PulseJob(
+        //     {
+        //         name: jobName,
+        //         pulse: this.#pulseRunner,
+        //     }
+        // );
+        //
+        // job.unique({'name': jobName});
+        //
+        // job.repeatEvery(
+        //     schedule,
+        //     {
+        //         timezone: 'Asia/Kolkata',
+        //     }
+        // );
+        //
+        // job.setShouldSaveResult(true);
+        // await job.save();
+    }
+
+
+    /**
+     * Schedules jobs
+     * @param {String} name The name of the job
+     * @param {String} schedule Cron string for the job schedule
+     * @param {Object} data Data to pass to the job
+     * @returns {Promise<void>} A promise that resolves once the job is scheduled
+     */
+    async schedule(name, schedule, data) {
+        if (!isValidCron(schedule)) {
+            throw new Error('Invalid cron schedule');
+        }
+        await this.#pulseRunner.every(schedule, name, data);
+    }
+
+    /**
+     * Closes the Pulse runner
+     * @returns {Promise<void>} A promise that resolves once the runner is closed
+     */
+    async close() {
+        if (this.#pulseRunner) {
+            await this.#pulseRunner.close();
+            this.#pulseRunner = null;
+        }
+    }
+}
+
+module.exports = JobRunnerPulseImpl;
\ No newline at end of file
diff --git a/jobServer/jobRunner/index.js b/jobServer/jobRunner/index.js
new file mode 100644
index 00000000000..24e62197db3
--- /dev/null
+++ b/jobServer/jobRunner/index.js
@@ -0,0 +1,39 @@
+const IJobRunner = require('./IJobRunner');
+// const JobRunnerBullImpl = require('./JobRunnerBullImpl');
+const JobRunnerPulseImpl = require('./JobRunnerPulseImpl');
+
+/**
+ * JobRunner implementation types
+ * @enum {string}
+ */
+const RUNNER_TYPES = {
+    // BULL: 'bull',
+    PULSE: 'pulse'
+};
+
+/**
+ * Job Runner factory
+ * 
+ * @param {Object} db The database connection
+ * @param {string} [type='pulse'] The type of runner to create ('bull' or 'pulse')
+ * @param {Object} [config={}] Configuration specific to the runner implementation
+ * @param {function} Logger - Logger constructor
+ * @returns {IJobRunner} An instance of the specified JobRunner implementation
+ * @throws {Error} If an invalid runner type is specified
+ */
+function createJobRunner(db, type = RUNNER_TYPES.PULSE, config = {}, Logger) {
+
+    switch (type.toLowerCase()) {
+    // case RUNNER_TYPES.BULL:
+    //     return new JobRunnerBullImpl(db, config, Logger);
+    case RUNNER_TYPES.PULSE:
+        return new JobRunnerPulseImpl(db, config, Logger);
+    default:
+        throw new Error(`Invalid runner type: ${type}. Must be one of: ${Object.values(RUNNER_TYPES).join(', ')} and implementation of ` + IJobRunner.name);
+    }
+}
+
+module.exports = {
+    createJobRunner,
+    RUNNER_TYPES
+};
\ No newline at end of file

From e9bb5b8b3722ec1473ba46ce031b66c1d7265e00 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 14 Jan 2025 11:26:12 +0530
Subject: [PATCH 02/90] lint and deps

---
 .eslintrc.json | 1 +
 package.json   | 3 +++
 2 files changed, 4 insertions(+)

diff --git a/.eslintrc.json b/.eslintrc.json
index ce496cb3e45..52cb1685c42 100644
--- a/.eslintrc.json
+++ b/.eslintrc.json
@@ -254,6 +254,7 @@
         {
             "files": [
                 "api/**/*.js",
+                "jobServer/**/*.js",
                 "frontend/express/*.js",
                 "frontend/express/libs/*.js",
                 "plugins/pluginManager.js",
diff --git a/package.json b/package.json
index 934a89a0f01..59570d343f0 100644
--- a/package.json
+++ b/package.json
@@ -31,6 +31,8 @@
     "test": "grunt --verbose"
   },
   "dependencies": {
+    "@breejs/later": "^4.2.0",
+    "@pulsecron/pulse": "^1.6.7",
     "all-the-cities": "3.1.0",
     "argon2": "0.41.1",
     "async": "3.2.6",
@@ -45,6 +47,7 @@
     "countly-root": "file:api/utils/countly-root",
     "countly-sdk-nodejs": "*",
     "countly-sdk-web": "*",
+    "cron-validator": "^1.3.1",
     "csurf": "^1.11.0",
     "csvtojson": "2.0.10",
     "ejs": "3.1.10",

From 2e2e2f44930ad6974c97c5a1d5555c9bf073fb9a Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 14 Jan 2025 13:29:02 +0530
Subject: [PATCH 03/90] deepscan errors

---
 jobServer/JobManager.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/jobServer/JobManager.js b/jobServer/JobManager.js
index abf51d104ce..1ec76a463d1 100644
--- a/jobServer/JobManager.js
+++ b/jobServer/JobManager.js
@@ -70,7 +70,7 @@ class JobManager {
         return Promise.all(
             Object.entries(jobClasses)
                 .map(([name, JobClass]) => {
-                    this.#jobRunner.createJob(name, JobClass);
+                    return this.#jobRunner.createJob(name, JobClass);
                 })
         );
     }
@@ -83,11 +83,11 @@ class JobManager {
     #scheduleJobs(jobClasses) {
         return Promise.all(
             Object.entries(jobClasses)
-                .map(([name, JobClass]) => {
+                .map(async([name, JobClass]) => {
                     let instance = new JobClass(name);
-                    let schedule = instance.schedule();
+                    let schedule = await instance.schedule();
                     if (schedule) {
-                        this.#jobRunner.schedule(name, schedule);
+                        return this.#jobRunner.schedule(name, schedule);
                     }
                 })
         );

From 8918bcadc5b1300e78c8cbc019b77d10bd676cf5 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 14 Jan 2025 14:32:53 +0530
Subject: [PATCH 04/90] test config

---
 jobServer/config.js | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/jobServer/config.js b/jobServer/config.js
index acb82171fbc..fea20aa24c8 100644
--- a/jobServer/config.js
+++ b/jobServer/config.js
@@ -3,12 +3,12 @@
  * @type {import('@pulsecron/pulse').PulseConfig}
  */
 const DEFAULT_PULSE_CONFIG = {
-    name: 'core', // Name of the Pulse instance
-    processEvery: '10 seconds', // Frequency to check for new jobs
-    maxConcurrency: 5, // Maximum number of jobs that can run concurrently
-    defaultConcurrency: 3, // Default number of jobs that can run concurrently
-    lockLimit: 3, // Maximum number of jobs that can be locked at the same time
-    defaultLockLimit: 2, // Default number of jobs that can be locked at the same time
+    // name: 'core', // Name of the Pulse instance
+    processEvery: '3 seconds', // Frequency to check for new jobs
+    maxConcurrency: 1, // Maximum number of jobs that can run concurrently
+    defaultConcurrency: 1, // Default number of jobs that can run concurrently
+    lockLimit: 1, // Maximum number of jobs that can be locked at the same time
+    defaultLockLimit: 1, // Default number of jobs that can be locked at the same time
     defaultLockLifetime: 55 * 60 * 1000, // 55 minutes, time in milliseconds for how long a job should be locked
     sort: { nextRunAt: 1, priority: -1 }, // Sorting order for job execution
     disableAutoIndex: false, // Whether to disable automatic index creation

From 3010da01113079e51a609f786696498815d5df45 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 14 Jan 2025 14:33:24 +0530
Subject: [PATCH 05/90] handle promise/callback/fn function for job run

---
 jobServer/Job.js | 63 ++++++++++++++++++++++++++++++------------------
 1 file changed, 40 insertions(+), 23 deletions(-)

diff --git a/jobServer/Job.js b/jobServer/Job.js
index 56b618874d3..e9b7c63ddda 100644
--- a/jobServer/Job.js
+++ b/jobServer/Job.js
@@ -16,7 +16,7 @@ class Job {
      * Creates an instance of Job.
      */
     constructor() {
-        this.doneCallback = this.doneCallback.bind(this);
+        this.logger.d(`Job instance"${this.jobName}" created`);
     }
 
     /**
@@ -58,35 +58,52 @@ class Job {
     }
 
     /**
-     * Callback function to be called when the job is done.
-     * @param {Error} error The error that occurred
-     * @param {Object} result The job that was run
-     * @returns {Error} The error to be returned to the job run by pulseScheduler
-     */
-    doneCallback(error, result) {
-        if (error) {
-            this.logger.e('Job failed with error:', error);
-            throw error;
-        }
-        else {
-            this.logger.i('Job completed successfully:', result ? result : '');
-            return result ? result : null;
-        }
-    }
-
-    /**
-     * Runs the job.
-     * @param { Object } db The database
+     * Runs the job and handles both Promise and callback patterns.
+     * @param {Object} db The database
+     * @param {Object} job The job instance
+     * @param {Function} done Callback to be called when job completes
      * @private
      */
-    _run(db) {
+    async _run(db, job, done) {
         this.logger.d(`Job "${this.jobName}" is starting with database:`, db?._cly_debug?.db);
+
         try {
-            this.run(db, this.doneCallback);
+            // Call run() and handle both Promise and callback patterns
+            const result = await new Promise((resolve, reject) => {
+                try {
+                    // Call the run method and capture its return value
+                    // If run() uses callback pattern, resolve/reject the promise when the callback is called
+                    const runResult = this.run(db, (error, callbackResult) => {
+                        if (error) {
+                            reject(error);
+                        }
+                        else {
+                            resolve(callbackResult);
+                        }
+                    });
+
+                    // If run() returns a Promise, handle it
+                    if (runResult instanceof Promise) {
+                        runResult.then(resolve).catch(reject);
+                    }
+                    // If run() returns a value directly
+                    else if (runResult !== undefined) {
+                        resolve(runResult);
+                    }
+                }
+                catch (error) {
+                    reject(error);
+                }
+            });
+
+            // Log success and call the job runner's callback
+            this.logger.i(`Job "${this.jobName}" completed successfully:`, result || '');
+            done(null, result);
         }
         catch (error) {
+            // Log error and call the job runner's callback with the error
             this.logger.e(`Job "${this.jobName}" encountered an error during execution:`, error);
-            this.doneCallback(error, this);
+            done(error);
         }
     }
 }

From 807434e3b63a1553883fe91170c43c1efdb963ff Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 14 Jan 2025 14:35:54 +0530
Subject: [PATCH 06/90] Job runner improvements

---
 jobServer/jobRunner/JobRunnerPulseImpl.js | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/jobServer/jobRunner/JobRunnerPulseImpl.js b/jobServer/jobRunner/JobRunnerPulseImpl.js
index c68b6afedd2..385e0355b65 100644
--- a/jobServer/jobRunner/JobRunnerPulseImpl.js
+++ b/jobServer/jobRunner/JobRunnerPulseImpl.js
@@ -25,6 +25,21 @@ class JobRunnerPulseImpl extends IJobRunner {
             ...config,
             mongo: this.db,
         });
+
+        // Monitor for progress
+        this.#pulseRunner.on('touch', (job) => {
+            console.debug(`PULSE_EVENT_LISTENER: Lock extended for job ${job?.attrs?.name}`);
+        });
+
+        // Monitor for failures
+        this.#pulseRunner.on('fail', (err, job) => {
+            console.error(`PULSE_EVENT_LISTENER: Job ${job?.attrs?.name} failed:`, err);
+        });
+
+        // Monitor for stalled jobs
+        this.#pulseRunner.on('stalled', (job) => {
+            console.warn(`PULSE_EVENT_LISTENER: Job ${job?.attrs?.name} has stalled`);
+        });
     }
 
     /**
@@ -45,7 +60,8 @@ class JobRunnerPulseImpl extends IJobRunner {
      */
     async createJob(jobName, JobClass) {
         const instance = new JobClass(jobName);
-        instance.setLogger(this.log);
+        // instance.setLogger(this.log);
+        instance.setJobName(jobName);
         // const schedule = instance.schedule();
 
         this.#pulseRunner.define(

From 05d3d4a3fd50228a43305ed77d550052914ee83d Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 14 Jan 2025 14:39:02 +0530
Subject: [PATCH 07/90] redundant log here, job baseclass already does this

---
 jobServer/jobRunner/JobRunnerPulseImpl.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/jobServer/jobRunner/JobRunnerPulseImpl.js b/jobServer/jobRunner/JobRunnerPulseImpl.js
index 385e0355b65..c9cd19d97fa 100644
--- a/jobServer/jobRunner/JobRunnerPulseImpl.js
+++ b/jobServer/jobRunner/JobRunnerPulseImpl.js
@@ -32,9 +32,9 @@ class JobRunnerPulseImpl extends IJobRunner {
         });
 
         // Monitor for failures
-        this.#pulseRunner.on('fail', (err, job) => {
-            console.error(`PULSE_EVENT_LISTENER: Job ${job?.attrs?.name} failed:`, err);
-        });
+        // this.#pulseRunner.on('fail', (err, job) => {
+        //     console.error(`PULSE_EVENT_LISTENER: Job ${job?.attrs?.name} failed:`, err);
+        // });
 
         // Monitor for stalled jobs
         this.#pulseRunner.on('stalled', (job) => {

From 3485290c2159410522cb105f8e61602c5ca3c546 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 14 Jan 2025 14:40:39 +0530
Subject: [PATCH 08/90] hard fix the pulse version, expecting changes in the
 development

---
 package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/package.json b/package.json
index e780bc67cbc..1484dba212c 100644
--- a/package.json
+++ b/package.json
@@ -32,7 +32,7 @@
   },
   "dependencies": {
     "@breejs/later": "^4.2.0",
-    "@pulsecron/pulse": "^1.6.7",
+    "@pulsecron/pulse": "1.6.7",
     "all-the-cities": "3.1.0",
     "argon2": "0.41.1",
     "async": "3.2.6",

From bdf6252fb76b36c485d099dd4a19fe55c021e30b Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 14 Jan 2025 14:48:01 +0530
Subject: [PATCH 09/90] define node engine in packagejson

---
 package.json | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/package.json b/package.json
index 1484dba212c..8625887ce9f 100644
--- a/package.json
+++ b/package.json
@@ -13,6 +13,9 @@
     "type": "git",
     "url": "git://github.com/countly/countly-server.git"
   },
+  "engines": {
+    "node": "^20.0.0 || ^22.0.0"
+  },
   "devDependencies": {
     "@stylistic/eslint-plugin": "^2.11.0",
     "docdash": "^2.0.1",

From d1eb6a58dbd72a1daeba898f1b16ccefbda52baf Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 14 Jan 2025 15:09:57 +0530
Subject: [PATCH 10/90] events dont exist :)

---
 jobServer/jobRunner/JobRunnerPulseImpl.js | 15 ---------------
 1 file changed, 15 deletions(-)

diff --git a/jobServer/jobRunner/JobRunnerPulseImpl.js b/jobServer/jobRunner/JobRunnerPulseImpl.js
index c9cd19d97fa..795ff699a00 100644
--- a/jobServer/jobRunner/JobRunnerPulseImpl.js
+++ b/jobServer/jobRunner/JobRunnerPulseImpl.js
@@ -25,21 +25,6 @@ class JobRunnerPulseImpl extends IJobRunner {
             ...config,
             mongo: this.db,
         });
-
-        // Monitor for progress
-        this.#pulseRunner.on('touch', (job) => {
-            console.debug(`PULSE_EVENT_LISTENER: Lock extended for job ${job?.attrs?.name}`);
-        });
-
-        // Monitor for failures
-        // this.#pulseRunner.on('fail', (err, job) => {
-        //     console.error(`PULSE_EVENT_LISTENER: Job ${job?.attrs?.name} failed:`, err);
-        // });
-
-        // Monitor for stalled jobs
-        this.#pulseRunner.on('stalled', (job) => {
-            console.warn(`PULSE_EVENT_LISTENER: Job ${job?.attrs?.name} has stalled`);
-        });
     }
 
     /**

From ed21279820c861693e5f7539e56ea4fa1d5c13f3 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 14 Jan 2025 15:28:15 +0530
Subject: [PATCH 11/90] remove obsolete

---
 jobServer/jobRunner/JobRunnerPulseImpl.js | 19 -------------------
 1 file changed, 19 deletions(-)

diff --git a/jobServer/jobRunner/JobRunnerPulseImpl.js b/jobServer/jobRunner/JobRunnerPulseImpl.js
index 795ff699a00..5b11542941d 100644
--- a/jobServer/jobRunner/JobRunnerPulseImpl.js
+++ b/jobServer/jobRunner/JobRunnerPulseImpl.js
@@ -64,25 +64,6 @@ class JobRunnerPulseImpl extends IJobRunner {
                 }
             }
         );
-        //
-        // const job = new PulseJob(
-        //     {
-        //         name: jobName,
-        //         pulse: this.#pulseRunner,
-        //     }
-        // );
-        //
-        // job.unique({'name': jobName});
-        //
-        // job.repeatEvery(
-        //     schedule,
-        //     {
-        //         timezone: 'Asia/Kolkata',
-        //     }
-        // );
-        //
-        // job.setShouldSaveResult(true);
-        // await job.save();
     }
 
 

From 3d4201dd0c08d92be8968c2e2c06c0a31d397cae Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 14 Jan 2025 22:17:19 +0530
Subject: [PATCH 12/90] support enable/disable of jobs externally

---
 jobServer/JobManager.js | 41 ++++++++++++++++++++++-------------------
 1 file changed, 22 insertions(+), 19 deletions(-)

diff --git a/jobServer/JobManager.js b/jobServer/JobManager.js
index 1ec76a463d1..f26ddacb233 100644
--- a/jobServer/JobManager.js
+++ b/jobServer/JobManager.js
@@ -57,10 +57,31 @@ class JobManager {
 
         await this.#loadJobs(jobClasses);
         await this.#jobRunner.start();
-        await this.#scheduleJobs(jobClasses);
         this.#log.d('JobManager started successfully');
     }
 
+    /**
+     * Enable a job
+     * @param {string} jobName Name of the job to enable
+     */
+    async enableJob(jobName) {
+        if (!this.#jobRunner) {
+            throw new Error('Job runner not initialized');
+        }
+        await this.#jobRunner.enableJob(jobName);
+    }
+
+    /**
+     * Disable a job
+     * @param {string} jobName Name of the job to disable
+     */
+    async disableJob(jobName) {
+        if (!this.#jobRunner) {
+            throw new Error('Job runner not initialized');
+        }
+        await this.#jobRunner.disableJob(jobName);
+    }
+
     /**
      * Loads the job classes into the job runner
      * @param {Object.<string, Function>} jobClasses Object containing job classes keyed by job name
@@ -75,24 +96,6 @@ class JobManager {
         );
     }
 
-    /**
-     * Schedules the jobs to run at their respective intervals
-     * @param { Object.<string, Function> } jobClasses Object containing job classes keyed by job name
-     * @returns {Promise<Awaited<void>[]>} A promise that resolves once the jobs are scheduled
-     */
-    #scheduleJobs(jobClasses) {
-        return Promise.all(
-            Object.entries(jobClasses)
-                .map(async([name, JobClass]) => {
-                    let instance = new JobClass(name);
-                    let schedule = await instance.schedule();
-                    if (schedule) {
-                        return this.#jobRunner.schedule(name, schedule);
-                    }
-                })
-        );
-    }
-
     /**
      * Closes the JobManager and cleans up resources
      * @returns {Promise<void>} A promise that resolves once cleanup is complete

From 24125d8e9d281eb38533e6e14edb534eb923f9ed Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 14 Jan 2025 22:20:16 +0530
Subject: [PATCH 13/90] self contained better job scheduling

---
 jobServer/jobRunner/IJobRunner.js         |  28 +++-
 jobServer/jobRunner/JobRunnerPulseImpl.js | 151 +++++++++++++++++-----
 2 files changed, 144 insertions(+), 35 deletions(-)

diff --git a/jobServer/jobRunner/IJobRunner.js b/jobServer/jobRunner/IJobRunner.js
index 86fd08daf59..258298000ae 100644
--- a/jobServer/jobRunner/IJobRunner.js
+++ b/jobServer/jobRunner/IJobRunner.js
@@ -32,13 +32,15 @@ class IJobRunner {
     }
 
     /**
-     * Schedules jobs
+     * Schedules a job based on its configuration
      * @param {String} name The name of the job
-     * @param {String} schedule Cron string for the job schedule
-     * @param {Object} data Data to pass to the job
+     * @param {Object} scheduleConfig Schedule configuration object
+     * @param {('once'|'schedule'|'now')} scheduleConfig.type Type of schedule
+     * @param {string|Date} [scheduleConfig.value] Cron string or Date object
+     * @param {Object} [data] Data to pass to the job
      * @returns {Promise<void>} A promise that resolves once the job is scheduled
      */
-    async schedule(/*name, schedule, data*/) {
+    async schedule(/*name, scheduleConfig, data*/) {
         throw new Error('Method not implemented');
     }
 
@@ -79,6 +81,24 @@ class IJobRunner {
     async close() {
         throw new Error('Method not implemented');
     }
+
+    /**
+     * Enable a job
+     * @param {string} jobName Name of the job to enable
+     * @returns {Promise<void>} A promise that resolves once the job is enabled
+     */
+    async enableJob(/*jobName*/) {
+        throw new Error('Method not implemented');
+    }
+
+    /**
+     * Disable a job
+     * @param {string} jobName Name of the job to disable
+     * @returns {Promise<void>} A promise that resolves once the job is disabled
+     */
+    async disableJob(/*jobName*/) {
+        throw new Error('Method not implemented');
+    }
 }
 
 module.exports = IJobRunner;
\ No newline at end of file
diff --git a/jobServer/jobRunner/JobRunnerPulseImpl.js b/jobServer/jobRunner/JobRunnerPulseImpl.js
index 5b11542941d..73d25c14610 100644
--- a/jobServer/jobRunner/JobRunnerPulseImpl.js
+++ b/jobServer/jobRunner/JobRunnerPulseImpl.js
@@ -12,6 +12,9 @@ class JobRunnerPulseImpl extends IJobRunner {
      */
     #pulseRunner;
 
+    /** @type {Map<string, Object>} Store job schedules until Pulse is started */
+    #pendingSchedules = new Map();
+
     /**
      * Creates a new Pulse job runner
      * @param {Object} db Database connection
@@ -28,57 +31,111 @@ class JobRunnerPulseImpl extends IJobRunner {
     }
 
     /**
-     * Starts the Pulse runner
+     * Starts the Pulse runner and schedules any pending jobs
      */
     async start() {
         if (!this.#pulseRunner) {
             throw new Error('Pulse runner not initialized');
         }
+
         await this.#pulseRunner.start();
+        this.log.i('Pulse runner started');
+
+        // Schedule all pending jobs
+        for (const [jobName, scheduleConfig] of this.#pendingSchedules) {
+            try {
+                await this.#scheduleJob(jobName, scheduleConfig);
+            }
+            catch (error) {
+                this.log.e(`Failed to schedule job ${jobName}:`, error);
+            }
+        }
+
+        this.#pendingSchedules.clear();
     }
 
     /**
-     * Creates a new job with the given class
+     * Creates and defines a new job
      * @param {string} jobName The name of the job
      * @param {Function} JobClass The job class to create
      * @returns {Promise<void>} A promise that resolves once the job is created
      */
     async createJob(jobName, JobClass) {
-        const instance = new JobClass(jobName);
-        // instance.setLogger(this.log);
-        instance.setJobName(jobName);
-        // const schedule = instance.schedule();
-
-        this.#pulseRunner.define(
-            jobName,
-            instance._run.bind(instance, this.db),
-            {
-                priority: JobPriority.normal,
-                concurrency: 1,
-                lockLifetime: 10000,
-                shouldSaveResult: true,
-                attempts: 3,
-                backoff: {
-                    type: 'exponential',
-                    delay: 2000
+        try {
+            const instance = new JobClass(jobName);
+            // instance.setLogger(this.log);
+            instance.setJobName(jobName);
+
+            this.#pulseRunner.define(
+                jobName,
+                async(job, done) => {
+                    instance._setTouchMethod(job.touch.bind(job));
+
+                    return instance._run(
+                        this.db,
+                        job,
+                        done
+                    );
+                },
+                {
+                    priority: JobPriority.normal,
+                    concurrency: 1,
+                    lockLifetime: 10000,
+                    shouldSaveResult: true,
+                    attempts: 3,
+                    backoff: {
+                        type: 'exponential',
+                        delay: 2000
+                    }
                 }
-            }
-        );
-    }
+            );
 
+            // Store schedule configuration for later
+            const scheduleConfig = instance.getSchedule();
+            this.#pendingSchedules.set(jobName, scheduleConfig);
+            this.log.d(`Job ${jobName} defined successfully`);
+        }
+        catch (error) {
+            this.log.e(`Failed to create job ${jobName}:`, error);
+            // Don't throw - allow other jobs to continue
+        }
+    }
 
     /**
-     * Schedules jobs
-     * @param {String} name The name of the job
-     * @param {String} schedule Cron string for the job schedule
-     * @param {Object} data Data to pass to the job
-     * @returns {Promise<void>} A promise that resolves once the job is scheduled
+     * Internal method to schedule a job
+     * @param {string} name The name of the job to schedule
+     * @param {Object} scheduleConfig Schedule configuration object
+     * @param {('once'|'schedule'|'now')} scheduleConfig.type Type of schedule
+     * @param {string|Date} [scheduleConfig.value] Cron string or Date object
+     * @param {Object} [data] Data to pass to the job
+     * @private
      */
-    async schedule(name, schedule, data) {
-        if (!isValidCron(schedule)) {
-            throw new Error('Invalid cron schedule');
+    async #scheduleJob(name, scheduleConfig, data) {
+        switch (scheduleConfig.type) {
+        case 'schedule':
+            if (!isValidCron(scheduleConfig.value)) {
+                throw new Error('Invalid cron schedule');
+            }
+            await this.#pulseRunner.every(scheduleConfig.value, name, data);
+            this.log.d(`Job ${name} scheduled with cron: ${scheduleConfig.value}`);
+            break;
+
+        case 'once':
+            if (!(scheduleConfig.value instanceof Date)) {
+                throw new Error('Invalid date for one-time schedule');
+            }
+            await this.#pulseRunner.schedule(scheduleConfig.value, name, data);
+            this.log.d(`Job ${name} scheduled for: ${scheduleConfig.value}`);
+            break;
+
+        case 'now':
+            await this.#pulseRunner.now(name, data);
+            this.log.d(`Job ${name} scheduled to run immediately`);
+            break;
+
+        default:
+            throw new Error(`Invalid schedule type: ${scheduleConfig.type}`);
         }
-        await this.#pulseRunner.every(schedule, name, data);
     }
 
     /**
@@ -91,6 +148,38 @@ class JobRunnerPulseImpl extends IJobRunner {
             this.#pulseRunner = null;
         }
     }
+
+    /**
+     * Enable a job
+     * @param {string} jobName Name of the job to enable
+     * @returns {Promise<void>} A promise that resolves once the job is enabled
+     */
+    async enableJob(jobName) {
+        try {
+            await this.#pulseRunner.enable({ name: jobName });
+            this.log.i(`Job ${jobName} enabled`);
+        }
+        catch (error) {
+            this.log.e(`Failed to enable job ${jobName}:`, error);
+            throw error;
+        }
+    }
+
+    /**
+     * Disable a job
+     * @param {string} jobName Name of the job to disable
+     * @returns {Promise<void>} A promise that resolves once the job is disabled
+     */
+    async disableJob(jobName) {
+        try {
+            await this.#pulseRunner.disable({ name: jobName });
+            this.log.i(`Job ${jobName} disabled`);
+        }
+        catch (error) {
+            this.log.e(`Failed to disable job ${jobName}:`, error);
+            throw error;
+        }
+    }
 }
 
 module.exports = JobRunnerPulseImpl;
\ No newline at end of file

From 967fa249669f2ca1741c585e378993cc725f727b Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 14 Jan 2025 22:24:09 +0530
Subject: [PATCH 14/90] Expanded job baseclass with documented example

---
 jobServer/Job.js                | 224 +++++++++++++++++++++++++++-----
 jobServer/example/ExampleJob.js | 110 ++++++++++++++++
 2 files changed, 302 insertions(+), 32 deletions(-)
 create mode 100644 jobServer/example/ExampleJob.js

diff --git a/jobServer/Job.js b/jobServer/Job.js
index e9b7c63ddda..49f5a57fcb8 100644
--- a/jobServer/Job.js
+++ b/jobServer/Job.js
@@ -1,28 +1,84 @@
+const defaultLogger = {
+    d: console.debug,
+    w: console.warn,
+    e: console.error,
+    i: console.info
+};
 /**
- * Represents a job.
+ * Base class for creating jobs.
+ * 
+ * @example
+ * // Example of a simple job that runs every minute
+ * class SimpleJob extends Job {
+ *     getSchedule() {
+ *         return {
+ *             type: 'schedule',
+ *             value: '* * * * *' // Runs every minute
+ *         };
+ *     }
+ * 
+ *     async run(db, done) {
+ *         try {
+ *             await db.collection('mycollection').updateMany({}, { $set: { updated: new Date() } });
+ *             done(null, 'Successfully updated records');
+ *         } catch (error) {
+ *             done(error);
+ *         }
+ *     }
+ * }
+ * 
+ * @example
+ * // Example of a one-time job scheduled for a specific date
+ * class OneTimeJob extends Job {
+ *     getSchedule() {
+ *         return {
+ *             type: 'once',
+ *             value: new Date('2024-12-31T23:59:59Z')
+ *         };
+ *     }
+ * 
+ *     async run(db, done) {
+ *         // Your job logic here
+ *         done();
+ *     }
+ * }
+ * 
+ * @example
+ * // Example of a job that runs immediately
+ * class ImmediateJob extends Job {
+ *     getSchedule() {
+ *         return {
+ *             type: 'now'
+ *         };
+ *     }
+ * 
+ *     async run(db, done) {
+ *         // Your job logic here
+ *         done();
+ *     }
+ * }
  */
 class Job {
-
+    /** @type {string} Name of the job */
     jobName = Job.name;
 
-    logger = {
-        d: console.debug,
-        w: console.warn,
-        e: console.error,
-        i: console.info
-    };
+    /** @type {Object} Logger instance */
+    logger;
+
+    /** @type {Function|null} Touch method from Pulse */
+    _touchMethod = null;
 
     /**
      * Creates an instance of Job.
      */
     constructor() {
-        this.logger.d(`Job instance"${this.jobName}" created`);
+        this.logger = defaultLogger;
+        this.logger.d(`Job instance "${this.jobName}" created`);
     }
 
     /**
      * Sets the name of the job.
      * @param {String} name The name of the job
-     * @returns {void} The name of the job
      */
     setJobName(name) {
         this.jobName = name;
@@ -30,49 +86,108 @@ class Job {
 
     /**
      * Sets the logger
-     * @param {Logger} logger The logger
+     * @param {Object} [logger=defaultLogger] The logger instance
      */
-    setLogger(logger) {
+    setLogger(logger = defaultLogger) {
         this.logger = logger;
     }
 
     /**
-     * Runs the job.
-     * @param {Object} db The database connection
-     * @param {Function} done The callback function to be called when the job is done
-     * @throws {Error} If the method is not overridden
+     * Runs the job. This method must be implemented by the child class.
+     * 
+     * @param {Db} db The database connection
+     * @param {Function} done Callback function to be called when the job is complete
+     *                       Call with error as first parameter if job fails
+     *                       Call with null and optional result as second parameter if job succeeds
+     * @param {Function} progress Progress reporting function for long-running jobs
+     *                           Call with (total, current, bookmark) to report progress
+     * 
+     * @example
+     * // Example implementation with progress reporting
+     * async run(db, done, progress) {
+     *     try {
+     *         const total = await db.collection('users').countDocuments({ active: false });
+     *         let processed = 0;
+     *         
+     *         const cursor = db.collection('users').find({ active: false });
+     *         while (await cursor.hasNext()) {
+     *             await cursor.next();
+     *             processed++;
+     *             if (processed % 100 === 0) {
+     *                 await progress(total, processed, `Processing inactive users`);
+     *             }
+     *         }
+     *         done(null, { processedCount: processed });
+     *     } catch (error) {
+     *         done(error);
+     *     }
+     * }
+     * 
      * @abstract
+     * @throws {Error} If the method is not overridden
      */
-    async run(/*db, done*/) {
+    async run(/*db, done, progress*/) {
         throw new Error('Job must be overridden');
     }
 
     /**
-     * Get the schedule for the job in cron format.
-     * @returns {string} The cron schedule
-     * @throws {Error} If the method is not overridden
+     * Get the schedule type and timing for the job.
+     * This method must be implemented by the child class.
+     * 
+     * @returns {Object} Schedule configuration object
+     * @property {('once'|'schedule'|'now')} type - Type of schedule
+     * @property {string|Date} [value] - Schedule value:
+     *   - For type='schedule': Cron expression (e.g., '0 * * * *' for hourly)
+     *   - For type='once': Date object for when to run
+     *   - For type='now': Not needed
+     * 
+     * @example
+     * // Run every day at midnight
+     * getSchedule() {
+     *     return {
+     *         type: 'schedule',
+     *         value: '0 0 * * *'
+     *     };
+     * }
+     * 
+     * @example
+     * // Run once at a specific time
+     * getSchedule() {
+     *     return {
+     *         type: 'once',
+     *         value: new Date('2024-01-01T00:00:00Z')
+     *     };
+     * }
+     * 
+     * @example
+     * // Run immediately
+     * getSchedule() {
+     *     return {
+     *         type: 'now'
+     *     };
+     * }
+     * 
      * @abstract
+     * @throws {Error} If the method is not overridden
      */
-    schedule() {
-        throw new Error('schedule must be overridden');
+    getSchedule() {
+        throw new Error('getSchedule must be overridden');
     }
 
     /**
-     * Runs the job and handles both Promise and callback patterns.
-     * @param {Object} db The database
+     * Internal method to run the job and handle both Promise and callback patterns.
+     * @param {Db} db The database connection
      * @param {Object} job The job instance
      * @param {Function} done Callback to be called when job completes
+     * @returns {Promise<void>} A promise that resolves once the job is completed
      * @private
      */
     async _run(db, job, done) {
         this.logger.d(`Job "${this.jobName}" is starting with database:`, db?._cly_debug?.db);
 
         try {
-            // Call run() and handle both Promise and callback patterns
             const result = await new Promise((resolve, reject) => {
                 try {
-                    // Call the run method and capture its return value
-                    // If run() uses callback pattern, resolve/reject the promise when the callback is called
                     const runResult = this.run(db, (error, callbackResult) => {
                         if (error) {
                             reject(error);
@@ -80,13 +195,11 @@ class Job {
                         else {
                             resolve(callbackResult);
                         }
-                    });
+                    }, this.reportProgress.bind(this));
 
-                    // If run() returns a Promise, handle it
                     if (runResult instanceof Promise) {
                         runResult.then(resolve).catch(reject);
                     }
-                    // If run() returns a value directly
                     else if (runResult !== undefined) {
                         resolve(runResult);
                     }
@@ -96,16 +209,63 @@ class Job {
                 }
             });
 
-            // Log success and call the job runner's callback
             this.logger.i(`Job "${this.jobName}" completed successfully:`, result || '');
             done(null, result);
         }
         catch (error) {
-            // Log error and call the job runner's callback with the error
             this.logger.e(`Job "${this.jobName}" encountered an error during execution:`, error);
             done(error);
         }
     }
+
+    /**
+     * Sets the touch method (called internally by job runner)
+     * @param {Function} touchMethod The touch method from Pulse
+     * @private
+     */
+    _setTouchMethod(touchMethod) {
+        this._touchMethod = touchMethod;
+    }
+
+    /**
+     * Reports progress for long-running jobs to prevent lock expiration
+     * @param {number} [total] Total number of stages
+     * @param {number} [current] Current stage number
+     * @param {string} [bookmark] Bookmark string for current stage
+     */
+    async reportProgress(total, current, bookmark) {
+        if (!this._touchMethod) {
+            throw new Error('Touch method not available');
+        }
+
+        // Calculate progress percentage
+        const progress = total && current ? Math.min(100, Math.floor((current / total) * 100)) : undefined;
+
+        // Store progress data
+        const progressData = {
+            total,
+            current,
+            bookmark,
+            progress,
+            timestamp: new Date()
+        };
+
+        // Store in job data
+        if (!this.data) {
+            this.data = {};
+        }
+        this.data.progress = progressData;
+
+        // Call Pulse's touch method with progress
+        if (progress !== undefined) {
+            await this._touchMethod(progress);
+        }
+        else {
+            await this._touchMethod();
+        }
+
+        this.logger?.d(`Progress reported for job "${this.jobName}":`, progressData);
+    }
 }
 
 module.exports = Job;
\ No newline at end of file
diff --git a/jobServer/example/ExampleJob.js b/jobServer/example/ExampleJob.js
new file mode 100644
index 00000000000..a05b0da6f7c
--- /dev/null
+++ b/jobServer/example/ExampleJob.js
@@ -0,0 +1,110 @@
+const job = require("../../jobServer");
+
+/**
+ * Example job implementation demonstrating all features of the job system.
+ * This job processes user records in batches, demonstrating:
+ * - Progress reporting
+ * - Error handling
+ * - Different schedule types
+ * - Database operations
+ * - Proper logging
+ * 
+ * @extends {job.Job}
+ */
+class ExampleJob extends job.Job {
+    /**
+     * Get the schedule configuration for the job.
+     * Demonstrates all possible schedule types.
+     * 
+     * @returns {Object} Schedule configuration object
+     */
+    getSchedule() {
+        // Example 1: Run every day at midnight
+        return {
+            type: 'schedule',
+            value: '0 0 * * *'
+        };
+
+        // Example 2: Run once at a specific future date
+        // return {
+        //     type: 'once',
+        //     value: new Date('2024-12-31T23:59:59Z')
+        // };
+
+        // Example 3: Run immediately
+        // return {
+        //     type: 'now'
+        // };
+    }
+
+    /**
+     * Simulates some processing work
+     * @private
+     * @param {number} ms Time to wait in milliseconds
+     */
+    async #simulateWork(ms) {
+        await new Promise(resolve => setTimeout(resolve, ms));
+    }
+
+    /**
+     * Main job execution method.
+     * Demonstrates:
+     * - Progress reporting
+     * - Synthetic delays to simulate work
+     * - Error handling
+     * - Logging
+     * 
+     * @param {Object} db Database connection
+     * @param {Function} done Callback to signal job completion
+     * @param {Function} progress Progress reporting function
+     */
+    async run(db, done, progress) {
+        try {
+            this.logger.d("Starting example job execution");
+
+            // Simulate total items to process
+            const total = 100;
+            let processed = 0;
+
+            // Process in batches of 10
+            for (let i = 0; i < total; i += 10) {
+                // Simulate batch processing (2 second per batch)
+                await this.#simulateWork(2000);
+                processed += 10;
+
+                // Report progress
+                await progress(
+                    total,
+                    processed,
+                    `Processing batch ${(i / 10) + 1}/10`
+                );
+
+                this.logger.d(`Completed batch ${(i / 10) + 1}/10`);
+
+                // Simulate random error (10% chance)
+                if (Math.random() < 0.1) {
+                    throw new Error('Random batch processing error');
+                }
+            }
+
+            // Simulate final processing
+            await this.#simulateWork(1000);
+
+            // Job completion
+            const result = {
+                processedCount: processed,
+                totalItems: total,
+                completedAt: new Date()
+            };
+
+            this.logger.i("Job completed successfully", result);
+            done(null, result);
+        }
+        catch (error) {
+            this.logger.e("Job failed:", error);
+            done(error);
+        }
+    }
+}
+
+module.exports = ExampleJob;
\ No newline at end of file

From 7b4dc94d1f5467263c260700dde41ce0ac6d06b3 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 14 Jan 2025 22:24:56 +0530
Subject: [PATCH 15/90] JobServer with oplog watcher to communicate with main
 countly process

---
 api/config.sample.js   |  1 +
 jobServer/JobServer.js | 64 ++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 65 insertions(+)

diff --git a/api/config.sample.js b/api/config.sample.js
index d3532c70d9f..44c548ad4d4 100644
--- a/api/config.sample.js
+++ b/api/config.sample.js
@@ -24,6 +24,7 @@ var countlyConfig = {
         db: "countly",
         port: 27017,
         max_pool_size: 500,
+        replicaName: "rs0",
         //username: test,
         //password: test,
         //mongos: false,
diff --git a/jobServer/JobServer.js b/jobServer/JobServer.js
index a2441896944..37a691cbc31 100644
--- a/jobServer/JobServer.js
+++ b/jobServer/JobServer.js
@@ -1,6 +1,7 @@
 const JobManager = require('./JobManager');
 const JobScanner = require('./JobScanner');
 
+const JOBS_CONFIG_COLLECTION = 'jobConfigs';
 /**
  * Class representing a job process.
  */
@@ -53,6 +54,20 @@ class JobServer {
      */
     #db = null;
 
+    /**
+     * Collection for job configurations
+     * @private
+     * @type {import('mongodb').Collection}
+     */
+    #jobConfigsCollection;
+
+    /**
+     * Flag indicating whether the job process is shutting down
+     * avoids multiple shutdown race conditions
+     * @private
+     */
+    #isShuttingDown = false;
+
     /**
      * Creates a new JobProcess instance.
      * @param {Object} common Countly common
@@ -90,7 +105,13 @@ class JobServer {
             this.#jobManager = new JobManager(this.#db, this.Logger);
             this.#jobScanner = new JobScanner(this.#db, this.Logger, this.#pluginManager);
 
+            this.#jobConfigsCollection = this.#db.collection(JOBS_CONFIG_COLLECTION);
+            // await this.#jobConfigsCollection.createIndex({ jobName: 1 }, /*{ unique: true }*/);
+
             this.#setupSignalHandlers();
+            // Watch for changes in job configurations
+            this.#watchJobConfigs();
+
             this.#log.i('Job process init successfully');
         }
         catch (error) {
@@ -155,12 +176,55 @@ class JobServer {
         });
     }
 
+    /**
+     * Watch for changes in job configurations
+     * @private
+     */
+    async #watchJobConfigs() {
+        const changeStream = this.#jobConfigsCollection.watch();
+
+        changeStream.on('change', async(change) => {
+            try {
+                if (change.operationType === 'update' || change.operationType === 'insert') {
+                    const jobName = change.fullDocument.jobName;
+                    const enabled = change.fullDocument.enabled;
+
+                    if (enabled) {
+                        await this.#jobManager.enableJob(jobName);
+                    }
+                    else {
+                        await this.#jobManager.disableJob(jobName);
+                    }
+
+                    this.#log.i(`Job ${jobName} ${enabled ? 'enabled' : 'disabled'}`);
+                }
+            }
+            catch (error) {
+                this.#log.e('Error processing job config change:', error);
+            }
+        });
+
+        changeStream.on('error', (error) => {
+            this.#log.e('Error in job configs change stream:', error);
+            // Implement reconnection logic here
+        });
+    }
+
     /**
      * Shuts down the job process.
      * @param {number} [exitCode=0] - The exit code to use when shutting down the process.
      * @returns {Promise<void>} A promise that resolves once the job process is shut down.
      */
     async #shutdown(exitCode = 0) {
+        if (this.#isShuttingDown) {
+            return;
+        }
+        this.#isShuttingDown = true;
+
+        if (this.#db && typeof this.#db.close === 'function') {
+            await this.#db.close();
+        }
+
         if (!this.#isRunning) {
             this.#log.w('Shutdown called but process is not running');
             process.exit(exitCode);

From 372ee079f7259b33fa921c6a6ef6b5777558a857 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 14 Jan 2025 23:42:23 +0530
Subject: [PATCH 16/90] progress job

---
 jobServer/Job.js                          | 58 +++++++++++++----------
 jobServer/jobRunner/JobRunnerPulseImpl.js | 14 ++++++
 2 files changed, 47 insertions(+), 25 deletions(-)

diff --git a/jobServer/Job.js b/jobServer/Job.js
index 49f5a57fcb8..d429a5a366f 100644
--- a/jobServer/Job.js
+++ b/jobServer/Job.js
@@ -65,9 +65,12 @@ class Job {
     /** @type {Object} Logger instance */
     logger;
 
-    /** @type {Function|null} Touch method from Pulse */
+    /** @type {Function|null} Touch method from job runner */
     _touchMethod = null;
 
+    /** @type {Function|null} Progress method from job runner */
+    _progressMethod = null;
+
     /**
      * Creates an instance of Job.
      */
@@ -188,14 +191,18 @@ class Job {
         try {
             const result = await new Promise((resolve, reject) => {
                 try {
-                    const runResult = this.run(db, (error, callbackResult) => {
-                        if (error) {
-                            reject(error);
-                        }
-                        else {
-                            resolve(callbackResult);
-                        }
-                    }, this.reportProgress.bind(this));
+                    const runResult = this.run(
+                        db,
+                        (error, callbackResult) => {
+                            if (error) {
+                                reject(error);
+                            }
+                            else {
+                                resolve(callbackResult);
+                            }
+                        },
+                        this.reportProgress.bind(this)
+                    );
 
                     if (runResult instanceof Promise) {
                         runResult.then(resolve).catch(reject);
@@ -228,39 +235,40 @@ class Job {
     }
 
     /**
-     * Reports progress for long-running jobs to prevent lock expiration
+     * Sets the progress method from the runner
+     * @param {Function} progressMethod Method to update progress
+     * @protected
+     */
+    _setProgressMethod(progressMethod) {
+        this._progressMethod = progressMethod;
+    }
+
+    /**
+     * Reports progress for long-running jobs
      * @param {number} [total] Total number of stages
      * @param {number} [current] Current stage number
      * @param {string} [bookmark] Bookmark string for current stage
      */
     async reportProgress(total, current, bookmark) {
-        if (!this._touchMethod) {
-            throw new Error('Touch method not available');
-        }
-
         // Calculate progress percentage
         const progress = total && current ? Math.min(100, Math.floor((current / total) * 100)) : undefined;
 
-        // Store progress data
+        // Build progress data
         const progressData = {
             total,
             current,
             bookmark,
-            progress,
+            percent: progress,
             timestamp: new Date()
         };
 
-        // Store in job data
-        if (!this.data) {
-            this.data = {};
+        // Update progress using runner's method if available
+        if (this._progressMethod) {
+            await this._progressMethod(progressData);
         }
-        this.data.progress = progressData;
 
-        // Call Pulse's touch method with progress
-        if (progress !== undefined) {
-            await this._touchMethod(progress);
-        }
-        else {
+        // Touch to prevent lock expiration
+        if (this._touchMethod) {
             await this._touchMethod();
         }
 
diff --git a/jobServer/jobRunner/JobRunnerPulseImpl.js b/jobServer/jobRunner/JobRunnerPulseImpl.js
index 73d25c14610..6b77e8362f2 100644
--- a/jobServer/jobRunner/JobRunnerPulseImpl.js
+++ b/jobServer/jobRunner/JobRunnerPulseImpl.js
@@ -54,6 +54,17 @@ class JobRunnerPulseImpl extends IJobRunner {
         this.#pendingSchedules.clear();
     }
 
+    /**
+     * Updates job progress in Pulse
+     * @param {Object} job Pulse job instance
+     * @param {Object} progressData Progress data to store
+     * @private
+     */
+    async #updateJobProgress(job, progressData) {
+        job.data = progressData;
+        await job.save();
+    }
+
     /**
      * Creates and defines a new job
      * @param {string} jobName The name of the job
@@ -70,6 +81,9 @@ class JobRunnerPulseImpl extends IJobRunner {
                 jobName,
                 async(job, done) => {
                     instance._setTouchMethod(job.touch.bind(job));
+                    instance._setProgressMethod(
+                        async(progressData) => this.#updateJobProgress(job, progressData)
+                    );
 
                     return instance._run(
                         this.db,

From 480cbc4e11fd6a898e07931b87cca84d4b668a6b Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Wed, 15 Jan 2025 11:02:28 +0530
Subject: [PATCH 17/90] codacy fix

---
 jobServer/JobUtils.js | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/jobServer/JobUtils.js b/jobServer/JobUtils.js
index 2b69927ae7b..1a4df6a65cf 100644
--- a/jobServer/JobUtils.js
+++ b/jobServer/JobUtils.js
@@ -30,6 +30,8 @@ class JobUtils {
     }
 
     /**
+     * @note
+     * We shouldn't need this and use cron string directly in job schedule to avoid conversion
      * Converts a later.js schedule to a cron string.
      * @param {String} laterString - The later.js schedule string.
      * @constructor
@@ -132,12 +134,12 @@ class JobUtils {
         }
 
         if (isEveryMinute) {
-            const interval = parseInt(isEveryMinute[1]);
+            const interval = parseInt(isEveryMinute[1], 10);
             return `*/${interval} * * * *`;
         }
 
         if (isEveryHour) {
-            const interval = parseInt(isEveryHour[1]);
+            const interval = parseInt(isEveryHour[1], 10);
             return laterString.includes('on the 1st min')
                 ? `1 */${interval} * * *`
                 : `0 */${interval} * * *`;
@@ -145,8 +147,8 @@ class JobUtils {
 
         if (isSpecificTime && isEveryDay) {
             let [, hour, minute, meridiem] = isSpecificTime;
-            hour = parseInt(hour);
-            minute = parseInt(minute);
+            hour = parseInt(hour, 10);
+            minute = parseInt(minute, 10);
             if (meridiem) {
                 hour = meridiem.toLowerCase() === 'pm' && hour < 12 ? hour + 12 : hour;
                 hour = meridiem.toLowerCase() === 'am' && hour === 12 ? 0 : hour;
@@ -157,8 +159,8 @@ class JobUtils {
         if (isWeekday) {
             if (isSpecificTime) {
                 let [, hour, minute, meridiem] = isSpecificTime;
-                hour = parseInt(hour);
-                minute = parseInt(minute);
+                hour = parseInt(hour, 10);
+                minute = parseInt(minute, 10);
                 if (meridiem) {
                     hour = meridiem.toLowerCase() === 'pm' && hour < 12 ? hour + 12 : hour;
                     hour = meridiem.toLowerCase() === 'am' && hour === 12 ? 0 : hour;
@@ -169,11 +171,11 @@ class JobUtils {
         }
 
         if (isEveryNDays) {
-            const interval = parseInt(isEveryNDays[1]);
+            const interval = parseInt(isEveryNDays[1], 10);
             if (isSpecificTime) {
                 let [, hour, minute, meridiem] = isSpecificTime;
-                hour = parseInt(hour);
-                minute = parseInt(minute);
+                hour = parseInt(hour, 10);
+                minute = parseInt(minute, 10);
                 if (meridiem) {
                     hour = meridiem.toLowerCase() === 'pm' && hour < 12 ? hour + 12 : hour;
                     hour = meridiem.toLowerCase() === 'am' && hour === 12 ? 0 : hour;
@@ -186,8 +188,8 @@ class JobUtils {
         if (isLastDayOfMonth) {
             if (isSpecificTime) {
                 let [, hour, minute, meridiem] = isSpecificTime;
-                hour = parseInt(hour);
-                minute = parseInt(minute);
+                hour = parseInt(hour, 10);
+                minute = parseInt(minute, 10);
                 if (meridiem) {
                     hour = meridiem.toLowerCase() === 'pm' && hour < 12 ? hour + 12 : hour;
                     hour = meridiem.toLowerCase() === 'am' && hour === 12 ? 0 : hour;
@@ -198,7 +200,7 @@ class JobUtils {
         }
 
         if (isOnceInHours) {
-            const interval = parseInt(isOnceInHours[1]);
+            const interval = parseInt(isOnceInHours[1], 10);
             return `0 */${interval} * * *`;
         }
 

From ef82822ed97905feb4a34e777326a32d76e648e4 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Wed, 15 Jan 2025 13:44:17 +0530
Subject: [PATCH 18/90] complex job operations from oplog

---
 jobServer/Job.js                          |  41 +++++
 jobServer/JobManager.js                   | 213 ++++++++++++++++++++--
 jobServer/JobServer.js                    |  36 ----
 jobServer/JobUtils.js                     |  14 ++
 jobServer/jobRunner/IJobRunner.js         |  41 +++++
 jobServer/jobRunner/JobRunnerPulseImpl.js | 109 +++++++++--
 6 files changed, 389 insertions(+), 65 deletions(-)

diff --git a/jobServer/Job.js b/jobServer/Job.js
index d429a5a366f..c1c47212aa5 100644
--- a/jobServer/Job.js
+++ b/jobServer/Job.js
@@ -4,6 +4,7 @@ const defaultLogger = {
     e: console.error,
     i: console.info
 };
+const { JOB_PRIORITIES } = require('./constants/JobPriorities');
 /**
  * Base class for creating jobs.
  * 
@@ -274,6 +275,46 @@ class Job {
 
         this.logger?.d(`Progress reported for job "${this.jobName}":`, progressData);
     }
+
+    /**
+     * Get job retry configuration
+     * @typedef {Object} RetryConfig
+     * @property {boolean} enabled - Whether retries are enabled
+     * @property {number} attempts - Number of retry attempts
+     * @property {number} delay - Delay between retry attempts in milliseconds, delay is by default exponentially increasing after each attempt
+     * @returns {RetryConfig|null} Retry configuration or null for default
+     */
+    getRetryConfig() {
+        return {
+            enabled: true,
+            attempts: 3,
+            delay: 2000 // 2 seconds
+        };
+    }
+
+    /**
+     * Get job priority
+     * @returns {string} Priority level from JOB_PRIORITIES
+     */
+    getPriority() {
+        return JOB_PRIORITIES.NORMAL;
+    }
+
+    /**
+     * Get job concurrency
+     * @returns {number|null} Maximum concurrent instances or null for default
+     */
+    getConcurrency() {
+        return 1;
+    }
+
+    /**
+     * Get job lock lifetime in milliseconds
+     * @returns {number|null} Lock lifetime or null for default
+     */
+    getLockLifetime() {
+        return 55 * 60 * 1000; // 55 minutes
+    }
 }
 
 module.exports = Job;
\ No newline at end of file
diff --git a/jobServer/JobManager.js b/jobServer/JobManager.js
index f26ddacb233..8e2104b9229 100644
--- a/jobServer/JobManager.js
+++ b/jobServer/JobManager.js
@@ -1,5 +1,6 @@
-const {RUNNER_TYPES, createJobRunner} = require('./JobRunner');
+const {RUNNER_TYPES, createJobRunner} = require('./JobRunner/index');
 const config = require("./config");
+const JobUtils = require('./JobUtils');
 
 /**
  * Manages job configurations and initialization.
@@ -25,6 +26,9 @@ class JobManager {
      */
     #jobRunner = null;
 
+
+    #jobConfigsCollection;
+
     /**
      * Creates a new JobManager instance
      * @param {Object} db Database connection
@@ -40,6 +44,199 @@ class JobManager {
         const pulseConfig = config.PULSE;
 
         this.#jobRunner = createJobRunner(this.#db, runnerType, pulseConfig, Logger);
+        this.#jobConfigsCollection = db.collection('jobConfigs');
+        this.#watchConfigs();
+    }
+
+    /**
+     * Watches for changes in job configurations
+     * @private
+     * @returns {Promise<void>} A promise that resolves once the watcher is started
+     */
+    async #watchConfigs() {
+        const changeStream = this.#jobConfigsCollection.watch();
+
+        changeStream.on('change', async(change) => {
+            if (change.operationType === 'update' || change.operationType === 'insert') {
+                const jobConfig = change.fullDocument;
+                await this.#applyConfig(jobConfig);
+            }
+        });
+    }
+
+    /**
+     * Applies job configuration changes
+     * @private
+     * @param {Object} jobConfig The job configuration to apply
+     * @param {string} jobConfig.jobName The name of the job
+     * @param {boolean} [jobConfig.runNow] Whether to run the job immediately
+     * @param {Object} [jobConfig.schedule] Schedule configuration
+     * @param {Object} [jobConfig.retry] Retry configuration
+     * @param {boolean} [jobConfig.enabled] Whether the job is enabled
+     * @returns {Promise<void>} A promise that resolves once the configuration is applied
+     */
+    async #applyConfig(jobConfig) {
+        try {
+            if (!this.#jobRunner) {
+                throw new Error('Job runner not initialized');
+            }
+
+            const { jobName } = jobConfig;
+
+            if (jobConfig.runNow === true) {
+                await this.#jobRunner.runJobNow(jobName);
+                await this.#jobConfigsCollection.updateOne(
+                    { jobName },
+                    { $unset: { runNow: "" } }
+                );
+            }
+
+            if (jobConfig.schedule) {
+                await this.#jobRunner.updateSchedule(jobName, jobConfig.schedule);
+            }
+
+            if (jobConfig.retry) {
+                await this.#jobRunner.configureRetry(jobName, jobConfig.retry);
+            }
+
+            if (typeof jobConfig.enabled === 'boolean') {
+                if (jobConfig.enabled) {
+                    await this.#jobRunner.enableJob(jobName);
+                    this.#log.i(`Job ${jobName} enabled via config`);
+                }
+                else {
+                    await this.#jobRunner.disableJob(jobName);
+                    this.#log.i(`Job ${jobName} disabled via config`);
+                }
+            }
+        }
+        catch (error) {
+            this.#log.e('Error applying job configuration:', error);
+        }
+    }
+
+    /**
+     * Loads job classes and manages their configurations
+     * @private
+     * @param {Object.<string, Function>} jobClasses - Object containing job class implementations keyed by job name
+     * @returns {Promise<void>} A promise that resolves once all jobs are loaded and configured
+     * @throws {Error} If job loading or configuration fails
+     */
+    async #loadJobs(jobClasses) {
+        // Calculate checksums for all job definitions
+        const jobDefinitionChecksums = Object.entries(jobClasses).reduce((acc, [name, JobClass]) => {
+            acc[name] = JobUtils.calculateJobChecksum(JobClass);
+            return acc;
+        }, {});
+
+        // Initialize or update job configurations
+        await this.#initializeJobConfigs(jobClasses, jobDefinitionChecksums);
+
+        // Load and apply configurations
+        await this.#applyJobConfigurations(jobClasses, jobDefinitionChecksums);
+    }
+
+    /**
+     * Initializes or updates job configurations in the database
+     * @private
+     * @param {Object.<string, Function>} jobClasses - Job class implementations
+     * @param {Object.<string, string>} jobDefinitionChecksums - Checksums of job definitions
+     */
+    async #initializeJobConfigs(jobClasses, jobDefinitionChecksums) {
+        await Promise.all(
+            Object.entries(jobClasses).map(async([jobName, JobClass ]) => {
+                this.#log.d(`Initializing job config for ${jobName}`);
+                const currentChecksum = jobDefinitionChecksums[jobName];
+                const existingConfigOverride = await this.#jobConfigsCollection.findOne({ jobName });
+
+                if (!existingConfigOverride) {
+                    // Create new configuration for new job
+                    await this.#createDefaultJobConfig(jobName, currentChecksum, JobClass);
+                }
+                else if (existingConfigOverride.checksum !== currentChecksum) {
+                    // Reset configuration if job implementation has changed
+                    await this.#resetJobConfig(jobName, currentChecksum);
+                }
+            })
+        );
+    }
+
+    /**
+     * Creates a default configuration for a new job
+     * @private
+     * @param {string} jobName - Name of the job
+     * @param {string} checksum - Checksum of the job definition
+     * @param {Function} JobClass - Job class implementation
+     */
+    async #createDefaultJobConfig(jobName, checksum, JobClass) {
+        const instance = new JobClass(jobName);
+
+        await this.#jobConfigsCollection.insertOne({
+            jobName,
+            enabled: true,
+            checksum,
+            createdAt: new Date(),
+            defaultConfig: {
+                schedule: instance.getSchedule(),
+                retry: instance.getRetryConfig(),
+                priority: instance.getPriority(),
+                concurrency: instance.getConcurrency(),
+                lockLifetime: instance.getLockLifetime()
+            }
+        });
+        this.#log.d(`Created default configuration for new job: ${jobName}`);
+    }
+
+    /**
+     * Resets job configuration when implementation changes
+     * @private
+     * @param {string} jobName - Name of the job
+     * @param {string} newChecksum - New checksum of the job definition
+     */
+    async #resetJobConfig(jobName, newChecksum) {
+        this.#log.w(`Job ${jobName} implementation changed, resetting configuration`);
+        await this.#jobConfigsCollection.updateOne(
+            { jobName },
+            {
+                $set: {
+                    enabled: true,
+                    checksum: newChecksum,
+                    updatedAt: new Date()
+                },
+                $unset: {
+                    schedule: "",
+                    retry: "",
+                    runNow: ""
+                }
+            }
+        );
+    }
+
+    /**
+     * Applies configurations to jobs
+     * @private
+     * @param {Object.<string, Function>} jobClasses - Job class implementations
+     * @param {Object.<string, string>} jobDefinitionChecksums - Checksums of job definitions
+     */
+    async #applyJobConfigurations(jobClasses, jobDefinitionChecksums) {
+        // Load all existing configuration overrides
+        const configOverrides = await this.#jobConfigsCollection.find({}).toArray();
+        const configOverridesMap = new Map(configOverrides.map(conf => [conf.jobName, conf]));
+
+        // Create and configure jobs
+        await Promise.all(
+            Object.entries(jobClasses).map(async([jobName, JobClass]) => {
+                // Create the job with default settings
+                await this.#jobRunner.createJob(jobName, JobClass);
+
+                // Apply configuration override if valid
+                const configOverride = configOverridesMap.get(jobName);
+                if (configOverride && configOverride.checksum === jobDefinitionChecksums[jobName]) {
+                    await this.#applyConfig(configOverride);
+                    this.#log.d(`Applied configuration override for job: ${jobName}`);
+                }
+            })
+        );
     }
 
     /**
@@ -82,20 +279,6 @@ class JobManager {
         await this.#jobRunner.disableJob(jobName);
     }
 
-    /**
-     * Loads the job classes into the job runner
-     * @param {Object.<string, Function>} jobClasses Object containing job classes keyed by job name
-     * @returns {Promise<void>} A promise that resolves once the jobs are loaded
-     */
-    #loadJobs(jobClasses) {
-        return Promise.all(
-            Object.entries(jobClasses)
-                .map(([name, JobClass]) => {
-                    return this.#jobRunner.createJob(name, JobClass);
-                })
-        );
-    }
-
     /**
      * Closes the JobManager and cleans up resources
      * @returns {Promise<void>} A promise that resolves once cleanup is complete
diff --git a/jobServer/JobServer.js b/jobServer/JobServer.js
index 37a691cbc31..5153fc1e2e6 100644
--- a/jobServer/JobServer.js
+++ b/jobServer/JobServer.js
@@ -109,8 +109,6 @@ class JobServer {
             // await this.#jobConfigsCollection.createIndex({ jobName: 1 }, /*{ unique: true }*/);
 
             this.#setupSignalHandlers();
-            // Watch for changes in job configurations
-            this.#watchJobConfigs();
 
             this.#log.i('Job process init successfully');
         }
@@ -176,40 +174,6 @@ class JobServer {
         });
     }
 
-    /**
-     * Watch for changes in job configurations
-     * @private
-     */
-    async #watchJobConfigs() {
-        const changeStream = this.#jobConfigsCollection.watch();
-
-        changeStream.on('change', async(change) => {
-            try {
-                if (change.operationType === 'update' || change.operationType === 'insert') {
-                    const jobName = change.fullDocument.jobName;
-                    const enabled = change.fullDocument.enabled;
-
-                    if (enabled) {
-                        await this.#jobManager.enableJob(jobName);
-                    }
-                    else {
-                        await this.#jobManager.disableJob(jobName);
-                    }
-
-                    this.#log.i(`Job ${jobName} ${enabled ? 'enabled' : 'disabled'}`);
-                }
-            }
-            catch (error) {
-                this.#log.e('Error processing job config change:', error);
-            }
-        });
-
-        changeStream.on('error', (error) => {
-            this.#log.e('Error in job configs change stream:', error);
-            // Implement reconnection logic here
-        });
-    }
-
     /**
      * Shuts down the job process.
      * @param {number} [exitCode=0] - The exit code to use when shutting down the process.
diff --git a/jobServer/JobUtils.js b/jobServer/JobUtils.js
index 1a4df6a65cf..c412d337c7b 100644
--- a/jobServer/JobUtils.js
+++ b/jobServer/JobUtils.js
@@ -1,6 +1,7 @@
 const Job = require('./Job');
 const {isValidCron} = require('cron-validator');
 const later = require('@breejs/later');
+const crypto = require('crypto');
 
 /**
  * Class responsible for validating job classes.
@@ -235,6 +236,19 @@ class JobUtils {
         return `${minutePart} ${hourPart} ${dayPart} ${monthPart} ${dayOfWeekPart}`;
     }
 
+    /**
+     * Calculates checksum for a job class
+     * @param {Function} JobClass The job class to calculate checksum for
+     * @returns {string} The calculated checksum
+     */
+    static calculateJobChecksum(JobClass) {
+        const jobString = JobClass.toString();
+        return crypto
+            .createHash('sha256')
+            .update(jobString)
+            .digest('hex');
+    }
+
 }
 
 module.exports = JobUtils;
\ No newline at end of file
diff --git a/jobServer/jobRunner/IJobRunner.js b/jobServer/jobRunner/IJobRunner.js
index 258298000ae..6bf89da7863 100644
--- a/jobServer/jobRunner/IJobRunner.js
+++ b/jobServer/jobRunner/IJobRunner.js
@@ -99,6 +99,47 @@ class IJobRunner {
     async disableJob(/*jobName*/) {
         throw new Error('Method not implemented');
     }
+
+    /**
+     * Run a job immediately
+     * @param {string} jobName Name of the job to run
+     * @returns {Promise<void>} A promise that resolves once the job is triggered
+     */
+    async runJobNow(/* jobName */) {
+        throw new Error('runJobNow must be implemented');
+    }
+
+    /**
+     * Update job schedule
+     * @param {string} jobName Name of the job
+     * @param {string|Date} schedule New schedule (cron string or date)
+     * @returns {Promise<void>} A promise that resolves once schedule is updated
+     */
+    async updateSchedule(/* jobName, schedule */) {
+        throw new Error('updateSchedule must be implemented');
+    }
+
+    /**
+     * Configure job retry settings
+     * @param {string} jobName Name of the job
+     * @param {Object} retryConfig Retry configuration
+     * @param {number} retryConfig.attempts Number of retry attempts
+     * @param {number} retryConfig.delay Delay between retries in ms
+     * @returns {Promise<void>} A promise that resolves once retry is configured
+     */
+    async configureRetry(/* jobName, retryConfig */) {
+        throw new Error('configureRetry must be implemented');
+    }
+
+    /**
+     * Maps generic priority to runner-specific priority
+     * @protected
+     * @param {string} priority Generic priority from JOB_PRIORITIES
+     * returns {any} Runner-specific priority value
+     */
+    _mapPriority(/* priority */) {
+        throw new Error('_mapPriority must be implemented');
+    }
 }
 
 module.exports = IJobRunner;
\ No newline at end of file
diff --git a/jobServer/jobRunner/JobRunnerPulseImpl.js b/jobServer/jobRunner/JobRunnerPulseImpl.js
index 6b77e8362f2..e25bb74afdc 100644
--- a/jobServer/jobRunner/JobRunnerPulseImpl.js
+++ b/jobServer/jobRunner/JobRunnerPulseImpl.js
@@ -1,6 +1,7 @@
 const IJobRunner = require('./IJobRunner');
 const { Pulse, JobPriority } = require('@pulsecron/pulse');
 const {isValidCron} = require('cron-validator');
+const { JOB_PRIORITIES } = require('../constants/JobPriorities');
 
 /**
  * Pulse implementation of the job runner
@@ -77,6 +78,12 @@ class JobRunnerPulseImpl extends IJobRunner {
             // instance.setLogger(this.log);
             instance.setJobName(jobName);
 
+            // Get job configurations
+            const retryConfig = instance.getRetryConfig();
+            const priority = this._mapPriority(instance.getPriority());
+            const concurrency = instance.getConcurrency();
+            const lockLifetime = instance.getLockLifetime();
+
             this.#pulseRunner.define(
                 jobName,
                 async(job, done) => {
@@ -84,27 +91,21 @@ class JobRunnerPulseImpl extends IJobRunner {
                     instance._setProgressMethod(
                         async(progressData) => this.#updateJobProgress(job, progressData)
                     );
-
-                    return instance._run(
-                        this.db,
-                        job,
-                        done
-                    );
+                    return instance._run(this.db, job, done);
                 },
                 {
-                    priority: JobPriority.normal,
-                    concurrency: 1,
-                    lockLifetime: 10000,
+                    priority,
+                    concurrency,
+                    lockLifetime,
                     shouldSaveResult: true,
-                    attempts: 3,
-                    backoff: {
+                    attempts: retryConfig?.enabled ? retryConfig.attempts : 1,
+                    backoff: retryConfig?.enabled ? {
                         type: 'exponential',
-                        delay: 2000
-                    }
+                        delay: retryConfig.delay
+                    } : undefined
                 }
             );
 
-            // Store schedule configuration for later
             const scheduleConfig = instance.getSchedule();
             this.#pendingSchedules.set(jobName, scheduleConfig);
             this.log.d(`Job ${jobName} defined successfully`);
@@ -194,6 +195,86 @@ class JobRunnerPulseImpl extends IJobRunner {
             throw error;
         }
     }
+
+    /**
+     * Triggers immediate execution of a job
+     * @param {string} jobName Name of the job to run
+     * @returns {Promise<void>} A promise that resolves when the job is triggered
+     */
+    async runJobNow(jobName) {
+        try {
+            await this.#pulseRunner.now({ name: jobName });
+            this.log.i(`Job ${jobName} triggered for immediate execution`);
+        }
+        catch (error) {
+            this.log.e(`Failed to run job ${jobName} immediately:`, error);
+            throw error;
+        }
+    }
+
+    /**
+     * Updates the schedule of an existing job
+     * @param {string} jobName Name of the job to update
+     * @param {Object} schedule New schedule configuration
+     * @returns {Promise<void>} A promise that resolves when the schedule is updated
+     */
+    async updateSchedule(jobName, schedule) {
+        try {
+            await this.#pulseRunner.reschedule({ name: jobName }, schedule);
+            this.log.i(`Schedule updated for job ${jobName}`);
+        }
+        catch (error) {
+            this.log.e(`Failed to update schedule for job ${jobName}:`, error);
+            throw error;
+        }
+    }
+
+    /**
+     * Configures retry settings for a job
+     * @param {string} jobName Name of the job
+     * @param {Object} retryConfig Retry configuration
+     * @param {number} retryConfig.attempts Number of retry attempts
+     * @param {number} retryConfig.delay Delay between retries in milliseconds
+     * @returns {Promise<void>} A promise that resolves when retry config is updated
+     */
+    async configureRetry(jobName, retryConfig) {
+        try {
+            await this.#pulseRunner.updateOne(
+                { name: jobName },
+                {
+                    $set: {
+                        attempts: retryConfig.attempts,
+                        backoff: {
+                            delay: retryConfig.delay,
+                            type: 'fixed'
+                        }
+                    }
+                }
+            );
+            this.log.i(`Retry configuration updated for job ${jobName}`);
+        }
+        catch (error) {
+            this.log.e(`Failed to configure retry for job ${jobName}:`, error);
+            throw error;
+        }
+    }
+
+    /**
+     * Maps generic priority to runner-specific priority
+     * @protected
+     * @param {string} priority Generic priority from JOB_PRIORITIES
+     * @returns {any} Runner-specific priority value
+     */
+    _mapPriority(priority) {
+        const priorityMap = {
+            [JOB_PRIORITIES.LOWEST]: JobPriority.lowest,
+            [JOB_PRIORITIES.LOW]: JobPriority.low,
+            [JOB_PRIORITIES.NORMAL]: JobPriority.normal,
+            [JOB_PRIORITIES.HIGH]: JobPriority.high,
+            [JOB_PRIORITIES.HIGHEST]: JobPriority.highest
+        };
+        return priorityMap[priority] || JobPriority.normal;
+    }
 }
 
 module.exports = JobRunnerPulseImpl;
\ No newline at end of file

From cc126901ae55300dfebd6cc59d333be278b771a4 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Wed, 15 Jan 2025 17:39:51 +0530
Subject: [PATCH 19/90] refactor for code cleanup because why not

---
 jobServer/Job.js                              |  36 ++--
 jobServer/JobUtils.js                         |  49 +++--
 jobServer/constants/JobPriorities.js          |  24 +++
 jobServer/example/ExampleJob.js               | 156 ++++++++++++---
 jobServer/jobRunner/BaseJobRunner.js          | 116 +++++++++++
 jobServer/jobRunner/PulseJobRunner.js         |  39 ++++
 .../jobRunner/impl/pulse/PulseJobExecutor.js  | 187 ++++++++++++++++++
 .../jobRunner/impl/pulse/PulseJobLifecycle.js |  64 ++++++
 .../jobRunner/impl/pulse/PulseJobScheduler.js | 142 +++++++++++++
 jobServer/jobRunner/index.js                  |  16 +-
 .../jobRunner/interfaces/IJobExecutor.js      |  42 ++++
 .../jobRunner/interfaces/IJobLifecycle.js     |  21 ++
 .../jobRunner/interfaces/IJobScheduler.js     |  35 ++++
 13 files changed, 856 insertions(+), 71 deletions(-)
 create mode 100644 jobServer/constants/JobPriorities.js
 create mode 100644 jobServer/jobRunner/BaseJobRunner.js
 create mode 100644 jobServer/jobRunner/PulseJobRunner.js
 create mode 100644 jobServer/jobRunner/impl/pulse/PulseJobExecutor.js
 create mode 100644 jobServer/jobRunner/impl/pulse/PulseJobLifecycle.js
 create mode 100644 jobServer/jobRunner/impl/pulse/PulseJobScheduler.js
 create mode 100644 jobServer/jobRunner/interfaces/IJobExecutor.js
 create mode 100644 jobServer/jobRunner/interfaces/IJobLifecycle.js
 create mode 100644 jobServer/jobRunner/interfaces/IJobScheduler.js

diff --git a/jobServer/Job.js b/jobServer/Job.js
index c1c47212aa5..6652d954c38 100644
--- a/jobServer/Job.js
+++ b/jobServer/Job.js
@@ -1,10 +1,6 @@
-const defaultLogger = {
-    d: console.debug,
-    w: console.warn,
-    e: console.error,
-    i: console.info
-};
+const defaultLogger = { d: console.debug, w: console.warn, e: console.error, i: console.info };
 const { JOB_PRIORITIES } = require('./constants/JobPriorities');
+
 /**
  * Base class for creating jobs.
  * 
@@ -67,10 +63,12 @@ class Job {
     logger;
 
     /** @type {Function|null} Touch method from job runner */
-    _touchMethod = null;
+    #touchMethod = null;
 
     /** @type {Function|null} Progress method from job runner */
-    _progressMethod = null;
+    #progressMethod;
+
+    priorities = JOB_PRIORITIES;
 
     /**
      * Creates an instance of Job.
@@ -229,19 +227,19 @@ class Job {
     /**
      * Sets the touch method (called internally by job runner)
      * @param {Function} touchMethod The touch method from Pulse
-     * @private
+     * @public
      */
-    _setTouchMethod(touchMethod) {
-        this._touchMethod = touchMethod;
+    setTouchMethod(touchMethod) {
+        this.#touchMethod = touchMethod;
     }
 
     /**
      * Sets the progress method from the runner
      * @param {Function} progressMethod Method to update progress
-     * @protected
+     * @public
      */
-    _setProgressMethod(progressMethod) {
-        this._progressMethod = progressMethod;
+    setProgressMethod(progressMethod) {
+        this.#progressMethod = progressMethod;
     }
 
     /**
@@ -264,13 +262,13 @@ class Job {
         };
 
         // Update progress using runner's method if available
-        if (this._progressMethod) {
-            await this._progressMethod(progressData);
+        if (this.#progressMethod) {
+            await this.#progressMethod(progressData);
         }
 
         // Touch to prevent lock expiration
-        if (this._touchMethod) {
-            await this._touchMethod();
+        if (this.#touchMethod) {
+            await this.#touchMethod();
         }
 
         this.logger?.d(`Progress reported for job "${this.jobName}":`, progressData);
@@ -297,7 +295,7 @@ class Job {
      * @returns {string} Priority level from JOB_PRIORITIES
      */
     getPriority() {
-        return JOB_PRIORITIES.NORMAL;
+        return this.priorities.NORMAL;
     }
 
     /**
diff --git a/jobServer/JobUtils.js b/jobServer/JobUtils.js
index c412d337c7b..ccc0b05d502 100644
--- a/jobServer/JobUtils.js
+++ b/jobServer/JobUtils.js
@@ -26,19 +26,47 @@ class JobUtils {
             throw new Error(`Job class must extend ${BaseClass.name}`);
         }
 
-        // Additional checks can be added here
+        // Check if required methods are overridden
+        const requiredMethods = ['run', 'getSchedule'];
+        for (const method of requiredMethods) {
+            // Get the method from the job class prototype
+            const jobMethod = JobClass.prototype[method];
+            // Get the method from the base class prototype
+            const baseMethod = BaseClass.prototype[method];
+
+            // Check if method exists and is different from base class implementation
+            if (!jobMethod || jobMethod === baseMethod) {
+                throw new Error(`Job class must override the '${method}' method`);
+            }
+        }
+
         return true;
     }
 
     /**
-     * @note
-     * We shouldn't need this and use cron string directly in job schedule to avoid conversion
-     * Converts a later.js schedule to a cron string.
+     * Calculates checksum for a job class
+     * @param {Function} JobClass The job class to calculate checksum for
+     * @returns {string} The calculated checksum
+     */
+    static calculateJobChecksum(JobClass) {
+        const jobString = JobClass.toString();
+        return crypto
+            .createHash('sha256')
+            .update(jobString)
+            .digest('hex');
+    }
+
+    /**
      * @param {String} laterString - The later.js schedule string.
      * @constructor
      * @retuns {String} The cron string.
      *
      * @note
+     * 
+     * We shouldn't need this and use cron string directly in job schedule to avoid conversion
+     * Converts a later.js schedule to a cron string.
+     * 
+     * if we do decide to use this, we need to add tests for all the possible cron strings currently used in countly
      * TESTS NEEDED
      *
      * "every 5 minutes"
@@ -236,19 +264,6 @@ class JobUtils {
         return `${minutePart} ${hourPart} ${dayPart} ${monthPart} ${dayOfWeekPart}`;
     }
 
-    /**
-     * Calculates checksum for a job class
-     * @param {Function} JobClass The job class to calculate checksum for
-     * @returns {string} The calculated checksum
-     */
-    static calculateJobChecksum(JobClass) {
-        const jobString = JobClass.toString();
-        return crypto
-            .createHash('sha256')
-            .update(jobString)
-            .digest('hex');
-    }
-
 }
 
 module.exports = JobUtils;
\ No newline at end of file
diff --git a/jobServer/constants/JobPriorities.js b/jobServer/constants/JobPriorities.js
new file mode 100644
index 00000000000..dad4b34cd8b
--- /dev/null
+++ b/jobServer/constants/JobPriorities.js
@@ -0,0 +1,24 @@
+/**
+ * Job priority levels
+ * @enum {string}
+ */
+const JOB_PRIORITIES = {
+    /** Lowest priority level */
+    LOWEST: 'lowest',
+
+    /** Low priority level */
+    LOW: 'low',
+
+    /** Normal/default priority level */
+    NORMAL: 'normal',
+
+    /** High priority level */
+    HIGH: 'high',
+
+    /** Highest priority level */
+    HIGHEST: 'highest'
+};
+
+module.exports = {
+    JOB_PRIORITIES
+};
\ No newline at end of file
diff --git a/jobServer/example/ExampleJob.js b/jobServer/example/ExampleJob.js
index a05b0da6f7c..11fb89a2114 100644
--- a/jobServer/example/ExampleJob.js
+++ b/jobServer/example/ExampleJob.js
@@ -2,45 +2,131 @@ const job = require("../../jobServer");
 
 /**
  * Example job implementation demonstrating all features of the job system.
- * This job processes user records in batches, demonstrating:
- * - Progress reporting
- * - Error handling
- * - Different schedule types
- * - Database operations
- * - Proper logging
+ * 
+ * Required methods:
+ * - getSchedule()
+ * - run()
+ * 
+ * Optional methods with defaults:
+ * - getRetryConfig() - Default: { enabled: true, attempts: 3, delay: 2000 }
+ * - getPriority() - Default: NORMAL
+ * - getConcurrency() - Default: 1
+ * - getLockLifetime() - Default: 55 minutes
  * 
  * @extends {job.Job}
  */
 class ExampleJob extends job.Job {
+
     /**
      * Get the schedule configuration for the job.
-     * Demonstrates all possible schedule types.
+     * @required
      * 
      * @returns {Object} Schedule configuration object
+     * @property {('once'|'schedule'|'now')} type - Type of schedule
+     * @property {string|Date} [value] - Schedule value (cron expression or Date)
+     * 
+     * @example
+     * // Run every day at midnight
+     * return {
+     *     type: 'schedule',
+     *     value: '0 0 * * *'
+     * }
+     * 
+     * @example
+     * // Run once at a specific future date
+     * return {
+     *     type: 'once',
+     *     value: new Date('2024-12-31T23:59:59Z')
+     * }
+     * 
+     * @example
+     * // Run immediately
+     * return {
+     *     type: 'now'
+     * }
+     * 
+     * @example
+     * // Run every 5 minutes
+     * return {
+     *     type: 'schedule',
+     *     value: '* /5 * * * *'
+     * }
+     * 
+     * @example
+     * // Run at specific times using cron
+     * return {
+     *     type: 'schedule',
+     *     value: '0 9,15,21 * * *'  // Runs at 9am, 3pm, and 9pm
+     * }
      */
     getSchedule() {
-        // Example 1: Run every day at midnight
+        // Example: Run every day at midnight
         return {
             type: 'schedule',
             value: '0 0 * * *'
         };
+    }
 
-        // Example 2: Run once at a specific future date
-        // return {
-        //     type: 'once',
-        //     value: new Date('2024-12-31T23:59:59Z')
-        // };
+    /**
+     * Configure retry behavior for failed jobs.
+     * @optional
+     * @default { enabled: false, attempts: 3, delay: 2000 }
+     * 
+     * @returns {Object} Retry configuration
+     * @property {boolean} enabled - Whether retries are enabled
+     * @property {number} attempts - Number of retry attempts
+     * @property {number} delay - Initial delay between retries in milliseconds
+     *                           (increases exponentially with each retry)
+     */
+    getRetryConfig() {
+        return {
+            enabled: true,
+            attempts: 3, // Will try up to 5 times
+            delay: 1000 * 60 // Start with 1-minute delay, then exponential backoff
+        };
+    }
 
-        // Example 3: Run immediately
-        // return {
-        //     type: 'now'
-        // };
+    /**
+     * Set job priority level.
+     * Higher priority jobs are processed first.
+     * @optional
+     * @default "NORMAL"
+     * 
+     * @returns {string} Priority level (HIGH, NORMAL, LOW)
+     */
+    getPriority() {
+        return this.priorities.HIGH;
+    }
+
+    /**
+     * Set maximum concurrent instances of this job.
+     * Limits how many copies of this job can run simultaneously.
+     * @optional
+     * @default 1
+     * 
+     * @returns {number} Maximum number of concurrent instances
+     */
+    getConcurrency() {
+        return 2; // Allow 2 instances to run simultaneously
     }
 
     /**
-     * Simulates some processing work
+     * Set job lock lifetime.
+     * Determines how long a job can run before its lock expires.
+     * @optional
+     * @default 55 * 60 * 1000 (55 minutes)
+     * 
+     * @returns {number} Lock lifetime in milliseconds
+     */
+    getLockLifetime() {
+        return 30 * 60 * 1000; // 30 minutes
+    }
+
+    /**
+     * Simulates processing work with a delay
      * @private
      * @param {number} ms Time to wait in milliseconds
+     * @returns {Promise<void>} Returns a promise that resolves when the delay is complete
      */
     async #simulateWork(ms) {
         await new Promise(resolve => setTimeout(resolve, ms));
@@ -48,13 +134,16 @@ class ExampleJob extends job.Job {
 
     /**
      * Main job execution method.
+     * @required
+     * 
      * Demonstrates:
      * - Progress reporting
      * - Synthetic delays to simulate work
      * - Error handling
-     * - Logging
+     * - Logging at different levels
+     * - Database operations (commented examples)
      * 
-     * @param {Object} db Database connection
+     * @param {Db} db Database connection
      * @param {Function} done Callback to signal job completion
      * @param {Function} progress Progress reporting function
      */
@@ -62,17 +151,25 @@ class ExampleJob extends job.Job {
         try {
             this.logger.d("Starting example job execution");
 
-            // Simulate total items to process
-            const total = 100;
+            // Example: Query total items to process
+            // const total = await db.collection('users').countDocuments({ active: false });
+            const total = 100; // Simulated total
             let processed = 0;
 
             // Process in batches of 10
             for (let i = 0; i < total; i += 10) {
+                // Example: Process a batch of records
+                // const batch = await db.collection('users')
+                //     .find({ active: false })
+                //     .skip(i)
+                //     .limit(10)
+                //     .toArray();
+
                 // Simulate batch processing (2 second per batch)
                 await this.#simulateWork(2000);
                 processed += 10;
 
-                // Report progress
+                // Report progress with detailed bookmark
                 await progress(
                     total,
                     processed,
@@ -81,6 +178,17 @@ class ExampleJob extends job.Job {
 
                 this.logger.d(`Completed batch ${(i / 10) + 1}/10`);
 
+                // Example: Update processed records
+                // await db.collection('users').updateMany(
+                //     { _id: { $in: batch.map(doc => doc._id) } },
+                //     { 
+                //         $set: { 
+                //             processed: true, 
+                //             updatedAt: new Date() 
+                //         } 
+                //     }
+                // );
+
                 // Simulate random error (10% chance)
                 if (Math.random() < 0.1) {
                     throw new Error('Random batch processing error');
@@ -90,7 +198,7 @@ class ExampleJob extends job.Job {
             // Simulate final processing
             await this.#simulateWork(1000);
 
-            // Job completion
+            // Job completion with result data
             const result = {
                 processedCount: processed,
                 totalItems: total,
diff --git a/jobServer/jobRunner/BaseJobRunner.js b/jobServer/jobRunner/BaseJobRunner.js
new file mode 100644
index 00000000000..84af66dcdb2
--- /dev/null
+++ b/jobServer/jobRunner/BaseJobRunner.js
@@ -0,0 +1,116 @@
+const IJobScheduler = require('./interfaces/IJobScheduler');
+const IJobExecutor = require('./interfaces/IJobExecutor');
+const IJobLifecycle = require('./interfaces/IJobLifecycle');
+
+/**
+ * Base class for job runners that implements all interfaces through composition
+ */
+class BaseJobRunner {
+    /**
+     * @param {IJobScheduler} scheduler - Scheduler implementation
+     * @param {IJobExecutor} executor - Executor implementation
+     * @param {IJobLifecycle} lifecycle - Lifecycle implementation
+     */
+    constructor(scheduler, executor, lifecycle) {
+        if (!(scheduler instanceof IJobScheduler)) {
+            throw new Error('Invalid scheduler implementation');
+        }
+        if (!(executor instanceof IJobExecutor)) {
+            throw new Error('Invalid executor implementation');
+        }
+        if (!(lifecycle instanceof IJobLifecycle)) {
+            throw new Error('Invalid lifecycle implementation');
+        }
+
+        this.scheduler = scheduler;
+        this.executor = executor;
+        this.lifecycle = lifecycle;
+    }
+
+    /**
+     * Schedules a job to run
+     * @param {string} name Job name
+     * @param {Object} scheduleConfig Schedule configuration
+     * @param {Object} [data] Optional data to pass to the job
+     * @returns {Promise<void>} A promise that resolves once the job is scheduled
+     */
+    async schedule(name, scheduleConfig, data) {
+        return this.scheduler.schedule(name, scheduleConfig, data);
+    }
+
+    /**
+     * Updates a job's schedule
+     * @param {string} jobName Name of the job
+     * @param {Object} schedule New schedule configuration
+     * @returns {Promise<void>} A promise that resolves once the schedule is updated
+     */
+    async updateSchedule(jobName, schedule) {
+        return this.scheduler.updateSchedule(jobName, schedule);
+    }
+
+    /**
+     * Runs a job immediately
+     * @param {string} jobName Name of the job
+     * @returns {Promise<void>} A promise that resolves when the job is triggered
+     */
+    async runJobNow(jobName) {
+        return this.scheduler.runJobNow(jobName);
+    }
+
+    /**
+     * Creates and registers a new job
+     * @param {string} jobName Name of the job
+     * @param {Function} JobClass Job class implementation
+     * @returns {Promise<void>} A promise that resolves once the job is created and registered
+     */
+    async createJob(jobName, JobClass) {
+        return this.executor.createJob(jobName, JobClass);
+    }
+
+    /**
+     * Enables a job
+     * @param {string} jobName Name of the job
+     * @returns {Promise<void>} A promise that resolves once the job is enabled
+     */
+    async enableJob(jobName) {
+        return this.executor.enableJob(jobName);
+    }
+
+    /**
+     * Disables a job
+     * @param {string} jobName Name of the job
+     * @returns {Promise<void>} A promise that resolves once the job is disabled
+     */
+    async disableJob(jobName) {
+        return this.executor.disableJob(jobName);
+    }
+
+    /**
+     * Configures retry settings for a job
+     * @param {string} jobName Name of the job
+     * @param {Object} retryConfig Retry configuration
+     * @returns {Promise<void>} A promise that resolves once the retry settings are configured
+     */
+    async configureRetry(jobName, retryConfig) {
+        return this.executor.configureRetry(jobName, retryConfig);
+    }
+
+    /**
+     * Starts the job runner
+     * @param {Object.<string, Function>} jobClasses Job classes to register
+     * @returns {Promise<void>} A promise that resolves once the job runner is started
+     */
+    async start(jobClasses) {
+        return this.lifecycle.start(jobClasses);
+    }
+
+    /**
+     * Closes the job runner and cleans up resources
+     * @returns {Promise<void>} A promise that resolves once the job runner is closed
+     */
+    async close() {
+        return this.lifecycle.close();
+    }
+}
+
+module.exports = BaseJobRunner;
\ No newline at end of file
diff --git a/jobServer/jobRunner/PulseJobRunner.js b/jobServer/jobRunner/PulseJobRunner.js
new file mode 100644
index 00000000000..03875504e38
--- /dev/null
+++ b/jobServer/jobRunner/PulseJobRunner.js
@@ -0,0 +1,39 @@
+const { Pulse } = require('@pulsecron/pulse');
+const BaseJobRunner = require('./BaseJobRunner');
+const PulseJobScheduler = require('./impl/pulse/PulseJobScheduler');
+const PulseJobExecutor = require('./impl/pulse/PulseJobExecutor');
+const PulseJobLifecycle = require('./impl/pulse/PulseJobLifecycle');
+
+/**
+ * Pulse-specific implementation of the job runner using BaseJobRunner composition
+ */
+class PulseJobRunner extends BaseJobRunner {
+    /**
+     * Creates a new Pulse job runner with all required implementations
+     * @param {Object} db Database connection
+     * @param {Object} config Configuration object
+     * @param {function} Logger - Logger constructor
+     */
+    constructor(db, config, Logger) {
+        const log = Logger('jobs:runner:pulse');
+
+        // Create the Pulse instance that will be shared across implementations
+        const pulseRunner = new Pulse({
+            ...config,
+            mongo: db,
+        });
+
+        // Create implementations with shared pulseRunner instance
+        const scheduler = new PulseJobScheduler(pulseRunner, log);
+        const executor = new PulseJobExecutor(pulseRunner, db, log);
+        const lifecycle = new PulseJobLifecycle(pulseRunner, executor, scheduler, log);
+
+        // Initialize base class with implementations
+        super(scheduler, executor, lifecycle);
+
+        this.log = log;
+        this.pulseRunner = pulseRunner;
+    }
+}
+
+module.exports = PulseJobRunner;
\ No newline at end of file
diff --git a/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js b/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js
new file mode 100644
index 00000000000..1016800817e
--- /dev/null
+++ b/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js
@@ -0,0 +1,187 @@
+const IJobExecutor = require('../../interfaces/IJobExecutor');
+const { JobPriority } = require('@pulsecron/pulse');
+const { JOB_PRIORITIES } = require('../../../constants/JobPriorities');
+
+/**
+ * Pulse implementation of job executor
+ */
+class PulseJobExecutor extends IJobExecutor {
+    /**
+     * Creates a new PulseJobExecutor instance
+     * @param {Object} pulseRunner The Pulse runner instance
+     * @param {Object} db Database connection
+     * @param {Object} logger Logger instance
+     */
+    constructor(pulseRunner, db, logger) {
+        super();
+        this.pulseRunner = pulseRunner;
+        this.db = db;
+        this.log = logger;
+        this.pendingSchedules = new Map();
+    }
+
+    /**
+     * Creates and registers a new job
+     * @param {string} jobName Name of the job
+     * @param {Function} JobClass Job class implementation
+     * @returns {Promise<void>} A promise that resolves once the job is created
+     */
+    async createJob(jobName, JobClass) {
+        try {
+            const instance = new JobClass(jobName);
+            instance.setJobName(jobName);
+
+            const retryConfig = instance.getRetryConfig();
+            const priority = this.#mapPriority(instance.getPriority());
+            const concurrency = instance.getConcurrency();
+            const lockLifetime = instance.getLockLifetime();
+
+            this.pulseRunner.define(
+                jobName,
+                async(job, done) => {
+                    instance.setTouchMethod(job.touch.bind(job));
+                    instance.setProgressMethod(
+                        async(progressData) => this.#updateJobProgress(job, progressData)
+                    );
+                    return instance._run(this.db, job, done);
+                },
+                {
+                    priority,
+                    concurrency,
+                    lockLifetime,
+                    shouldSaveResult: true,
+                    attempts: retryConfig?.enabled ? retryConfig.attempts : 1,
+                    backoff: retryConfig?.enabled ? {
+                        type: 'exponential',
+                        delay: retryConfig.delay
+                    } : undefined
+                }
+            );
+
+            const scheduleConfig = instance.getSchedule();
+            this.pendingSchedules.set(jobName, scheduleConfig);
+            this.log.d(`Job ${jobName} defined successfully`);
+        }
+        catch (error) {
+            this.log.e(`Failed to create job ${jobName}:`, error);
+        }
+    }
+
+    /**
+     * Enables a job
+     * @param {string} jobName Name of the job
+     * @returns {Promise<void>} A promise that resolves once the job is enabled
+     */
+    async enableJob(jobName) {
+        try {
+            await this.pulseRunner.enable({ name: jobName });
+            this.log.i(`Job ${jobName} enabled`);
+        }
+        catch (error) {
+            this.log.e(`Failed to enable job ${jobName}:`, error);
+            throw error;
+        }
+    }
+
+    /**
+     * Disables a job
+     * @param {string} jobName Name of the job
+     * @returns {Promise<void>} A promise that resolves once the job is disabled
+     */
+    async disableJob(jobName) {
+        try {
+            await this.pulseRunner.disable({ name: jobName });
+            this.log.i(`Job ${jobName} disabled`);
+        }
+        catch (error) {
+            this.log.e(`Failed to disable job ${jobName}:`, error);
+            throw error;
+        }
+    }
+
+    /**
+     * Configures retry settings for a job
+     * @param {string} jobName Name of the job
+     * @param {Object} retryConfig Retry configuration
+     * @returns {Promise<void>} A promise that resolves once retry settings are updated
+     */
+    async configureRetry(jobName, retryConfig) {
+        try {
+            // First get the job definition
+            const definition = this.pulseRunner._definitions[jobName];
+            if (!definition) {
+                throw new Error(`Job ${jobName} not found`);
+            }
+
+            // Update the definition for future job instances
+            definition.attempts = retryConfig.attempts;
+            definition.backoff = {
+                delay: retryConfig.delay,
+                type: 'exponential'
+            };
+
+            // Update existing jobs in the queue using the MongoDB collection directly
+            await this.pulseRunner._collection.updateMany(
+                { name: jobName },
+                {
+                    $set: {
+                        attempts: retryConfig.attempts,
+                        backoff: {
+                            delay: retryConfig.delay,
+                            type: 'exponential'
+                        }
+                    }
+                }
+            );
+
+            this.log.i(`Retry configuration updated for job ${jobName}`);
+        }
+        catch (error) {
+            this.log.e(`Failed to configure retry for job ${jobName}:`, error);
+            throw error;
+        }
+    }
+
+    /**
+     * Updates job progress data
+     * @private
+     * @param {Object} job Job instance
+     * @param {Object} progressData Progress data to store
+     * @returns {Promise<void>} A promise that resolves once progress is updated
+     */
+    async #updateJobProgress(job, progressData) {
+        try {
+            job.data = progressData;
+            await job.save();
+        }
+        catch (error) {
+            this.log.e(`Failed to update job progress: ${error.message}`);
+            // Consider whether to throw
+        }
+    }
+
+    /**
+     * Maps generic priority to Pulse-specific priority with validation
+     * @private
+     * @param {string} priority Generic priority from JOB_PRIORITIES
+     * @returns {JobPriority} Pulse-specific priority value
+     */
+    #mapPriority(priority) {
+        const priorityMap = {
+            [JOB_PRIORITIES.LOWEST]: JobPriority.lowest,
+            [JOB_PRIORITIES.LOW]: JobPriority.low,
+            [JOB_PRIORITIES.NORMAL]: JobPriority.normal,
+            [JOB_PRIORITIES.HIGH]: JobPriority.high,
+            [JOB_PRIORITIES.HIGHEST]: JobPriority.highest
+        };
+
+        if (!priority || !priorityMap[priority]) {
+            this.log.w(`Invalid priority "${priority}", defaulting to normal`);
+            return JobPriority.normal;
+        }
+
+        return priorityMap[priority];
+    }
+}
+
+module.exports = PulseJobExecutor;
\ No newline at end of file
diff --git a/jobServer/jobRunner/impl/pulse/PulseJobLifecycle.js b/jobServer/jobRunner/impl/pulse/PulseJobLifecycle.js
new file mode 100644
index 00000000000..e9955f4334c
--- /dev/null
+++ b/jobServer/jobRunner/impl/pulse/PulseJobLifecycle.js
@@ -0,0 +1,64 @@
+const IJobLifecycle = require('../../interfaces/IJobLifecycle');
+
+/**
+ * Pulse implementation of job lifecycle management
+ */
+class PulseJobLifecycle extends IJobLifecycle {
+    /**
+     * Creates a new PulseJobLifecycle instance
+     * @param {Object} pulseRunner The Pulse runner instance
+     * @param {Object} executor Job executor instance
+     * @param {Object} scheduler Job scheduler instance
+     * @param {Object} logger Logger instance
+     */
+    constructor(pulseRunner, executor, scheduler, logger) {
+        super();
+        this.pulseRunner = pulseRunner;
+        this.executor = executor;
+        this.scheduler = scheduler;
+        this.log = logger;
+    }
+
+    /**
+     * Starts the job runner and schedules pending jobs
+     * @param {Object.<string, Function>} jobClasses Job classes to register
+     * @returns {Promise<void>} A promise that resolves once the runner is started
+     */
+    async start(/* jobClasses */) {
+        if (!this.pulseRunner) {
+            throw new Error('Pulse runner not initialized');
+        }
+
+        await this.pulseRunner.start();
+        this.log.i('Pulse runner started');
+
+        // Schedule all pending jobs from the executor
+        for (const [jobName, scheduleConfig] of this.executor.pendingSchedules) {
+            try {
+                await this.scheduler.schedule(jobName, scheduleConfig);
+            }
+            catch (error) {
+                this.log.e(`Failed to schedule job ${jobName}:`, error);
+            }
+        }
+
+        this.executor.pendingSchedules.clear();
+    }
+
+    /**
+     * Closes the job runner and cleans up resources
+     * @returns {Promise<void>} A promise that resolves once the runner is closed
+     */
+    async close() {
+        try {
+            await this.pulseRunner.close();
+            this.log.i('Pulse runner closed');
+        }
+        catch (error) {
+            this.log.e('Error closing Pulse runner:', error);
+            throw error;
+        }
+    }
+}
+
+module.exports = PulseJobLifecycle;
\ No newline at end of file
diff --git a/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js b/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js
new file mode 100644
index 00000000000..86a09274b7f
--- /dev/null
+++ b/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js
@@ -0,0 +1,142 @@
+const IJobScheduler = require('../../interfaces/IJobScheduler');
+const {isValidCron} = require('cron-validator');
+
+/**
+ * Pulse implementation of job scheduler
+ */
+class PulseJobScheduler extends IJobScheduler {
+    /**
+     * Creates a new PulseJobScheduler instance
+     * @param {Object} pulseRunner The Pulse runner instance
+     * @param {Object} logger Logger instance
+     */
+    constructor(pulseRunner, logger) {
+        super();
+        this.pulseRunner = pulseRunner;
+        this.log = logger;
+    }
+
+    /**
+     * Validates schedule configuration
+     * @private
+     * @param {Object} config Schedule configuration
+     * @throws {Error} If configuration is invalid
+     */
+    #validateScheduleConfig(config) {
+        if (!config || typeof config !== 'object') {
+            throw new Error('Schedule configuration must be an object');
+        }
+
+        if (!config.type) {
+            throw new Error('Schedule type is required');
+        }
+
+        const validTypes = ['schedule', 'once', 'now'];
+        if (!validTypes.includes(config.type)) {
+            throw new Error(`Invalid schedule type: ${config.type}`);
+        }
+
+        if (config.type !== 'now' && !config.value) {
+            throw new Error('Schedule value is required');
+        }
+    }
+
+    /**
+     * Schedules a job to run
+     * @param {string} name Job name
+     * @param {Object} scheduleConfig Schedule configuration
+     * @param {Object} [data] Optional data to pass to the job
+     * @returns {Promise<void>} A promise that resolves once the job is scheduled
+     */
+    async schedule(name, scheduleConfig, data = {}) {
+        try {
+            this.#validateScheduleConfig(scheduleConfig);
+
+            switch (scheduleConfig.type) {
+            case 'schedule':
+                if (!isValidCron(scheduleConfig.value)) {
+                    throw new Error('Invalid cron schedule');
+                }
+                await this.pulseRunner.every(scheduleConfig.value, name, data);
+                break;
+
+            case 'once':
+                if (!(scheduleConfig.value instanceof Date)) {
+                    throw new Error('Invalid date for one-time schedule');
+                }
+                await this.pulseRunner.schedule(scheduleConfig.value, name, data);
+                break;
+
+            case 'now':
+                await this.pulseRunner.now(name, data);
+                break;
+            }
+
+            this.log.d(`Job ${name} scheduled successfully with type: ${scheduleConfig.type}`);
+        }
+        catch (error) {
+            this.log.e(`Failed to schedule job ${name}:`, error);
+            throw error;
+        }
+    }
+
+    /**
+     * Updates a job's schedule
+     * @param {string} jobName Name of the job
+     * @param {Object} schedule New schedule configuration
+     * @returns {Promise<void>} A promise that resolves once the schedule is updated
+     */
+    async updateSchedule(jobName, schedule) {
+        try {
+            this.#validateScheduleConfig(schedule);
+
+            // First remove the existing job
+            await this.pulseRunner.remove({ name: jobName });
+
+            // Then create a new schedule based on the type
+            switch (schedule.type) {
+            case 'schedule':
+                if (!isValidCron(schedule.value)) {
+                    throw new Error('Invalid cron schedule');
+                }
+                await this.pulseRunner.every(schedule.value, jobName);
+                break;
+
+            case 'once':
+                if (!(schedule.value instanceof Date)) {
+                    throw new Error('Invalid date for one-time schedule');
+                }
+                await this.pulseRunner.schedule(schedule.value, jobName);
+                break;
+
+            case 'now':
+                await this.pulseRunner.now(jobName);
+                break;
+            }
+
+            this.log.i(`Schedule updated for job ${jobName}`);
+        }
+        catch (error) {
+            this.log.e(`Failed to update schedule for job ${jobName}:`, error);
+            throw error;
+        }
+    }
+
+    /**
+     * Runs a job immediately
+     * @param {string} jobName Name of the job
+     * @returns {Promise<void>} A promise that resolves when the job is triggered
+     */
+    async runJobNow(jobName) {
+        try {
+            await this.pulseRunner.now({ name: jobName });
+            this.log.i(`Job ${jobName} triggered for immediate execution`);
+        }
+        catch (error) {
+            this.log.e(`Failed to run job ${jobName} immediately:`, error);
+            throw error;
+        }
+    }
+}
+
+module.exports = PulseJobScheduler;
\ No newline at end of file
diff --git a/jobServer/jobRunner/index.js b/jobServer/jobRunner/index.js
index 24e62197db3..7e6c624646c 100644
--- a/jobServer/jobRunner/index.js
+++ b/jobServer/jobRunner/index.js
@@ -1,13 +1,10 @@
-const IJobRunner = require('./IJobRunner');
-// const JobRunnerBullImpl = require('./JobRunnerBullImpl');
-const JobRunnerPulseImpl = require('./JobRunnerPulseImpl');
+const PulseJobRunner = require('./PulseJobRunner.js');
 
 /**
  * JobRunner implementation types
  * @enum {string}
  */
 const RUNNER_TYPES = {
-    // BULL: 'bull',
     PULSE: 'pulse'
 };
 
@@ -15,21 +12,18 @@ const RUNNER_TYPES = {
  * Job Runner factory
  * 
  * @param {Object} db The database connection
- * @param {string} [type='pulse'] The type of runner to create ('bull' or 'pulse')
+ * @param {string} [type='pulse'] The type of runner to create
  * @param {Object} [config={}] Configuration specific to the runner implementation
  * @param {function} Logger - Logger constructor
- * @returns {IJobRunner} An instance of the specified JobRunner implementation
+ * @returns {BaseJobRunner} An instance of BaseJobRunner with specific implementation
  * @throws {Error} If an invalid runner type is specified
  */
 function createJobRunner(db, type = RUNNER_TYPES.PULSE, config = {}, Logger) {
-
     switch (type.toLowerCase()) {
-    // case RUNNER_TYPES.BULL:
-    //     return new JobRunnerBullImpl(db, config, Logger);
     case RUNNER_TYPES.PULSE:
-        return new JobRunnerPulseImpl(db, config, Logger);
+        return new PulseJobRunner(db, config, Logger);
     default:
-        throw new Error(`Invalid runner type: ${type}. Must be one of: ${Object.values(RUNNER_TYPES).join(', ')} and implementation of ` + IJobRunner.name);
+        throw new Error(`Invalid runner type: ${type}. Must be one of: ${Object.values(RUNNER_TYPES).join(', ')}`);
     }
 }
 
diff --git a/jobServer/jobRunner/interfaces/IJobExecutor.js b/jobServer/jobRunner/interfaces/IJobExecutor.js
new file mode 100644
index 00000000000..5757cc756c8
--- /dev/null
+++ b/jobServer/jobRunner/interfaces/IJobExecutor.js
@@ -0,0 +1,42 @@
+/**
+ * Interface for job execution operations
+ */
+class IJobExecutor {
+    /**
+     * Creates and defines a new job
+     * @param {string} jobName The name of the job
+     * @param {Function} JobClass The job class to create
+     */
+    async createJob(/* jobName, JobClass */) {
+        throw new Error('Method not implemented');
+    }
+
+    /**
+     * Enable a job
+     * @param {string} jobName Name of the job to enable
+     */
+    async enableJob(/* jobName */) {
+        throw new Error('Method not implemented');
+    }
+
+    /**
+     * Disable a job
+     * @param {string} jobName Name of the job to disable
+     */
+    async disableJob(/* jobName */) {
+        throw new Error('Method not implemented');
+    }
+
+    /**
+     * Configure job retry settings
+     * @param {string} jobName Name of the job
+     * @param {Object} retryConfig Retry configuration
+     * @param {number} retryConfig.attempts Number of retry attempts
+     * @param {number} retryConfig.delay Delay between retries in ms
+     */
+    async configureRetry(/* jobName, retryConfig */) {
+        throw new Error('Method not implemented');
+    }
+}
+
+module.exports = IJobExecutor;
\ No newline at end of file
diff --git a/jobServer/jobRunner/interfaces/IJobLifecycle.js b/jobServer/jobRunner/interfaces/IJobLifecycle.js
new file mode 100644
index 00000000000..92b1bd83222
--- /dev/null
+++ b/jobServer/jobRunner/interfaces/IJobLifecycle.js
@@ -0,0 +1,21 @@
+/**
+ * Interface for job lifecycle operations
+ */
+class IJobLifecycle {
+    /**
+     * Starts the job runner
+     * @param {Object.<string, Function>} jobClasses Object containing job classes keyed by job name
+     */
+    async start(/* jobClasses */) {
+        throw new Error('Method not implemented');
+    }
+
+    /**
+     * Closes the job runner and cleans up resources
+     */
+    async close() {
+        throw new Error('Method not implemented');
+    }
+}
+
+module.exports = IJobLifecycle;
\ No newline at end of file
diff --git a/jobServer/jobRunner/interfaces/IJobScheduler.js b/jobServer/jobRunner/interfaces/IJobScheduler.js
new file mode 100644
index 00000000000..9046ec7ee30
--- /dev/null
+++ b/jobServer/jobRunner/interfaces/IJobScheduler.js
@@ -0,0 +1,35 @@
+/**
+ * Interface for job scheduling operations
+ */
+class IJobScheduler {
+    /**
+     * Schedules a job based on its configuration
+     * @param {String} name The name of the job
+     * @param {Object} scheduleConfig Schedule configuration object
+     * @param {('once'|'schedule'|'now')} scheduleConfig.type Type of schedule
+     * @param {string|Date} [scheduleConfig.value] Cron string or Date object
+     * @param {Object} [data] Data to pass to the job
+     */
+    async schedule(/* name, scheduleConfig, data */) {
+        throw new Error('Method not implemented');
+    }
+
+    /**
+     * Update job schedule
+     * @param {string} jobName Name of the job
+     * @param {string|Date} schedule New schedule (cron string or date)
+     */
+    async updateSchedule(/* jobName, schedule */) {
+        throw new Error('Method not implemented');
+    }
+
+    /**
+     * Runs a job now
+     * @param {string} jobName Name of the job to run
+     */
+    async runJobNow(/* jobName */) {
+        throw new Error('Method not implemented');
+    }
+}
+
+module.exports = IJobScheduler;
\ No newline at end of file

From 7b2162617d37561ac59bb24a256ecb7b1efe8d99 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Wed, 15 Jan 2025 18:15:08 +0530
Subject: [PATCH 20/90] README update

---
 jobServer/README.md | 964 +++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 963 insertions(+), 1 deletion(-)

diff --git a/jobServer/README.md b/jobServer/README.md
index c858d63ada6..8118adc6d0f 100644
--- a/jobServer/README.md
+++ b/jobServer/README.md
@@ -1 +1,963 @@
-# Job Server
\ No newline at end of file
+# Job Server Module
+
+A flexible, extensible job scheduling and execution system built on MongoDB with support for multiple runner implementations.
+
+## Table of Contents
+
+1. [Overview](#overview)
+2. [Architecture](#architecture)
+3. [Dependencies](#dependencies)
+4. [Installation](#installation)
+5. [Basic Usage](#basic-usage)
+6. [Collections](#collections)
+7. [Server Configuration](#server-configuration)
+8. [Job Implementation Guide](#job-implementation-guide)
+9. [Job Configuration Management](#job-configuration-management)
+10. [Parallel Processing](#parallel-processing)
+11. [Class Reference](#class-reference)
+12. [File Structure](#file-structure)
+13. [Interface Contracts](#interface-contracts)
+14. [Implementing New Runners](#implementing-new-runners)
+15. [BullMQ Implementation Guide](#bullmq-implementation-guide)
+
+## Overview
+
+The Job Server Module provides a robust framework for scheduling and executing background jobs in a distributed environment. Built with extensibility in mind, it currently supports Pulse (MongoDB-based) as the default runner, with capability to add other runners like BullMQ.
+
+### Key Features:
+
+- Flexible job scheduling (cron, one-time, immediate)
+- Job progress tracking
+- Automatic retries with exponential backoff
+- Priority-based execution
+- Concurrent job execution control
+- Dynamic configuration updates
+- Distributed execution across multiple processes
+- Extensible runner architecture
+
+## Dependencies
+
+Core dependencies:
+```json
+{
+  "@pulsecron/pulse": "^1.0.0",
+  "cron-validator": "^1.3.1",
+  "@breejs/later": "^4.1.0", // should be removed soon
+  "mongodb": "^4.0.0"
+}
+```
+
+## Installation
+
+1. Install dependencies:
+```bash
+npm install @pulsecron/pulse cron-validator @breejs/later mongodb
+```
+
+2. Set up MongoDB connection:
+```javascript
+const { MongoClient } = require('mongodb');
+const client = await MongoClient.connect('mongodb://localhost:27017');
+const db = client.db('your_database');
+```
+
+## Collections
+
+The module uses two MongoDB collections:
+
+1. `pulseJobs` (configurable name)
+   - Stores job definitions and execution state
+   - Managed by Pulse runner
+   - Schema includes job name, schedule, status, locks, etc.
+
+2. `jobConfigs`
+   - Stores job configuration overrides
+   - Used for dynamic configuration management
+   - Schema:
+     ```javascript
+     {
+       jobName: String,          // Unique job identifier
+       enabled: Boolean,         // Job enabled state
+       checksum: String,        // Job implementation checksum
+       schedule: Object,        // Schedule override
+       retry: Object,          // Retry configuration
+       runNow: Boolean,        // Trigger immediate execution
+       createdAt: Date,
+       updatedAt: Date,
+       defaultConfig: Object   // Original job configuration
+     }
+     ```
+
+## Basic Usage
+
+1. Create a job server instance:
+```javascript
+const JobServer = require('./JobServer');
+const server = await JobServer.create(common, Logger, pluginManager);
+```
+
+2. Start the server:
+```javascript
+await server.start();
+```
+
+3. Handle graceful shutdown:
+```javascript
+process.on('SIGTERM', () => server.shutdown());
+process.on('SIGINT', () => server.shutdown());
+```
+
+## Architecture
+
+### Dependency Flow
+
+```
+JobServer
+    |
+    +------------------+------------------+
+    |                  |                  |
+JobScanner         JobManager      Signal Handlers
+    |                  |
+    |          +------+------+
+    |          |      |      |
+Plugin System  Jobs JobRunner Config
+    |                  |
+    |          +------+------+
+    |          |      |      |
+JobExecutor JobScheduler JobLifecycle
+    |          |      |
+    +----------+------+
+              |
+         Runner Impl
+    (Pulse/BullMQ/etc)
+```
+
+### Core Components Flow
+
+```
+[Job Files] -> [JobScanner] -> [JobManager] -> [JobRunner] -> [Database]
+    ^            |               |               |
+    |            v               v               v
+[Plugin Jobs] [Configuration] [Execution]    [Storage]
+```
+
+### Component Roles
+
+1. **JobServer**
+   - Entry point and orchestrator
+   - Manages lifecycle of all components
+   - Handles process signals and shutdown
+
+2. **JobScanner**
+   - Discovers job files in project and plugins
+   - Validates job implementations
+   - Tracks job file changes
+
+3. **JobManager**
+   - Manages job configurations
+   - Handles dynamic updates
+   - Coordinates with runner implementation
+
+4. **JobRunner**
+   - Abstract interface for runner implementations
+   - Provides common job operations
+   - Delegates to specific implementations
+
+5. **Runner Implementation**
+   - Concrete implementation (e.g., Pulse)
+   - Handles actual job execution
+   - Manages scheduling and state
+
+## Server Configuration
+
+### Pulse Runner Configuration
+
+```javascript
+const DEFAULT_PULSE_CONFIG = {
+    processEvery: '3 seconds',     // Job check frequency
+    maxConcurrency: 1,             // Max concurrent jobs
+    defaultConcurrency: 1,         // Default concurrent jobs per type
+    lockLimit: 1,                  // Max locked jobs
+    defaultLockLimit: 1,           // Default locked jobs per type
+    defaultLockLifetime: 55 * 60 * 1000,  // Lock timeout (55 mins)
+    sort: { nextRunAt: 1, priority: -1 },
+    db: {
+        collection: 'pulseJobs',   // Collection name
+    }
+};
+```
+
+### Process-Level Configuration
+
+```javascript
+const server = await JobServer.create(common, Logger, pluginManager, {
+    runner: {
+        type: 'pulse',
+        config: {
+            maxConcurrency: 5,
+            processEvery: '5 seconds',
+            defaultLockLifetime: 30 * 60 * 1000
+        }
+    },
+    scanner: {
+        watchEnabled: true,
+        scanInterval: 60000
+    }
+});
+```
+
+## Job Implementation Guide
+
+### Basic Job Structure
+
+```javascript
+const { Job } = require('./jobServer');
+
+class MyJob extends Job {
+    // Required: Define schedule
+    getSchedule() {
+        return {
+            type: 'schedule',
+            value: '0 * * * *'  // Run hourly
+        };
+    }
+
+    // Required: Implement job logic
+    async run(db, done, progress) {
+        try {
+            // Your job logic here
+            await progress(100, 50, 'Processing...');
+            done(null, { success: true });
+        }
+        catch (error) {
+            done(error);
+        }
+    }
+
+    // Optional: Configure retries
+    getRetryConfig() {
+        return {
+            enabled: true,
+            attempts: 3,
+            delay: 60000  // 1 minute
+        };
+    }
+
+    // Optional: Set priority
+    getPriority() {
+        return this.priorities.HIGH;
+    }
+
+    // Optional: Set concurrency
+    getConcurrency() {
+        return 2;
+    }
+}
+```
+
+### Long-Running Job Example
+
+```javascript
+class DataProcessingJob extends Job {
+    getSchedule() {
+        return {
+            type: 'schedule',
+            value: '0 0 * * *'  // Daily at midnight
+        };
+    }
+
+    async run(db, done, progress) {
+        try {
+            const total = await db.collection('data').countDocuments();
+            let processed = 0;
+
+            // Process in batches
+            for (let i = 0; i < total; i += 100) {
+                const batch = await db.collection('data')
+                    .find()
+                    .skip(i)
+                    .limit(100)
+                    .toArray();
+
+                await this.processBatch(batch);
+                processed += batch.length;
+
+                // Report progress
+                await progress(
+                    total,
+                    processed,
+                    `Processed ${processed} of ${total} records`
+                );
+
+                // Extend lock if needed
+                await this.touch();
+            }
+
+            done(null, { processed });
+        }
+        catch (error) {
+            done(error);
+        }
+    }
+
+    getLockLifetime() {
+        return 2 * 60 * 60 * 1000; // 2 hours
+    }
+}
+```
+
+## Job Configuration Management
+
+### Dynamic Configuration via MongoDB
+
+Jobs can be configured dynamically by modifying the `jobConfigs` collection. The server watches for changes and applies them in real-time.
+
+### Configuration Operations
+
+1. **Enable/Disable Job**
+```javascript
+await db.collection('jobConfigs').updateOne(
+    { jobName: 'api:myJob' },
+    { $set: { enabled: false } }
+);
+```
+
+2. **Update Schedule**
+```javascript
+await db.collection('jobConfigs').updateOne(
+    { jobName: 'api:myJob' },
+    {
+        $set: {
+            schedule: {
+                type: 'schedule',
+                value: '*/30 * * * *'  // Every 30 minutes
+            }
+        }
+    }
+);
+```
+
+3. **Modify Retry Settings**
+```javascript
+await db.collection('jobConfigs').updateOne(
+    { jobName: 'api:myJob' },
+    {
+        $set: {
+            retry: {
+                enabled: true,
+                attempts: 5,
+                delay: 120000  // 2 minutes
+            }
+        }
+    }
+);
+```
+
+4. **Trigger Immediate Execution**
+```javascript
+await db.collection('jobConfigs').updateOne(
+    { jobName: 'api:myJob' },
+    { $set: { runNow: true } }
+);
+```
+
+### Configuration Lifecycle
+
+1. **Initial Configuration**
+   - Created when job is first discovered
+   - Contains default values from job implementation
+   - Stores implementation checksum
+
+2. **Configuration Updates**
+   - Applied immediately to running jobs
+   - Persisted across server restarts
+   - Validated against job interface
+
+3. **Implementation Changes**
+   - Detected via checksum comparison
+   - Triggers configuration reset
+   - Maintains enabled/disabled state
+
+## Parallel Processing
+
+### Process-Level Parallelism
+
+Multiple job server processes can run simultaneously, coordinating through MongoDB:
+
+1. **Process Configuration**
+```javascript
+node index.js
+```
+
+All the running processes will look at the jobs collection and start processing the next available job based on
+* runtime
+* lock status
+* priority
+
+2. **Lock-Based Coordination**
+   - Jobs are locked when claimed by a process
+   - Locks expire after `lockLifetime`
+   - Failed processes release locks automatically
+
+### Job-Level Concurrency
+
+Control parallel execution at the job level:
+
+1. **Global Settings**
+```javascript
+const config = {
+    maxConcurrency: 5,      // Process-wide limit
+    defaultConcurrency: 1,  // Default per job
+    lockLimit: 3,          // Max locks per process
+};
+```
+
+2. **Per-Job Settings**
+```javascript
+class MyJob extends Job {
+    getConcurrency() {
+        return 2;  // Allow 2 concurrent instances
+    }
+
+    getLockLifetime() {
+        return 30 * 60 * 1000;  // 30 minute lock
+    }
+}
+```
+
+3. **Dynamic Adjustment**
+```javascript
+await db.collection('jobConfigs').updateOne(
+    { jobName: 'api:myJob' },
+    {
+        $set: {
+            concurrency: 3,
+            lockLifetime: 45 * 60 * 1000
+        }
+    }
+);
+```
+
+### Load Distribution
+
+Jobs are distributed across processes based on:
+
+1. **Priority**
+   - Higher priority jobs run first
+   - Configurable via `getPriority()`
+
+2. **Process Capacity**
+   - Respects `maxConcurrency` limits
+   - Considers current load
+
+3. **Lock Management**
+   - Prevents duplicate execution
+   - Handles failed processes
+   - Supports job recovery
+
+## File Structure
+
+```
+jobServer/
+├── constants/
+│   └── JobPriorities.js       # Priority level definitions
+├── jobRunner/
+│   ├── interfaces/            # Core interfaces
+│   │   ├── IJobExecutor.js
+│   │   ├── IJobLifecycle.js
+│   │   └── IJobScheduler.js
+│   ├── impl/                  # Runner implementations
+│   │   ├── pulse/            # Pulse runner
+│   │   │   ├── PulseJobExecutor.js
+│   │   │   ├── PulseJobLifecycle.js
+│   │   │   └── PulseJobScheduler.js
+│   │   └── bullmq/          # Future BullMQ implementation
+│   ├── BaseJobRunner.js      # Abstract runner base
+│   ├── PulseJobRunner.js     # Pulse runner composition
+│   └── index.js             # Runner factory
+├── Job.js                    # Base job class
+├── JobManager.js            # Job management
+├── JobScanner.js           # Job discovery
+├── JobServer.js            # Main entry point
+├── JobUtils.js             # Utility functions
+└── config.js               # Default configurations
+```
+
+## Interface Contracts
+
+### IJobExecutor
+
+Handles job creation and execution control:
+
+```javascript
+class IJobExecutor {
+    async createJob(jobName, JobClass) {}
+    async enableJob(jobName) {}
+    async disableJob(jobName) {}
+    async configureRetry(jobName, retryConfig) {}
+}
+```
+
+### IJobScheduler
+
+Manages job scheduling and timing:
+
+```javascript
+class IJobScheduler {
+    async schedule(name, scheduleConfig, data) {}
+    async updateSchedule(jobName, schedule) {}
+    async runJobNow(jobName) {}
+}
+```
+
+### IJobLifecycle
+
+Controls runner lifecycle:
+
+```javascript
+class IJobLifecycle {
+    async start(jobClasses) {}
+    async close() {}
+}
+```
+
+## Implementing New Runners
+
+### Runner Implementation Steps
+
+1. **Create Implementation Directory**
+```bash
+mkdir -p jobRunner/impl/myRunner
+```
+
+2. **Implement Required Classes**
+   - MyJobExecutor extends IJobExecutor
+   - MyJobScheduler extends IJobScheduler
+   - MyJobLifecycle extends IJobLifecycle
+
+3. **Create Runner Class**
+```javascript
+class MyJobRunner extends BaseJobRunner {
+    constructor(db, config, Logger) {
+        const executor = new MyJobExecutor(/* ... */);
+        const scheduler = new MyJobScheduler(/* ... */);
+        const lifecycle = new MyJobLifecycle(/* ... */);
+        super(scheduler, executor, lifecycle);
+    }
+}
+```
+
+4. **Register Runner Type**
+```javascript
+// jobRunner/index.js
+const RUNNER_TYPES = {
+    PULSE: 'pulse',
+    MY_RUNNER: 'myRunner'
+};
+```
+
+### Example: Custom Runner Implementation
+
+```javascript
+// jobRunner/impl/myRunner/MyJobExecutor.js
+class MyJobExecutor extends IJobExecutor {
+    async createJob(jobName, JobClass) {
+        const instance = new JobClass();
+        // Initialize job in your system
+        await this.runner.define(jobName, {
+            concurrency: instance.getConcurrency(),
+            priority: instance.getPriority(),
+            // ... other settings
+        });
+    }
+
+    async enableJob(jobName) {
+        await this.runner.enable(jobName);
+    }
+
+    async disableJob(jobName) {
+        await this.runner.disable(jobName);
+    }
+
+    async configureRetry(jobName, retryConfig) {
+        await this.runner.updateJobSettings(jobName, {
+            retries: retryConfig.attempts,
+            backoff: {
+                type: 'exponential',
+                delay: retryConfig.delay
+            }
+        });
+    }
+}
+```
+
+## Error Handling & Monitoring
+
+### Job Error Handling
+
+1. **Error Types**
+```javascript
+class MyJob extends Job {
+    async run(db, done, progress) {
+        try {
+            // Transient errors (will retry)
+            throw new RetryableError('Database timeout');
+            
+            // Fatal errors (won't retry)
+            throw new FatalError('Invalid configuration');
+            
+            // Unknown errors (will retry based on config)
+            throw new Error('Unexpected error');
+        }
+        catch (error) {
+            done(error);
+        }
+    }
+}
+```
+
+2. **Retry Configuration**
+```javascript
+{
+    retry: {
+        enabled: true,
+        attempts: 3,
+        delay: 60000,
+        backoff: {
+            type: 'exponential',
+            factor: 2
+        },
+    }
+}
+```
+
+### Job Progress Monitoring
+
+1. **Progress Updates**
+```javascript
+await progress(
+    totalItems,      // Total items to process
+    processedItems,  // Items processed so far
+    'Processing batch 3/10', // Status message
+);
+```
+
+2. **Job Status Queries**
+```javascript
+const jobStatus = await db.collection('pulseJobs').findOne(
+    { name: 'api:myJob' },
+    { 
+        projection: { 
+            status: 1,
+            progress: 1,
+            lastRunAt: 1,
+            nextRunAt: 1,
+            failCount: 1,
+            lastError: 1
+        } 
+    }
+);
+```
+
+## Best Practices
+
+### Job Implementation
+
+1. **Idempotency**
+```javascript
+class IdempotentJob extends Job {
+    async run(db, done) {
+        const operationId = 'batch_20240101';
+        
+        // Check if already processed
+        const existing = await db.collection('processed')
+            .findOne({ operationId });
+            
+        if (existing) {
+            return done(null, { skipped: true });
+        }
+        
+        // Process and mark as done
+        await db.collection('processed')
+            .insertOne({ 
+                operationId,
+                processedAt: new Date()
+            });
+    }
+}
+```
+
+2. **Resource Management**
+```javascript
+class ResourceEfficientJob extends Job {
+    async run(db, done) {
+        const cursor = db.collection('large_data')
+            .find()
+            .batchSize(1000);
+            
+        try {
+            while (await cursor.hasNext()) {
+                const doc = await cursor.next();
+                await this.processDocument(doc);
+            }
+        }
+        finally {
+            await cursor.close();
+        }
+    }
+}
+```
+
+## Troubleshooting Guide
+
+### Common Issues
+
+1. **Job Not Running**
+   - Check job configuration in `jobConfigs` collection
+   - Verify schedule configuration
+   - Check if job is enabled
+   - Look for lock conflicts
+
+2. **Job Failing**
+   - Check error messages in job document
+   - Verify retry configuration
+   - Check resource availability
+   - Monitor lock timeouts
+
+3. **Performance Issues**
+   - Review concurrency settings
+   - Check database indexes
+   - Monitor memory usage
+   - Optimize batch sizes
+
+### Debugging Tools
+
+1. **Job Status Check**
+```javascript
+const status = await db.collection('pulseJobs')
+    .find(
+        { name: 'api:myJob' },
+        { 
+            sort: { lastRunAt: -1 },
+            limit: 1
+        }
+    ).toArray();
+```
+
+2. **Lock Investigation**
+```javascript
+const locks = await db.collection('pulseJobs')
+    .find({
+        lockedAt: { $exists: true },
+        lastRunAt: { 
+            $lt: new Date(Date.now() - 30 * 60 * 1000)
+        }
+    }).toArray();
+```
+
+3. **Configuration Validation**
+```javascript
+const config = await db.collection('jobConfigs')
+    .findOne({ jobName: 'api:myJob' });
+
+const isValid = JobUtils.validateConfig(config);
+```
+
+## Monitoring & Metrics
+
+### System Health Metrics
+
+1. **Job Statistics**
+```javascript
+const stats = await db.collection('pulseJobs').aggregate([
+    {
+        $group: {
+            _id: '$name',
+            totalRuns: { $sum: 1 },
+            failures: { $sum: { $cond: ['$failedAt', 1, 0] } },
+            avgDuration: { $avg: '$duration' },
+            lastRun: { $max: '$lastRunAt' }
+        }
+    }
+]).toArray();
+```
+
+2. **Process Metrics**
+```javascript
+const metrics = {
+    activeJobs: await db.collection('pulseJobs').countDocuments({ 
+        lockedAt: { $exists: true } 
+    }),
+    pendingJobs: await db.collection('pulseJobs').countDocuments({ 
+        nextRunAt: { $lte: new Date() },
+        lockedAt: { $exists: false }
+    }),
+    failedJobs: await db.collection('pulseJobs').countDocuments({ 
+        failedAt: { $exists: true } 
+    })
+};
+```
+
+### Health Checks
+
+1. **Lock Health**
+```javascript
+const staleLocks = await db.collection('pulseJobs').find({
+    lockedAt: { 
+        $lt: new Date(Date.now() - 60 * 60 * 1000) // 1 hour
+    }
+}).toArray();
+
+if (staleLocks.length > 0) {
+    // Alert: Stale locks detected
+}
+```
+
+### Alerting Integration
+
+### Logging Best Practices
+
+1. **Structured Logging**
+```javascript
+class LoggingJob extends Job {
+    async run(db, done) {
+        this.logger.i('Starting job', {
+            job: this.jobName,
+            timestamp: new Date(),
+            params: this.params
+        });
+
+        // Job logic
+
+        this.logger.i('Job completed', {
+            job: this.jobName,
+            duration: Date.now() - startTime,
+            result: result
+        });
+    }
+}
+```
+
+2. **Log Levels**
+```javascript
+{
+    d: 'Detailed debugging information',
+    i: 'General operational information',
+    w: 'Warning messages for potentially harmful situations',
+    e: 'Error events that might still allow the application to continue running'
+}
+```
+
+## Lock Extension & Progress Reporting
+
+### Lock Extension
+Long-running jobs need to periodically extend their locks to prevent expiration and avoid duplicate execution. The job system provides two ways to extend locks:
+
+1. **Automatic Extension with Progress Updates**
+When using the progress reporting function, locks are automatically extended:
+
+```javascript
+class MyLongJob extends Job {
+    async run(db, done, progress) {
+        const total = 1000;
+        for (let i = 0; i < total; i++) {
+            // Process item...
+            
+            // Automatically extends lock when reporting progress
+            await progress(
+                total,
+                i + 1,
+                `Processing item ${i + 1}/${total}`
+            );
+        }
+        done();
+    }
+}
+```
+
+## BullMQ Implementation Guide
+
+### Setup Requirements
+
+1. **Redis Connection**
+```javascript
+const { Queue, Worker } = require('bullmq');
+const Redis = require('ioredis');
+
+const connection = new Redis({
+    host: 'localhost',
+    port: 6379
+});
+```
+
+2. **Basic Structure**
+```
+jobRunner/impl/bullmq/
+├── BullMQJobExecutor.js
+├── BullMQJobLifecycle.js
+├── BullMQJobScheduler.js
+└── BullMQJobRunner.js
+```
+
+### Implementation Strategy
+
+1. **Queue Management**
+```javascript
+class BullMQJobExecutor extends IJobExecutor {
+    constructor() {
+        this.queues = new Map();
+        this.workers = new Map();
+    }
+
+    async createJob(jobName, JobClass) {
+        const queue = new Queue(jobName, { connection });
+        const worker = new Worker(jobName, async job => {
+            const instance = new JobClass();
+            return instance.run(this.db, job.done, job.progress);
+        }, { connection });
+
+        this.queues.set(jobName, queue);
+        this.workers.set(jobName, worker);
+    }
+}
+```
+
+2. **Scheduling**
+```javascript
+class BullMQJobScheduler extends IJobScheduler {
+    async schedule(name, config, data) {
+        const queue = this.queues.get(name);
+        
+        switch (config.type) {
+        case 'schedule':
+            await queue.add(name, data, {
+                repeat: { cron: config.value }
+            });
+            break;
+        case 'once':
+            await queue.add(name, data, {
+                delay: config.value - Date.now()
+            });
+            break;
+        case 'now':
+            await queue.add(name, data);
+            break;
+        }
+    }
+}
+```
+
+3. **Lifecycle Management**
+```javascript
+class BullMQJobLifecycle extends IJobLifecycle {
+    async close() {
+        await Promise.all([
+            ...this.queues.values()
+        ].map(queue => queue.close()));
+        
+        await Promise.all([
+            ...this.workers.values()
+        ].map(worker => worker.close()));
+    }
+}
+```

From 984016c8a56ac3d0d03e622cc35ea918c54ce18e Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Wed, 15 Jan 2025 18:19:09 +0530
Subject: [PATCH 21/90] fix docs

---
 jobServer/README.md | 115 ++++++++++++++++----------------------------
 1 file changed, 42 insertions(+), 73 deletions(-)

diff --git a/jobServer/README.md b/jobServer/README.md
index 8118adc6d0f..122615509cb 100644
--- a/jobServer/README.md
+++ b/jobServer/README.md
@@ -5,20 +5,24 @@ A flexible, extensible job scheduling and execution system built on MongoDB with
 ## Table of Contents
 
 1. [Overview](#overview)
-2. [Architecture](#architecture)
-3. [Dependencies](#dependencies)
-4. [Installation](#installation)
+2. [Dependencies](#dependencies)
+3. [Installation](#installation)
+4. [Collections](#collections)
 5. [Basic Usage](#basic-usage)
-6. [Collections](#collections)
+6. [Architecture](#architecture)
 7. [Server Configuration](#server-configuration)
 8. [Job Implementation Guide](#job-implementation-guide)
-9. [Job Configuration Management](#job-configuration-management)
-10. [Parallel Processing](#parallel-processing)
-11. [Class Reference](#class-reference)
+9. [Lock Extension & Progress Reporting](#lock-extension--progress-reporting)
+10. [Job Configuration Management](#job-configuration-management)
+11. [Parallel Processing](#parallel-processing)
 12. [File Structure](#file-structure)
 13. [Interface Contracts](#interface-contracts)
 14. [Implementing New Runners](#implementing-new-runners)
-15. [BullMQ Implementation Guide](#bullmq-implementation-guide)
+15. [Error Handling & Monitoring](#error-handling--monitoring)
+16. [Best Practices](#best-practices)
+17. [Troubleshooting Guide](#troubleshooting-guide)
+18. [Monitoring & Metrics](#monitoring--metrics)
+19. [BullMQ Implementation Guide](#bullmq-implementation-guide)
 
 ## Overview
 
@@ -40,10 +44,10 @@ The Job Server Module provides a robust framework for scheduling and executing b
 Core dependencies:
 ```json
 {
-  "@pulsecron/pulse": "^1.0.0",
+  "@pulsecron/pulse": "1.6.7",
   "cron-validator": "^1.3.1",
-  "@breejs/later": "^4.1.0", // should be removed soon
-  "mongodb": "^4.0.0"
+  "@breejs/later": "^4.2.0", // should be removed soon
+  "mongodb": "6.11.0"
 }
 ```
 
@@ -306,6 +310,33 @@ class DataProcessingJob extends Job {
 }
 ```
 
+## Lock Extension & Progress Reporting
+
+### Lock Extension
+Long-running jobs need to periodically extend their locks to prevent expiration and avoid duplicate execution. The job system provides two ways to extend locks:
+
+1. **Automatic Extension with Progress Updates**
+   When using the progress reporting function, locks are automatically extended:
+
+```javascript
+class MyLongJob extends Job {
+    async run(db, done, progress) {
+        const total = 1000;
+        for (let i = 0; i < total; i++) {
+            // Process item...
+            
+            // Automatically extends lock when reporting progress
+            await progress(
+                total,
+                i + 1,
+                `Processing item ${i + 1}/${total}`
+            );
+        }
+        done();
+    }
+}
+```
+
 ## Job Configuration Management
 
 ### Dynamic Configuration via MongoDB
@@ -556,41 +587,6 @@ const RUNNER_TYPES = {
 };
 ```
 
-### Example: Custom Runner Implementation
-
-```javascript
-// jobRunner/impl/myRunner/MyJobExecutor.js
-class MyJobExecutor extends IJobExecutor {
-    async createJob(jobName, JobClass) {
-        const instance = new JobClass();
-        // Initialize job in your system
-        await this.runner.define(jobName, {
-            concurrency: instance.getConcurrency(),
-            priority: instance.getPriority(),
-            // ... other settings
-        });
-    }
-
-    async enableJob(jobName) {
-        await this.runner.enable(jobName);
-    }
-
-    async disableJob(jobName) {
-        await this.runner.disable(jobName);
-    }
-
-    async configureRetry(jobName, retryConfig) {
-        await this.runner.updateJobSettings(jobName, {
-            retries: retryConfig.attempts,
-            backoff: {
-                type: 'exponential',
-                delay: retryConfig.delay
-            }
-        });
-    }
-}
-```
-
 ## Error Handling & Monitoring
 
 ### Job Error Handling
@@ -848,33 +844,6 @@ class LoggingJob extends Job {
 }
 ```
 
-## Lock Extension & Progress Reporting
-
-### Lock Extension
-Long-running jobs need to periodically extend their locks to prevent expiration and avoid duplicate execution. The job system provides two ways to extend locks:
-
-1. **Automatic Extension with Progress Updates**
-When using the progress reporting function, locks are automatically extended:
-
-```javascript
-class MyLongJob extends Job {
-    async run(db, done, progress) {
-        const total = 1000;
-        for (let i = 0; i < total; i++) {
-            // Process item...
-            
-            // Automatically extends lock when reporting progress
-            await progress(
-                total,
-                i + 1,
-                `Processing item ${i + 1}/${total}`
-            );
-        }
-        done();
-    }
-}
-```
-
 ## BullMQ Implementation Guide
 
 ### Setup Requirements

From 5acfa2f6c22b57dc813bfd9a31d65fc384dde93c Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Wed, 15 Jan 2025 18:55:47 +0530
Subject: [PATCH 22/90] remove new logger file :D

---
 api/utils/log.v2.js | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 api/utils/log.v2.js

diff --git a/api/utils/log.v2.js b/api/utils/log.v2.js
deleted file mode 100644
index e69de29bb2d..00000000000

From baf399fc150bc70f7128457edfab141fc3fd361f Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Wed, 15 Jan 2025 19:27:32 +0530
Subject: [PATCH 23/90] progress with bookmark data

---
 jobServer/Job.js                                   | 2 +-
 jobServer/jobRunner/impl/pulse/PulseJobExecutor.js | 5 ++++-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/jobServer/Job.js b/jobServer/Job.js
index 6652d954c38..3119717731b 100644
--- a/jobServer/Job.js
+++ b/jobServer/Job.js
@@ -268,7 +268,7 @@ class Job {
 
         // Touch to prevent lock expiration
         if (this.#touchMethod) {
-            await this.#touchMethod();
+            await this.#touchMethod(progress);
         }
 
         this.logger?.d(`Progress reported for job "${this.jobName}":`, progressData);
diff --git a/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js b/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js
index 1016800817e..f0d5940e6bf 100644
--- a/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js
+++ b/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js
@@ -151,7 +151,10 @@ class PulseJobExecutor extends IJobExecutor {
      */
     async #updateJobProgress(job, progressData) {
         try {
-            job.data = progressData;
+            job.attrs.data = {
+                ...job.attrs.data,
+                progressData
+            };
             await job.save();
         }
         catch (error) {

From 3c38282d979e02f3b7a512476ba93157db754a74 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Wed, 15 Jan 2025 23:33:18 +0530
Subject: [PATCH 24/90] so many tiny docs and logs

---
 jobServer/Job.js                              |  55 +++-
 jobServer/JobManager.js                       |  81 +++--
 jobServer/JobScanner.js                       |  50 +++-
 jobServer/JobServer.js                        |  95 +++---
 jobServer/config.js                           |  35 ++-
 jobServer/constants/JobPriorities.js          |  39 ++-
 jobServer/index.js                            |  92 +++---
 jobServer/jobRunner/BaseJobRunner.js          |  46 ++-
 jobServer/jobRunner/IJobRunner.js             | 145 ---------
 jobServer/jobRunner/JobRunnerBullImpl.js      |  95 ------
 jobServer/jobRunner/JobRunnerPulseImpl.js     | 280 ------------------
 jobServer/jobRunner/PulseJobRunner.js         |  48 ++-
 .../jobRunner/impl/pulse/PulseJobExecutor.js  |  54 +++-
 .../jobRunner/impl/pulse/PulseJobLifecycle.js | 117 ++++++--
 .../jobRunner/impl/pulse/PulseJobScheduler.js |  86 ++++--
 jobServer/jobRunner/index.js                  |  61 +++-
 .../jobRunner/interfaces/IJobExecutor.js      |  40 ++-
 .../jobRunner/interfaces/IJobLifecycle.js     |  21 +-
 .../jobRunner/interfaces/IJobScheduler.js     |  38 ++-
 19 files changed, 708 insertions(+), 770 deletions(-)
 delete mode 100644 jobServer/jobRunner/IJobRunner.js
 delete mode 100644 jobServer/jobRunner/JobRunnerBullImpl.js
 delete mode 100644 jobServer/jobRunner/JobRunnerPulseImpl.js

diff --git a/jobServer/Job.js b/jobServer/Job.js
index 3119717731b..1410965b1ae 100644
--- a/jobServer/Job.js
+++ b/jobServer/Job.js
@@ -1,6 +1,29 @@
 const defaultLogger = { d: console.debug, w: console.warn, e: console.error, i: console.info };
 const { JOB_PRIORITIES } = require('./constants/JobPriorities');
 
+/**
+ * @typedef {Object} Logger
+ * @property {Function} d - Debug logging function
+ * @property {Function} w - Warning logging function
+ * @property {Function} e - Error logging function
+ * @property {Function} i - Info logging function
+ */
+
+/**
+ * @typedef {Object} ProgressData
+ * @property {number} [total] - Total number of items to process
+ * @property {number} [current] - Current number of items processed
+ * @property {string} [bookmark] - Current processing stage description
+ * @property {number} [percent] - Progress percentage (0-100)
+ * @property {Date} timestamp - When the progress was reported
+ */
+
+/**
+ * @typedef {Object} ScheduleConfig
+ * @property {('once'|'schedule'|'now')} type - Type of schedule
+ * @property {(string|Date)} [value] - Schedule value (cron expression or Date)
+ */
+
 /**
  * Base class for creating jobs.
  * 
@@ -59,15 +82,16 @@ class Job {
     /** @type {string} Name of the job */
     jobName = Job.name;
 
-    /** @type {Object} Logger instance */
+    /** @type {Logger} Logger instance */
     logger;
 
     /** @type {Function|null} Touch method from job runner */
     #touchMethod = null;
 
     /** @type {Function|null} Progress method from job runner */
-    #progressMethod;
+    #progressMethod = null;
 
+    /** @type {Object.<string, string>} Available job priorities */
     priorities = JOB_PRIORITIES;
 
     /**
@@ -178,14 +202,18 @@ class Job {
 
     /**
      * Internal method to run the job and handle both Promise and callback patterns.
-     * @param {Db} db The database connection
+     * @param {MongoDb} db The MongoDB database connection
      * @param {Object} job The job instance
      * @param {Function} done Callback to be called when job completes
      * @returns {Promise<void>} A promise that resolves once the job is completed
      * @private
      */
     async _run(db, job, done) {
-        this.logger.d(`Job "${this.jobName}" is starting with database:`, db?._cly_debug?.db);
+        this.logger.d(`[Job:${this.jobName}] Starting execution`, {
+            database: db?._cly_debug?.db,
+            jobId: job?.attrs._id,
+            jobName: this.jobName
+        });
 
         try {
             const result = await new Promise((resolve, reject) => {
@@ -215,11 +243,18 @@ class Job {
                 }
             });
 
-            this.logger.i(`Job "${this.jobName}" completed successfully:`, result || '');
+            this.logger.i(`[Job:${this.jobName}] Completed successfully`, {
+                result: result || null,
+                duration: `${Date.now() - job?.attrs?.lastRunAt?.getTime()}ms`
+            });
             done(null, result);
         }
         catch (error) {
-            this.logger.e(`Job "${this.jobName}" encountered an error during execution:`, error);
+            this.logger.e(`[Job:${this.jobName}] Execution failed`, {
+                error: error.message,
+                stack: error.stack,
+                duration: `${Date.now() - job?.attrs?.lastRunAt?.getTime()}ms`
+            });
             done(error);
         }
     }
@@ -247,12 +282,12 @@ class Job {
      * @param {number} [total] Total number of stages
      * @param {number} [current] Current stage number
      * @param {string} [bookmark] Bookmark string for current stage
+     * @returns {Promise<void>} A promise that resolves once the progress is reported
      */
     async reportProgress(total, current, bookmark) {
-        // Calculate progress percentage
         const progress = total && current ? Math.min(100, Math.floor((current / total) * 100)) : undefined;
 
-        // Build progress data
+        /** @type {ProgressData} */
         const progressData = {
             total,
             current,
@@ -261,17 +296,15 @@ class Job {
             timestamp: new Date()
         };
 
-        // Update progress using runner's method if available
         if (this.#progressMethod) {
             await this.#progressMethod(progressData);
         }
 
-        // Touch to prevent lock expiration
         if (this.#touchMethod) {
             await this.#touchMethod(progress);
         }
 
-        this.logger?.d(`Progress reported for job "${this.jobName}":`, progressData);
+        this.logger?.d(`[Job:${this.jobName}] Progress update`, progressData);
     }
 
     /**
diff --git a/jobServer/JobManager.js b/jobServer/JobManager.js
index 8e2104b9229..1f3c675975e 100644
--- a/jobServer/JobManager.js
+++ b/jobServer/JobManager.js
@@ -3,36 +3,50 @@ const config = require("./config");
 const JobUtils = require('./JobUtils');
 
 /**
- * Manages job configurations and initialization.
+ * @typedef {import('../api/utils/log.js').Logger} Logger
+ * @typedef {import('mongodb').Db} MongoDb
+ * @typedef {import('./JobRunner/types').IJobRunner} IJobRunner
+ * @typedef {import('./JobRunner/PulseImpl').JobRunnerPulseImpl} JobRunnerPulseImpl
+ * 
+ * @typedef {Object} JobConfig
+ * @property {string} jobName - The unique identifier for the job
+ * @property {boolean} [enabled] - Whether the job is enabled
+ * @property {boolean} [runNow] - Whether to run the job immediately
+ * @property {Object} [schedule] - Cron or interval-based schedule configuration
+ * @property {Object} [retry] - Retry strategy configuration
+ * @property {Date} createdAt - When the config was first created
+ * @property {Date} [updatedAt] - When the config was last updated
+ * @property {string} checksum - Hash of the job implementation
+ * @property {Object} defaultConfig - Original job configuration
+ */
+
+/**
+ * Manages job configurations, scheduling, and lifecycle.
+ * Provides functionality to:
+ * - Load and initialize jobs from class definitions
+ * - Track and apply configuration changes
+ * - Enable/disable jobs
+ * - Handle job implementation updates
  */
 class JobManager {
 
-    /**
-     * The logger instance
-     * @private
-     * @type {import('../api/utils/log.js').Logger}
-     * */
+    /** @type {Logger} */
     #log;
 
-    /**
-     * The database connection
-     * @type {import('mongodb').Db | null}
-     */
+    /** @type {MongoDb} */
     #db = null;
 
-    /**
-     * The job runner instance
-     * @type {IJobRunner | JobRunnerPulseImpl |null}
-     */
+    /** @type {IJobRunner | JobRunnerPulseImpl | null} */
     #jobRunner = null;
 
-
+    /** @type {import('mongodb').Collection<JobConfig>} */
     #jobConfigsCollection;
 
     /**
      * Creates a new JobManager instance
-     * @param {Object} db Database connection
-     * @param {function} Logger - Logger constructor
+     * @param {MongoDb} db - MongoDB database connection
+     * @param {function(string): Logger} Logger - Logger factory function
+     * @throws {Error} If database connection is invalid
      */
     constructor(db, Logger) {
         this.Logger = Logger;
@@ -49,14 +63,20 @@ class JobManager {
     }
 
     /**
-     * Watches for changes in job configurations
+     * Watches for changes in job configurations and applies them in real-time
      * @private
      * @returns {Promise<void>} A promise that resolves once the watcher is started
+     * @throws {Error} If watch stream cannot be established
      */
     async #watchConfigs() {
+        this.#log.d('Initializing config change stream watcher');
         const changeStream = this.#jobConfigsCollection.watch();
 
         changeStream.on('change', async(change) => {
+            this.#log.d('Detected config change:', {
+                operationType: change.operationType,
+                jobName: change.fullDocument?.jobName
+            });
             if (change.operationType === 'update' || change.operationType === 'insert') {
                 const jobConfig = change.fullDocument;
                 await this.#applyConfig(jobConfig);
@@ -67,13 +87,8 @@ class JobManager {
     /**
      * Applies job configuration changes
      * @private
-     * @param {Object} jobConfig The job configuration to apply
-     * @param {string} jobConfig.jobName The name of the job
-     * @param {boolean} [jobConfig.runNow] Whether to run the job immediately
-     * @param {Object} [jobConfig.schedule] Schedule configuration
-     * @param {Object} [jobConfig.retry] Retry configuration
-     * @param {boolean} [jobConfig.enabled] Whether the job is enabled
-     * @returns {Promise<void>} A promise that resolves once the configuration is applied
+     * @param {JobConfig} jobConfig The job configuration to apply
+     * @throws {Error} If job runner is not initialized or configuration is invalid
      */
     async #applyConfig(jobConfig) {
         try {
@@ -82,6 +97,15 @@ class JobManager {
             }
 
             const { jobName } = jobConfig;
+            this.#log.d('Applying config changes for job:', {
+                jobName,
+                changes: {
+                    runNow: jobConfig.runNow,
+                    scheduleUpdated: !!jobConfig.schedule,
+                    retryUpdated: !!jobConfig.retry,
+                    enabledStateChanged: typeof jobConfig.enabled === 'boolean'
+                }
+            });
 
             if (jobConfig.runNow === true) {
                 await this.#jobRunner.runJobNow(jobName);
@@ -111,7 +135,12 @@ class JobManager {
             }
         }
         catch (error) {
-            this.#log.e('Error applying job configuration:', error);
+            this.#log.e('Failed to apply job configuration:', {
+                jobName: jobConfig.jobName,
+                error: error.message,
+                stack: error.stack
+            });
+            throw error; // Re-throw to allow caller to handle
         }
     }
 
diff --git a/jobServer/JobScanner.js b/jobServer/JobScanner.js
index 374172438fa..616dc988375 100644
--- a/jobServer/JobScanner.js
+++ b/jobServer/JobScanner.js
@@ -11,6 +11,7 @@ const JobUtils = require('./JobUtils');
  * @typedef {Object} JobConfig
  * @property {string} category - Category of the job (e.g., 'api' or plugin name)
  * @property {string} dir - Directory path containing job files
+ * @property {boolean} [required=false] - Whether this job directory must exist
  */
 
 /**
@@ -18,12 +19,16 @@ const JobUtils = require('./JobUtils');
  * @property {string} category - Category of the job
  * @property {string} name - Name of the job file (without extension)
  * @property {string} file - Full path to the job file
+ * @property {Date} lastModified - Last modification timestamp of the job file
  */
 
 /**
  * @typedef {Object} ScanResult
  * @property {Object.<string, string>} files - Object storing job file paths, keyed by job name
  * @property {Object.<string, Function>} classes - Object storing job class implementations, keyed by job name
+ * @property {number} totalScanned - Total number of files scanned
+ * @property {number} successfullyLoaded - Number of successfully loaded jobs
+ * @property {string[]} errors - Array of error messages from failed loads
  */
 
 /**
@@ -97,27 +102,37 @@ class JobScanner {
             const files = await fs.readdir(jobConfig.dir);
             const jobFiles = [];
 
+            this.#log.d(`Scanning directory ${jobConfig.dir} for jobs`);
+
             for (const file of files) {
                 const fullPath = path.join(jobConfig.dir, file);
                 try {
                     const stats = await fs.stat(fullPath);
-                    if (stats.isFile()) {
+                    if (stats.isFile() && file.endsWith('.js')) {
                         jobFiles.push({
                             category: jobConfig.category,
                             name: path.basename(file, '.js'),
-                            file: fullPath
+                            file: fullPath,
+                            lastModified: stats.mtime
                         });
+                        this.#log.d(`Found job file: ${file}`);
                     }
                 }
                 catch (err) {
-                    this.#log.w(`Failed to stat file ${fullPath}: ${err.message}`);
+                    this.#log.w(`Failed to stat file ${fullPath}:`, err);
                 }
             }
 
+            this.#log.i(`Found ${jobFiles.length} job files in ${jobConfig.dir}`);
             return jobFiles;
         }
         catch (err) {
-            this.#log.w(`Failed to read directory ${jobConfig.dir}: ${err.message}`);
+            const message = `Failed to read directory ${jobConfig.dir}: ${err.message}`;
+            if (jobConfig.required) {
+                this.#log.e(message, err);
+                throw new Error(message);
+            }
+            this.#log.w(message);
             return [];
         }
     }
@@ -197,7 +212,8 @@ class JobScanner {
      * @throws {Error} If plugin configuration is invalid or missing
      */
     async scan() {
-        // Initialize plugin manager
+        this.#log.i('Starting job scan...');
+
         await this.#initializeConfig();
 
         const plugins = this.#pluginManager.getPlugins(true);
@@ -205,11 +221,16 @@ class JobScanner {
             throw new Error('No valid plugins.json configuration found');
         }
 
-        this.#log.i('Scanning plugins:', plugins);
+        this.#log.i(`Found ${plugins.length} plugins to scan:`, plugins);
 
         // Reset current collections
         this.#currentFiles = {};
         this.#currentClasses = {};
+        const scanStats = {
+            totalScanned: 0,
+            successfullyLoaded: 0,
+            errors: []
+        };
 
         // Build list of directories to scan
         const jobDirs = this.#getJobDirectories(plugins);
@@ -223,13 +244,26 @@ class JobScanner {
         jobFiles.flat()
             .filter(Boolean)
             .forEach(job => {
+                scanStats.totalScanned++;
                 const loaded = this.#loadJobFile(job);
-                this.#storeLoadedJob(loaded);
+                if (loaded) {
+                    scanStats.successfullyLoaded++;
+                    this.#storeLoadedJob(loaded);
+                }
+                else {
+                    scanStats.errors.push(`Failed to load job: ${job.file}`);
+                }
             });
 
+        this.#log.i(`Job scan complete. Loaded ${scanStats.successfullyLoaded}/${scanStats.totalScanned} jobs`);
+        if (scanStats.errors.length) {
+            this.#log.w(`Failed to load ${scanStats.errors.length} jobs`);
+        }
+
         return {
             files: this.#currentFiles,
-            classes: this.#currentClasses
+            classes: this.#currentClasses,
+            ...scanStats
         };
     }
 
diff --git a/jobServer/JobServer.js b/jobServer/JobServer.js
index 5153fc1e2e6..e73415c2838 100644
--- a/jobServer/JobServer.js
+++ b/jobServer/JobServer.js
@@ -1,23 +1,33 @@
+/**
+ * @typedef {import('../api/utils/log.js').Logger} Logger
+ * @typedef {import('../plugins/pluginManager.js')} PluginManager
+ * @typedef {import('mongodb').Db} MongoDb
+ * @typedef {import('mongodb').Collection} MongoCollection
+ * @typedef {import('./JobManager')} JobManager
+ * @typedef {import('./JobScanner')} JobScanner
+ */
+
 const JobManager = require('./JobManager');
 const JobScanner = require('./JobScanner');
 
 const JOBS_CONFIG_COLLECTION = 'jobConfigs';
 /**
- * Class representing a job process.
+ * Class representing a job server that manages background job processing.
+ * Handles job scheduling, execution, and lifecycle management.
  */
 class JobServer {
 
     /**
      * The logger instance
      * @private
-     * @type {import('../api/utils/log.js').Logger}
+     * @type {Logger}
      * */
     #log;
 
     /**
      * The plugin manager instance
      * @private
-     * @type {import('../plugins/pluginManager.js')}
+     * @type {PluginManager}
      */
     #pluginManager;
 
@@ -50,14 +60,14 @@ class JobServer {
 
     /**
      * The database connection
-     * @type {import('mongodb').Db | null}
+     * @type {MongoDb | null}
      */
     #db = null;
 
     /**
      * Collection for job configurations
      * @private
-     * @type {import('mongodb').Collection}
+     * @type {MongoCollection}
      */
     #jobConfigsCollection;
 
@@ -69,11 +79,12 @@ class JobServer {
     #isShuttingDown = false;
 
     /**
-     * Creates a new JobProcess instance.
-     * @param {Object} common Countly common
-     * @param {function} Logger - Logger constructor
-     * @param {pluginManager} pluginManager - Plugin manager instance
-     * @returns {Promise<JobServer>} A promise that resolves to a new JobProcess instance.
+     * Factory method to create and initialize a new JobServer instance.
+     * @param {Object} common - Countly common utilities
+     * @param {Logger} Logger - Logger constructor
+     * @param {PluginManager} pluginManager - Plugin manager instance
+     * @returns {Promise<JobServer>} A fully initialized JobServer instance
+     * @throws {Error} If initialization fails
      */
     static async create(common, Logger, pluginManager) {
         const process = new JobServer(common, Logger, pluginManager);
@@ -95,11 +106,14 @@ class JobServer {
     }
 
     /**
-     * init the job process.
-     * @returns {Promise<void>} A promise that resolves once the job process is initialized.
+     * Initializes the job server by establishing database connections,
+     * setting up components, and configuring signal handlers.
+     * @returns {Promise<void>} A promise that resolves once the job server is initialized
+     * @throws {Error} If initialization fails
      */
     async init() {
         try {
+            this.#log.d('Initializing job server...');
             await this.#connectToDb();
 
             this.#jobManager = new JobManager(this.#db, this.Logger);
@@ -110,37 +124,37 @@ class JobServer {
 
             this.#setupSignalHandlers();
 
-            this.#log.i('Job process init successfully');
+            this.#log.i('Job server initialized successfully');
         }
         catch (error) {
-            this.#log.e('Failed to initialize job process:', error);
+            this.#log.e('Failed to initialize job server: %j', error);
             throw error;
         }
     }
 
     /**
-     * Starts the job process.
-     * @returns {Promise<void>} A promise that resolves once the job process is started.
+     * Starts the job processing server.
+     * @returns {Promise<void>} A promise that resolves once the job server is started
+     * @throws {Error} If startup fails
      */
     async start() {
         if (this.#isRunning) {
-            this.#log.w('Process is already running');
+            this.#log.w('Start requested but server is already running');
             return;
         }
 
         try {
             this.#isRunning = true;
-            this.#log.i('Starting job process');
+            this.#log.i('Starting job server...');
 
-            // Load job classes
             const { classes } = await this.#jobScanner.scan();
-            // Start job manager
+            this.#log.d('Discovered %d job classes', Object.keys(classes).length);
 
             await this.#jobManager.start(classes);
-            this.#log.i('Job process is running');
+            this.#log.i('Job server is now running and processing jobs');
         }
         catch (error) {
-            this.#log.e('Error starting job process:', error);
+            this.#log.e('Critical error during server startup: %j', error);
             await this.#shutdown(1);
         }
     }
@@ -160,52 +174,63 @@ class JobServer {
     }
 
     /**
-     * Sets up signal handlers for graceful shutdown and uncaught exceptions.
+     * Sets up process signal handlers for graceful shutdown and error handling.
+     * @private
      */
     #setupSignalHandlers() {
-        // Handle graceful shutdown
-        process.on('SIGTERM', () => this.#shutdown());
-        process.on('SIGINT', () => this.#shutdown());
+        process.on('SIGTERM', () => {
+            this.#log.i('Received SIGTERM signal');
+            this.#shutdown();
+        });
+
+        process.on('SIGINT', () => {
+            this.#log.i('Received SIGINT signal');
+            this.#shutdown();
+        });
 
-        // Handle uncaught errors
         process.on('uncaughtException', (error) => {
-            this.#log.e('Uncaught exception:', error);
+            this.#log.e('Uncaught exception in job server: %j', error);
             this.#shutdown(1);
         });
     }
 
     /**
-     * Shuts down the job process.
-     * @param {number} [exitCode=0] - The exit code to use when shutting down the process.
-     * @returns {Promise<void>} A promise that resolves once the job process is shut down.
+     * Gracefully shuts down the job server, closing connections and stopping jobs.
+     * @private
+     * @param {number} [exitCode=0] - Process exit code
+     * @returns {Promise<void>} A promise that resolves once the job server is shut down
      */
     async #shutdown(exitCode = 0) {
         if (this.#isShuttingDown) {
+            this.#log.d('Shutdown already in progress, skipping duplicate request');
             return;
         }
         this.#isShuttingDown = true;
 
+        this.#log.i('Initiating job server shutdown...');
+
         if (this.#db && typeof this.#db.close === 'function') {
+            this.#log.d('Closing database connection');
             await this.#db.close();
         }
 
         if (!this.#isRunning) {
-            this.#log.w('Shutdown called but process is not running');
+            this.#log.w('Shutdown called but server was not running');
             process.exit(exitCode);
             return;
         }
 
-        this.#log.i('Shutting down job process...');
         this.#isRunning = false;
 
         try {
             if (this.#jobManager) {
+                this.#log.d('Stopping job manager');
                 await this.#jobManager.close();
             }
-            this.#log.i('Job process shutdown complete');
+            this.#log.i('Job server shutdown completed successfully');
         }
         catch (error) {
-            this.#log.e('Error during shutdown:', error);
+            this.#log.e('Error during shutdown: %j', error);
             exitCode = 1;
         }
         finally {
diff --git a/jobServer/config.js b/jobServer/config.js
index fea20aa24c8..9941659c934 100644
--- a/jobServer/config.js
+++ b/jobServer/config.js
@@ -1,20 +1,31 @@
 /**
  * Default configuration for Pulse jobs
- * @type {import('@pulsecron/pulse').PulseConfig}
+ * @typedef {Object} PulseConfig
+ * @property {string} [name] - Name of the Pulse instance for identification
+ * @property {string} processEvery - Frequency to check for new jobs (e.g., '3 seconds', '1 minute')
+ * @property {number} maxConcurrency - Maximum number of jobs that can run concurrently across all job types
+ * @property {number} defaultConcurrency - Default concurrent jobs limit for each job type
+ * @property {number} lockLimit - Maximum number of jobs that can be locked globally
+ * @property {number} defaultLockLimit - Default lock limit for each job type
+ * @property {number} defaultLockLifetime - Time in milliseconds before a job's lock expires
+ * @property {{nextRunAt: 1|-1, priority: 1|-1}} sort - Job execution sorting criteria
+ * @property {boolean} disableAutoIndex - Whether to disable automatic MongoDB index creation
+ * @property {boolean} resumeOnRestart - Whether to resume pending jobs on service restart
+ * @property {Object} db - Database configuration options
+ * @property {string} db.collection - MongoDB collection name for storing jobs
  */
 const DEFAULT_PULSE_CONFIG = {
-    // name: 'core', // Name of the Pulse instance
-    processEvery: '3 seconds', // Frequency to check for new jobs
-    maxConcurrency: 1, // Maximum number of jobs that can run concurrently
-    defaultConcurrency: 1, // Default number of jobs that can run concurrently
-    lockLimit: 1, // Maximum number of jobs that can be locked at the same time
-    defaultLockLimit: 1, // Default number of jobs that can be locked at the same time
-    defaultLockLifetime: 55 * 60 * 1000, // 55 minutes, time in milliseconds for how long a job should be locked
-    sort: { nextRunAt: 1, priority: -1 }, // Sorting order for job execution
-    disableAutoIndex: false, // Whether to disable automatic index creation
-    resumeOnRestart: true, // Whether to resume jobs on restart
+    processEvery: '3 seconds',
+    maxConcurrency: 1,
+    defaultConcurrency: 1,
+    lockLimit: 1,
+    defaultLockLimit: 1,
+    defaultLockLifetime: 55 * 60 * 1000, // 55 minutes
+    sort: { nextRunAt: 1, priority: -1 },
+    disableAutoIndex: false,
+    resumeOnRestart: true,
     db: {
-        collection: 'pulseJobs', // MongoDB collection to store jobs
+        collection: 'pulseJobs',
     }
 };
 
diff --git a/jobServer/constants/JobPriorities.js b/jobServer/constants/JobPriorities.js
index dad4b34cd8b..f910b844db9 100644
--- a/jobServer/constants/JobPriorities.js
+++ b/jobServer/constants/JobPriorities.js
@@ -1,24 +1,49 @@
 /**
- * Job priority levels
- * @enum {string}
+ * Defines the priority levels for job processing
+ * @readonly
+ * @enum {PriorityLevel}
  */
+
+/**
+ * @typedef {'lowest' | 'low' | 'normal' | 'high' | 'highest'} PriorityLevel
+ * @description Represents the available priority levels for jobs
+ */
+
 const JOB_PRIORITIES = {
-    /** Lowest priority level */
+    /** 
+     * Lowest priority - Use for background tasks that can be delayed
+     * @type {PriorityLevel}
+     */
     LOWEST: 'lowest',
 
-    /** Low priority level */
+    /** 
+     * Low priority - Use for non-time-critical operations
+     * @type {PriorityLevel}
+     */
     LOW: 'low',
 
-    /** Normal/default priority level */
+    /** 
+     * Normal priority - Default priority level for most jobs
+     * @type {PriorityLevel}
+     */
     NORMAL: 'normal',
 
-    /** High priority level */
+    /** 
+     * High priority - Use for time-sensitive operations
+     * @type {PriorityLevel}
+     */
     HIGH: 'high',
 
-    /** Highest priority level */
+    /** 
+     * Highest priority - Use for critical operations that need immediate processing
+     * @type {PriorityLevel}
+     */
     HIGHEST: 'highest'
 };
 
+// Freeze the object to prevent modifications
+Object.freeze(JOB_PRIORITIES);
+
 module.exports = {
     JOB_PRIORITIES
 };
\ No newline at end of file
diff --git a/jobServer/index.js b/jobServer/index.js
index c551753bed4..c942a38f769 100644
--- a/jobServer/index.js
+++ b/jobServer/index.js
@@ -2,43 +2,39 @@
  * @module jobServer
  * @version 2.0
  * @author Countly
- *
+ * 
+ * @typedef {import('../api/utils/common.js')} Common
+ * @typedef {import('../plugins/pluginManager.js')} PluginManager
+ * @typedef {import('./Job')} JobType
+ * @typedef {import('./JobServer')} JobServerType
+ * 
  * @note
- *  Dependencies like common utilities and plugin manager should only be imported in this entry file
- *  and injected into the respective modules via their constructors or create methods.
- *
+ * Dependencies like common utilities and plugin manager should only be imported in this entry file
+ * and injected into the respective modules via their constructors or create methods.
+ * 
  * @description
- * This module provides the job management system for countly.
- * It handles job scheduling, execution, and management through a flexible API.
- *
- *
- * @property {typeof import('./Job')} Job - Class for creating and managing individual jobs
- * @property {typeof import('./JobServer')} JobServer - Class for running jobs in a separate process
- *
- * @throws {Error} When database connection fails during initialization
- * @throws {Error} When job definition is invalid
- *
- * @requires './Job'
- * @requires './JobServer'
- *
- * @external Common
- * @see {@import ../api/utils/common.js|common}
- *
- * @external PluginManager
- * @see {@import ../plugins/pluginManager.js|PluginManager}
- *
- * @execution
- * This module can be run directly as a standalone process:
- *
- * ```bash
- * node index.js
- * ```
- * When run directly, it will:
- * 1. Create a new JobServer instance
- * 2. Initialize it with the common utilities and plugin manager
- * 3. Start the job processing
- * 4. Handle process signals (SIGTERM, SIGINT) for graceful shutdown
- *
+ * This module serves as the entry point for Countly's job management system.
+ * It provides a robust infrastructure for:
+ * - Scheduling and executing background tasks
+ * - Managing job lifecycles and states
+ * - Handling job dependencies and priorities
+ * - Providing process isolation for job execution
+ * 
+ * @exports {Object} module.exports
+ * @property {JobType} Job - Class for creating and managing individual jobs
+ * @property {JobServerType} JobServer - Class for running jobs in a separate process
+ * 
+ * @throws {Error} DatabaseConnectionError When database connection fails during initialization
+ * @throws {Error} InvalidJobError When job definition is invalid
+ * 
+ * @example
+ * // Import and create a new job
+ * const { Job } = require('./jobs');
+ * const job = new Job({
+ *   name: 'example',
+ *   type: 'report',
+ *   schedule: '0 0 * * *'
+ * });
  */
 
 const JobServer = require('./JobServer');
@@ -46,17 +42,37 @@ const Job = require('./Job');
 
 // Start the process if this file is run directly
 if (require.main === module) {
-
     const common = require('../api/utils/common.js');
     const pluginManager = require('../plugins/pluginManager.js');
     const Logger = common.log;
+    const log = Logger('jobs:server');
+
+    log.i('Initializing job server process...');
 
     JobServer.create(common, Logger, pluginManager)
-        .then(process => process.start())
+        .then(process => {
+            log.i('Job server successfully created, starting process...');
+            return process.start();
+        })
         .catch(error => {
-            console.error('Failed to start job process:', error);
+            log.e('Critical error during job server initialization:', {
+                error: error.message,
+                stack: error.stack
+            });
             process.exit(1);
         });
+
+    // Handle process signals for graceful shutdown
+    ['SIGTERM', 'SIGINT'].forEach(signal => {
+        process.on(signal, () => {
+            log.i(`Received ${signal}, initiating graceful shutdown...`);
+            // Allow time for cleanup before force exit
+            setTimeout(() => {
+                log.e('Forced shutdown after timeout');
+                process.exit(1);
+            }, 10000);
+        });
+    });
 }
 
 module.exports = {
diff --git a/jobServer/jobRunner/BaseJobRunner.js b/jobServer/jobRunner/BaseJobRunner.js
index 84af66dcdb2..083f9beeac4 100644
--- a/jobServer/jobRunner/BaseJobRunner.js
+++ b/jobServer/jobRunner/BaseJobRunner.js
@@ -1,15 +1,35 @@
+/**
+ * @typedef {import('./interfaces/IJobScheduler')} IJobScheduler
+ * @typedef {import('./interfaces/IJobExecutor')} IJobExecutor
+ * @typedef {import('./interfaces/IJobLifecycle')} IJobLifecycle
+ * 
+ * @typedef {Object} ScheduleConfig
+ * @property {string} [cron] - Cron expression for scheduled execution
+ * @property {number} [interval] - Interval in milliseconds between executions
+ * @property {Date} [startDate] - Date when the job should start
+ * @property {Date} [endDate] - Date when the job should end
+ * 
+ * @typedef {Object} RetryConfig
+ * @property {number} attempts - Maximum number of retry attempts
+ * @property {number} backoff - Delay between retries in milliseconds
+ * @property {('exponential'|'linear'|'fixed')} strategy - Retry backoff strategy
+ */
+
 const IJobScheduler = require('./interfaces/IJobScheduler');
 const IJobExecutor = require('./interfaces/IJobExecutor');
 const IJobLifecycle = require('./interfaces/IJobLifecycle');
 
 /**
- * Base class for job runners that implements all interfaces through composition
+ * Base class for job runners that implements scheduling, execution, and lifecycle management
+ * through composition. Provides a unified interface for job management operations.
  */
 class BaseJobRunner {
     /**
-     * @param {IJobScheduler} scheduler - Scheduler implementation
-     * @param {IJobExecutor} executor - Executor implementation
-     * @param {IJobLifecycle} lifecycle - Lifecycle implementation
+     * Creates a new BaseJobRunner instance
+     * @param {IJobScheduler} scheduler - Handles job scheduling and timing
+     * @param {IJobExecutor} executor - Manages job execution and retry logic
+     * @param {IJobLifecycle} lifecycle - Controls runner lifecycle and resource management
+     * @throws {Error} If any of the implementations are invalid
      */
     constructor(scheduler, executor, lifecycle) {
         if (!(scheduler instanceof IJobScheduler)) {
@@ -28,11 +48,12 @@ class BaseJobRunner {
     }
 
     /**
-     * Schedules a job to run
-     * @param {string} name Job name
-     * @param {Object} scheduleConfig Schedule configuration
-     * @param {Object} [data] Optional data to pass to the job
-     * @returns {Promise<void>} A promise that resolves once the job is scheduled
+     * Schedules a job for execution based on the provided configuration
+     * @param {string} name - Unique identifier for the job
+     * @param {ScheduleConfig} scheduleConfig - Configuration for when the job should run
+     * @param {Object} [data] - Optional data passed to the job during execution
+     * @returns {Promise<void>} Resolves when scheduling is complete
+     * @throws {Error} If scheduling fails or job doesn't exist
      */
     async schedule(name, scheduleConfig, data) {
         return this.scheduler.schedule(name, scheduleConfig, data);
@@ -96,9 +117,10 @@ class BaseJobRunner {
     }
 
     /**
-     * Starts the job runner
-     * @param {Object.<string, Function>} jobClasses Job classes to register
-     * @returns {Promise<void>} A promise that resolves once the job runner is started
+     * Initializes and starts the job runner with the provided job implementations
+     * @param {Object.<string, Function>} jobClasses - Map of job names to their implementing classes
+     * @returns {Promise<void>} Resolves when all jobs are registered and the runner is ready
+     * @throws {Error} If initialization fails or invalid job classes are provided
      */
     async start(jobClasses) {
         return this.lifecycle.start(jobClasses);
diff --git a/jobServer/jobRunner/IJobRunner.js b/jobServer/jobRunner/IJobRunner.js
deleted file mode 100644
index 6bf89da7863..00000000000
--- a/jobServer/jobRunner/IJobRunner.js
+++ /dev/null
@@ -1,145 +0,0 @@
-/**
- * Interface for job runner implementations
- */
-class IJobRunner {
-
-    db;
-
-    config;
-
-    log;
-
-    /**
-     *@param {Object} db Database connection
-     * @param {Object} config Configuration
-     * @param {function} Logger - Logger constructor
-     */
-    constructor(db, config, Logger) {
-        this.log = Logger('jobs:runner');
-        this.db = db;
-        this.config = config;
-    }
-
-    /**
-     * Defines a new job
-     * @param {String} jobName The name of the job
-     * @param {function} jobRunner The job runner function
-     * @param {Object} [jobOptions=null] The job options
-     * @returns {Promise<void>} A promise that resolves once the job is defined
-     */
-    async createJob(/*jobName, jobRunner, jobOptions=null*/) {
-        throw new Error('Method not implemented');
-    }
-
-    /**
-     * Schedules a job based on its configuration
-     * @param {String} name The name of the job
-     * @param {Object} scheduleConfig Schedule configuration object
-     * @param {('once'|'schedule'|'now')} scheduleConfig.type Type of schedule
-     * @param {string|Date} [scheduleConfig.value] Cron string or Date object
-     * @param {Object} [data] Data to pass to the job
-     * @returns {Promise<void>} A promise that resolves once the job is scheduled
-     */
-    async schedule(/*name, scheduleConfig, data*/) {
-        throw new Error('Method not implemented');
-    }
-
-    /**
-     * Runs a job once
-     * @param {String} name The name of the job
-     * @param {Date} date The schedule
-     * @param {Object} data Data to pass to the job
-     * @returns {Promise<void>} A promise that resolves once the job is scheduled
-     */
-    async once(/*name, date, data*/) {
-        throw new Error('Method not implemented');
-    }
-
-    /**
-     * Runs a job now
-     * @param {String} name The name of the job
-     * @param {Object} data Data to pass to the job
-     * @returns {Promise<void>} A promise that resolves once the job is run
-     */
-    async now(/*name, data*/) {
-        throw new Error('Method not implemented');
-    }
-
-    /**
-     * Starts the job runner
-     * @param {Object.<string, Function>} jobClasses Object containing job classes keyed by job name
-     * @returns {Promise<void>} A promise that resolves once the runner is started
-     */
-    async start(/*jobClasses*/) {
-        throw new Error('Method not implemented');
-    }
-
-    /**
-     * Closes the job runner and cleans up resources
-     * @returns {Promise<void>} A promise that resolves once the runner is closed
-     */
-    async close() {
-        throw new Error('Method not implemented');
-    }
-
-    /**
-     * Enable a job
-     * @param {string} jobName Name of the job to enable
-     * @returns {Promise<void>} A promise that resolves once the job is enabled
-     */
-    async enableJob(/*jobName*/) {
-        throw new Error('Method not implemented');
-    }
-
-    /**
-     * Disable a job
-     * @param {string} jobName Name of the job to disable
-     * @returns {Promise<void>} A promise that resolves once the job is disabled
-     */
-    async disableJob(/*jobName*/) {
-        throw new Error('Method not implemented');
-    }
-
-    /**
-     * Run a job immediately
-     * @param {string} jobName Name of the job to run
-     * @returns {Promise<void>} A promise that resolves once the job is triggered
-     */
-    async runJobNow(/* jobName */) {
-        throw new Error('runJobNow must be implemented');
-    }
-
-    /**
-     * Update job schedule
-     * @param {string} jobName Name of the job
-     * @param {string|Date} schedule New schedule (cron string or date)
-     * @returns {Promise<void>} A promise that resolves once schedule is updated
-     */
-    async updateSchedule(/* jobName, schedule */) {
-        throw new Error('updateSchedule must be implemented');
-    }
-
-    /**
-     * Configure job retry settings
-     * @param {string} jobName Name of the job
-     * @param {Object} retryConfig Retry configuration
-     * @param {number} retryConfig.attempts Number of retry attempts
-     * @param {number} retryConfig.delay Delay between retries in ms
-     * @returns {Promise<void>} A promise that resolves once retry is configured
-     */
-    async configureRetry(/* jobName, retryConfig */) {
-        throw new Error('configureRetry must be implemented');
-    }
-
-    /**
-     * Maps generic priority to runner-specific priority
-     * @protected
-     * @param {string} priority Generic priority from JOB_PRIORITIES
-     * returns {any} Runner-specific priority value
-     */
-    _mapPriority(/* priority */) {
-        throw new Error('_mapPriority must be implemented');
-    }
-}
-
-module.exports = IJobRunner;
\ No newline at end of file
diff --git a/jobServer/jobRunner/JobRunnerBullImpl.js b/jobServer/jobRunner/JobRunnerBullImpl.js
deleted file mode 100644
index 826e0ae7497..00000000000
--- a/jobServer/jobRunner/JobRunnerBullImpl.js
+++ /dev/null
@@ -1,95 +0,0 @@
-const IJobRunner = require('./IJobRunner');
-const { Queue, Worker } = require('bullmq');
-
-/**
- * BullMQ implementation of the job runner
- */
-class JobRunnerBullImpl extends IJobRunner {
-    /**
-     * Map of queue names to BullMQ Queue instances
-     * @type {Map<string, import('bullmq').Queue>}
-     */
-    #queues = new Map();
-
-    /**
-     * Map of queue names to BullMQ Worker instances
-     * @type {Map<string, import('bullmq').Worker>}
-     */
-    #workers = new Map();
-
-    #bullConfig;
-
-    #redisConnection;
-
-    #Queue;
-
-    #Worker;
-
-    /**
-     * Creates a new BullMQ job runner
-     * @param {Object} db Database connection
-     * @param {Object} config Configuration object
-     */
-    constructor(db, config) {
-        super(db, config);
-        this.#Queue = Queue;
-        this.#Worker = Worker;
-        this.#redisConnection = config.redis;
-        this.#bullConfig = config.bullConfig;
-    }
-
-    /**
-     * @param {Object.<string, Function>} jobClasses Object containing job classes keyed by job name
-     */
-    async start(jobClasses) {
-
-        // Create queues and workers for each job
-        for (const [name, JobClass] of Object.entries(jobClasses)) {
-            // Create queue
-            const queue = new this.#Queue(name, {
-                connection: this.#redisConnection,
-                ...this.#bullConfig
-            });
-            this.#queues.set(name, queue);
-
-            // Create worker
-            const worker = new this.#Worker(name, async(job) => {
-                const instance = new JobClass(name);
-                await instance.run(job);
-            }, {
-                connection: this.#redisConnection,
-                ...this.#bullConfig
-            });
-            this.#workers.set(name, worker);
-
-            // Handle worker events
-            worker.on('completed', (job) => {
-                console.log(`Job ${job.id} completed`);
-            });
-
-            worker.on('failed', (job, err) => {
-                console.error(`Job ${job.id} failed:`, err);
-            });
-        }
-    }
-
-    /**
-     * Closes all queues and workers
-     * @returns {Promise<void>} A promise that resolves once all queues and workers are closed
-     */
-    async close() {
-        // Close all workers
-        for (const worker of this.#workers.values()) {
-            await worker.close();
-        }
-        this.#workers.clear();
-
-        // Close all queues
-        for (const queue of this.#queues.values()) {
-            await queue.close();
-        }
-        this.#queues.clear();
-    }
-}
-
-module.exports = JobRunnerBullImpl;
\ No newline at end of file
diff --git a/jobServer/jobRunner/JobRunnerPulseImpl.js b/jobServer/jobRunner/JobRunnerPulseImpl.js
deleted file mode 100644
index e25bb74afdc..00000000000
--- a/jobServer/jobRunner/JobRunnerPulseImpl.js
+++ /dev/null
@@ -1,280 +0,0 @@
-const IJobRunner = require('./IJobRunner');
-const { Pulse, JobPriority } = require('@pulsecron/pulse');
-const {isValidCron} = require('cron-validator');
-const { JOB_PRIORITIES } = require('../constants/JobPriorities');
-
-/**
- * Pulse implementation of the job runner
- */
-class JobRunnerPulseImpl extends IJobRunner {
-    /**
-     * The Pulse runner instance
-     * @type {import('@pulsecron/pulse').Pulse}
-     */
-    #pulseRunner;
-
-    /** @type {Map<string, Object>} Store job schedules until Pulse is started */
-    #pendingSchedules = new Map();
-
-    /**
-     * Creates a new Pulse job runner
-     * @param {Object} db Database connection
-     * @param {Object} config Configuration object
-     * @param {function} Logger - Logger constructor
-     */
-    constructor(db, config, Logger) {
-        super(db, config, Logger);
-        this.log = Logger('jobs:runner:pulse');
-        this.#pulseRunner = new Pulse({
-            ...config,
-            mongo: this.db,
-        });
-    }
-
-    /**
-     * Starts the Pulse runner and schedules any pending jobs
-     */
-    async start() {
-        if (!this.#pulseRunner) {
-            throw new Error('Pulse runner not initialized');
-        }
-
-        await this.#pulseRunner.start();
-        this.log.i('Pulse runner started');
-
-        // Schedule all pending jobs
-        for (const [jobName, scheduleConfig] of this.#pendingSchedules) {
-            try {
-                await this.#scheduleJob(jobName, scheduleConfig);
-            }
-            catch (error) {
-                this.log.e(`Failed to schedule job ${jobName}:`, error);
-            }
-        }
-
-        this.#pendingSchedules.clear();
-    }
-
-    /**
-     * Updates job progress in Pulse
-     * @param {Object} job Pulse job instance
-     * @param {Object} progressData Progress data to store
-     * @private
-     */
-    async #updateJobProgress(job, progressData) {
-        job.data = progressData;
-        await job.save();
-    }
-
-    /**
-     * Creates and defines a new job
-     * @param {string} jobName The name of the job
-     * @param {Function} JobClass The job class to create
-     * @returns {Promise<void>} A promise that resolves once the job is created
-     */
-    async createJob(jobName, JobClass) {
-        try {
-            const instance = new JobClass(jobName);
-            // instance.setLogger(this.log);
-            instance.setJobName(jobName);
-
-            // Get job configurations
-            const retryConfig = instance.getRetryConfig();
-            const priority = this._mapPriority(instance.getPriority());
-            const concurrency = instance.getConcurrency();
-            const lockLifetime = instance.getLockLifetime();
-
-            this.#pulseRunner.define(
-                jobName,
-                async(job, done) => {
-                    instance._setTouchMethod(job.touch.bind(job));
-                    instance._setProgressMethod(
-                        async(progressData) => this.#updateJobProgress(job, progressData)
-                    );
-                    return instance._run(this.db, job, done);
-                },
-                {
-                    priority,
-                    concurrency,
-                    lockLifetime,
-                    shouldSaveResult: true,
-                    attempts: retryConfig?.enabled ? retryConfig.attempts : 1,
-                    backoff: retryConfig?.enabled ? {
-                        type: 'exponential',
-                        delay: retryConfig.delay
-                    } : undefined
-                }
-            );
-
-            const scheduleConfig = instance.getSchedule();
-            this.#pendingSchedules.set(jobName, scheduleConfig);
-            this.log.d(`Job ${jobName} defined successfully`);
-        }
-        catch (error) {
-            this.log.e(`Failed to create job ${jobName}:`, error);
-            // Don't throw - allow other jobs to continue
-        }
-    }
-
-    /**
-     * Internal method to schedule a job
-     * @param {string} name The name of the job to schedule
-     * @param {Object} scheduleConfig Schedule configuration object
-     * @param {('once'|'schedule'|'now')} scheduleConfig.type Type of schedule
-     * @param {string|Date} [scheduleConfig.value] Cron string or Date object
-     * @param {Object} [data] Data to pass to the job
-     * @private
-     */
-    async #scheduleJob(name, scheduleConfig, data) {
-        switch (scheduleConfig.type) {
-        case 'schedule':
-            if (!isValidCron(scheduleConfig.value)) {
-                throw new Error('Invalid cron schedule');
-            }
-            await this.#pulseRunner.every(scheduleConfig.value, name, data);
-            this.log.d(`Job ${name} scheduled with cron: ${scheduleConfig.value}`);
-            break;
-
-        case 'once':
-            if (!(scheduleConfig.value instanceof Date)) {
-                throw new Error('Invalid date for one-time schedule');
-            }
-            await this.#pulseRunner.schedule(scheduleConfig.value, name, data);
-            this.log.d(`Job ${name} scheduled for: ${scheduleConfig.value}`);
-            break;
-
-        case 'now':
-            await this.#pulseRunner.now(name, data);
-            this.log.d(`Job ${name} scheduled to run immediately`);
-            break;
-
-        default:
-            throw new Error(`Invalid schedule type: ${scheduleConfig.type}`);
-        }
-    }
-
-    /**
-     * Closes the Pulse runner
-     * @returns {Promise<void>} A promise that resolves once the runner is closed
-     */
-    async close() {
-        if (this.#pulseRunner) {
-            await this.#pulseRunner.close();
-            this.#pulseRunner = null;
-        }
-    }
-
-    /**
-     * Enable a job
-     * @param {string} jobName Name of the job to enable
-     * @returns {Promise<void>} A promise that resolves once the job is enabled
-     */
-    async enableJob(jobName) {
-        try {
-            await this.#pulseRunner.enable({ name: jobName });
-            this.log.i(`Job ${jobName} enabled`);
-        }
-        catch (error) {
-            this.log.e(`Failed to enable job ${jobName}:`, error);
-            throw error;
-        }
-    }
-
-    /**
-     * Disable a job
-     * @param {string} jobName Name of the job to disable
-     * @returns {Promise<void>} A promise that resolves once the job is disabled
-     */
-    async disableJob(jobName) {
-        try {
-            await this.#pulseRunner.disable({ name: jobName });
-            this.log.i(`Job ${jobName} disabled`);
-        }
-        catch (error) {
-            this.log.e(`Failed to disable job ${jobName}:`, error);
-            throw error;
-        }
-    }
-
-    /**
-     * Triggers immediate execution of a job
-     * @param {string} jobName Name of the job to run
-     * @returns {Promise<void>} A promise that resolves when the job is triggered
-     */
-    async runJobNow(jobName) {
-        try {
-            await this.#pulseRunner.now({ name: jobName });
-            this.log.i(`Job ${jobName} triggered for immediate execution`);
-        }
-        catch (error) {
-            this.log.e(`Failed to run job ${jobName} immediately:`, error);
-            throw error;
-        }
-    }
-
-    /**
-     * Updates the schedule of an existing job
-     * @param {string} jobName Name of the job to update
-     * @param {Object} schedule New schedule configuration
-     * @returns {Promise<void>} A promise that resolves when the schedule is updated
-     */
-    async updateSchedule(jobName, schedule) {
-        try {
-            await this.#pulseRunner.reschedule({ name: jobName }, schedule);
-            this.log.i(`Schedule updated for job ${jobName}`);
-        }
-        catch (error) {
-            this.log.e(`Failed to update schedule for job ${jobName}:`, error);
-            throw error;
-        }
-    }
-
-    /**
-     * Configures retry settings for a job
-     * @param {string} jobName Name of the job
-     * @param {Object} retryConfig Retry configuration
-     * @param {number} retryConfig.attempts Number of retry attempts
-     * @param {number} retryConfig.delay Delay between retries in milliseconds
-     * @returns {Promise<void>} A promise that resolves when retry config is updated
-     */
-    async configureRetry(jobName, retryConfig) {
-        try {
-            await this.#pulseRunner.updateOne(
-                { name: jobName },
-                {
-                    $set: {
-                        attempts: retryConfig.attempts,
-                        backoff: {
-                            delay: retryConfig.delay,
-                            type: 'fixed'
-                        }
-                    }
-                }
-            );
-            this.log.i(`Retry configuration updated for job ${jobName}`);
-        }
-        catch (error) {
-            this.log.e(`Failed to configure retry for job ${jobName}:`, error);
-            throw error;
-        }
-    }
-
-    /**
-     * Maps generic priority to runner-specific priority
-     * @protected
-     * @param {string} priority Generic priority from JOB_PRIORITIES
-     * @returns {any} Runner-specific priority value
-     */
-    _mapPriority(priority) {
-        const priorityMap = {
-            [JOB_PRIORITIES.LOWEST]: JobPriority.lowest,
-            [JOB_PRIORITIES.LOW]: JobPriority.low,
-            [JOB_PRIORITIES.NORMAL]: JobPriority.normal,
-            [JOB_PRIORITIES.HIGH]: JobPriority.high,
-            [JOB_PRIORITIES.HIGHEST]: JobPriority.highest
-        };
-        return priorityMap[priority] || JobPriority.normal;
-    }
-}
-
-module.exports = JobRunnerPulseImpl;
\ No newline at end of file
diff --git a/jobServer/jobRunner/PulseJobRunner.js b/jobServer/jobRunner/PulseJobRunner.js
index 03875504e38..1478cf33bca 100644
--- a/jobServer/jobRunner/PulseJobRunner.js
+++ b/jobServer/jobRunner/PulseJobRunner.js
@@ -5,23 +5,59 @@ const PulseJobExecutor = require('./impl/pulse/PulseJobExecutor');
 const PulseJobLifecycle = require('./impl/pulse/PulseJobLifecycle');
 
 /**
- * Pulse-specific implementation of the job runner using BaseJobRunner composition
+ * @typedef {import('@pulsecron/pulse').Pulse} PulseInstance
+ * @typedef {import('./BaseJobRunner')} BaseJobRunner
+ * @typedef {import('./impl/pulse/PulseJobScheduler')} PulseJobScheduler
+ * @typedef {import('./impl/pulse/PulseJobExecutor')} PulseJobExecutor
+ * @typedef {import('./impl/pulse/PulseJobLifecycle')} PulseJobLifecycle
+ * @typedef {import('mongodb').Db} Mongodb
+ */
+
+/**
+ * Pulse-specific implementation of the job runner using BaseJobRunner composition.
+ * This class provides a concrete implementation of job running capabilities using
+ * the Pulse framework for scheduling and executing jobs.
+ * 
+ * @extends {BaseJobRunner}
  */
 class PulseJobRunner extends BaseJobRunner {
+    /**
+     * @type {PulseInstance}
+     * @private
+     */
+    #pulseRunner;
+
+    /**
+     * @type {Logger}
+     * @private
+     */
+    log;
+
     /**
      * Creates a new Pulse job runner with all required implementations
-     * @param {Object} db Database connection
-     * @param {Object} config Configuration object
-     * @param {function} Logger - Logger constructor
+     * 
+     * @param {Mongodb} db - MongoDB database connection
+     * @param {Object} config - Configuration object for Pulse
+     * @param {string} [config.name] - Name of the Pulse instance
+     * @param {Object} [config.options] - Additional Pulse configuration options
+     * @param {function} Logger - Logger constructor function
+     * @throws {Error} If required dependencies are not provided
      */
     constructor(db, config, Logger) {
         const log = Logger('jobs:runner:pulse');
+        log.d('Initializing PulseJobRunner');
+
+        if (!db || !config || !Logger) {
+            log.e('Missing required dependencies');
+            throw new Error('Missing required dependencies for PulseJobRunner');
+        }
 
         // Create the Pulse instance that will be shared across implementations
         const pulseRunner = new Pulse({
             ...config,
             mongo: db,
         });
+        log.i('Created Pulse instance', { config: { ...config, mongo: '[Connection]' } });
 
         // Create implementations with shared pulseRunner instance
         const scheduler = new PulseJobScheduler(pulseRunner, log);
@@ -32,7 +68,9 @@ class PulseJobRunner extends BaseJobRunner {
         super(scheduler, executor, lifecycle);
 
         this.log = log;
-        this.pulseRunner = pulseRunner;
+        this.#pulseRunner = pulseRunner;
+
+        log.i('PulseJobRunner initialized successfully');
     }
 }
 
diff --git a/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js b/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js
index f0d5940e6bf..a55c3bc1fd1 100644
--- a/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js
+++ b/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js
@@ -3,14 +3,22 @@ const { JobPriority } = require('@pulsecron/pulse');
 const { JOB_PRIORITIES } = require('../../../constants/JobPriorities');
 
 /**
- * Pulse implementation of job executor
+ * @typedef {import('@pulsecron/pulse').JobPriority} JobPriority
+ * @typedef {import('@pulsecron/pulse').PulseRunner} PulseRunner
+ * @typedef {import('@pulsecron/pulse').Job} PulseJob
+ * @typedef {import('mongodb').Db} MongoDB
+ */
+
+/**
+ * Pulse implementation of job executor that handles job lifecycle management
+ * @implements {IJobExecutor}
  */
 class PulseJobExecutor extends IJobExecutor {
     /**
      * Creates a new PulseJobExecutor instance
-     * @param {Object} pulseRunner The Pulse runner instance
-     * @param {Object} db Database connection
-     * @param {Object} logger Logger instance
+     * @param {PulseRunner} pulseRunner - The Pulse runner instance for job management
+     * @param {MongoDB} db - MongoDB database connection
+     * @param {Logger} logger - Logger instance for operational logging
      */
     constructor(pulseRunner, db, logger) {
         super();
@@ -21,12 +29,14 @@ class PulseJobExecutor extends IJobExecutor {
     }
 
     /**
-     * Creates and registers a new job
-     * @param {string} jobName Name of the job
-     * @param {Function} JobClass Job class implementation
+     * Creates and registers a new job with the Pulse runner
+     * @param {string} jobName - Unique identifier for the job
+     * @param {Constructor} JobClass - Job class constructor
+     * @throws {Error} When job creation or registration fails
      * @returns {Promise<void>} A promise that resolves once the job is created
      */
     async createJob(jobName, JobClass) {
+        this.log.d(`Attempting to create job: ${jobName}`);
         try {
             const instance = new JobClass(jobName);
             instance.setJobName(jobName);
@@ -61,9 +71,14 @@ class PulseJobExecutor extends IJobExecutor {
             const scheduleConfig = instance.getSchedule();
             this.pendingSchedules.set(jobName, scheduleConfig);
             this.log.d(`Job ${jobName} defined successfully`);
+
+            this.log.d(`Configuring job ${jobName} with priority: ${priority}, concurrency: ${concurrency}, lockLifetime: ${lockLifetime}`);
+
+            this.log.i(`Job ${jobName} created and configured successfully with retry attempts: ${retryConfig?.attempts || 1}`);
         }
         catch (error) {
-            this.log.e(`Failed to create job ${jobName}:`, error);
+            this.log.e(`Failed to create job ${jobName}`, { error, stack: error.stack });
+            throw error; // Propagate error for proper handling
         }
     }
 
@@ -143,31 +158,38 @@ class PulseJobExecutor extends IJobExecutor {
     }
 
     /**
-     * Updates job progress data
+     * Updates progress information for a running job
      * @private
-     * @param {Object} job Job instance
-     * @param {Object} progressData Progress data to store
+     * @param {PulseJob} job - Pulse job instance
+     * @param {Object} progressData - Progress information to store
+     * @throws {Error} When progress update fails
      * @returns {Promise<void>} A promise that resolves once progress is updated
      */
     async #updateJobProgress(job, progressData) {
+        this.log.d(`Updating progress for job ${job.attrs.name}`, { progressData });
         try {
             job.attrs.data = {
                 ...job.attrs.data,
                 progressData
             };
             await job.save();
+            this.log.d(`Progress updated successfully for job ${job.attrs.name}`);
         }
         catch (error) {
-            this.log.e(`Failed to update job progress: ${error.message}`);
-            // Consider whether to throw
+            this.log.e(`Failed to update job progress for ${job.attrs.name}`, {
+                error,
+                stack: error.stack,
+                jobId: job.attrs._id
+            });
+            throw error; // Propagate error for proper handling
         }
     }
 
     /**
-     * Maps generic priority to Pulse-specific priority with validation
+     * Maps generic priority levels to Pulse-specific priority values
      * @private
-     * @param {string} priority Generic priority from JOB_PRIORITIES
-     * @returns {JobPriority} Pulse-specific priority value
+     * @param {string} priority - Generic priority from JOB_PRIORITIES
+     * @returns {JobPriority} Mapped Pulse priority
      */
     #mapPriority(priority) {
         const priorityMap = {
diff --git a/jobServer/jobRunner/impl/pulse/PulseJobLifecycle.js b/jobServer/jobRunner/impl/pulse/PulseJobLifecycle.js
index e9955f4334c..30d7ad9f605 100644
--- a/jobServer/jobRunner/impl/pulse/PulseJobLifecycle.js
+++ b/jobServer/jobRunner/impl/pulse/PulseJobLifecycle.js
@@ -1,7 +1,33 @@
 const IJobLifecycle = require('../../interfaces/IJobLifecycle');
 
 /**
- * Pulse implementation of job lifecycle management
+ * @typedef {import('../../interfaces/IJobLifecycle')} IJobLifecycle
+ * @typedef {import('../PulseRunner')} PulseRunner
+ * @typedef {import('../JobExecutor')} JobExecutor
+ * @typedef {import('../JobScheduler')} JobScheduler
+ */
+
+/**
+ * @typedef {Object} ScheduleConfig
+ * @property {string} cron - The cron expression for job scheduling
+ * @property {Object} [options] - Additional scheduling options
+ * @property {boolean} [options.immediate] - Whether to run the job immediately
+ * @property {string} [options.timezone] - Timezone for the cron schedule
+ * @property {number} [options.retryAttempts] - Number of retry attempts on failure
+ */
+
+/**
+ * @typedef {Object} JobClass
+ * @property {string} name - The name of the job class
+ * @property {Function} execute - The main execution method
+ * @property {Object} [config] - Optional job configuration
+ */
+
+/**
+ * Pulse implementation of job lifecycle management.
+ * Handles the initialization, scheduling, and cleanup of jobs in the Pulse system.
+ * 
+ * @implements {IJobLifecycle}
  */
 class PulseJobLifecycle extends IJobLifecycle {
     /**
@@ -20,42 +46,95 @@ class PulseJobLifecycle extends IJobLifecycle {
     }
 
     /**
-     * Starts the job runner and schedules pending jobs
-     * @param {Object.<string, Function>} jobClasses Job classes to register
-     * @returns {Promise<void>} A promise that resolves once the runner is started
+     * Initializes and starts the job runner, then schedules all pending jobs.
+     * This method performs the following steps:
+     * 1. Validates the pulse runner initialization
+     * 2. Starts the pulse runner instance
+     * 3. Schedules all pending jobs from the executor
+     * 
+     * @param {Object.<string, JobClass>} jobClasses - Map of job names to their implementing classes
+     * @returns {Promise<void>} Resolves when the runner is started and all jobs are scheduled
+     * @throws {Error} If pulse runner is not initialized or scheduling fails
      */
     async start(/* jobClasses */) {
         if (!this.pulseRunner) {
+            this.log.e('Lifecycle start failed: Pulse runner not initialized', {
+                component: 'PulseJobLifecycle',
+                method: 'start'
+            });
             throw new Error('Pulse runner not initialized');
         }
 
-        await this.pulseRunner.start();
-        this.log.i('Pulse runner started');
+        try {
+            await this.pulseRunner.start();
+            this.log.i('Pulse runner initialization complete', {
+                component: 'PulseJobLifecycle',
+                status: 'started'
+            });
 
-        // Schedule all pending jobs from the executor
-        for (const [jobName, scheduleConfig] of this.executor.pendingSchedules) {
-            try {
-                await this.scheduler.schedule(jobName, scheduleConfig);
-            }
-            catch (error) {
-                this.log.e(`Failed to schedule job ${jobName}:`, error);
+            const pendingJobCount = this.executor.pendingSchedules.size;
+            this.log.i('Beginning job schedule processing', {
+                component: 'PulseJobLifecycle',
+                pendingJobs: pendingJobCount
+            });
+
+            // Schedule all pending jobs from the executor
+            for (const [jobName, scheduleConfig] of this.executor.pendingSchedules) {
+                try {
+                    await this.scheduler.schedule(jobName, scheduleConfig);
+                    this.log.d('Job scheduled successfully', {
+                        component: 'PulseJobLifecycle',
+                        job: jobName,
+                        config: scheduleConfig
+                    });
+                }
+                catch (error) {
+                    this.log.e('Job scheduling failed', {
+                        component: 'PulseJobLifecycle',
+                        job: jobName,
+                        error: error.message,
+                        stack: error.stack
+                    });
+                }
             }
-        }
 
-        this.executor.pendingSchedules.clear();
+            this.executor.pendingSchedules.clear();
+            this.log.i('Job scheduling process complete', {
+                component: 'PulseJobLifecycle',
+                totalProcessed: pendingJobCount
+            });
+        }
+        catch (error) {
+            this.log.e('Pulse runner start failed', {
+                component: 'PulseJobLifecycle',
+                error: error.message,
+                stack: error.stack
+            });
+            throw error;
+        }
     }
 
     /**
-     * Closes the job runner and cleans up resources
-     * @returns {Promise<void>} A promise that resolves once the runner is closed
+     * Gracefully shuts down the job runner and cleans up resources.
+     * Ensures all running jobs are properly terminated and resources are released.
+     * 
+     * @returns {Promise<void>} Resolves when the runner is successfully closed
+     * @throws {Error} If the shutdown process fails
      */
     async close() {
         try {
             await this.pulseRunner.close();
-            this.log.i('Pulse runner closed');
+            this.log.i('Pulse runner shutdown complete', {
+                component: 'PulseJobLifecycle',
+                status: 'closed'
+            });
         }
         catch (error) {
-            this.log.e('Error closing Pulse runner:', error);
+            this.log.e('Pulse runner shutdown failed', {
+                component: 'PulseJobLifecycle',
+                error: error.message,
+                stack: error.stack
+            });
             throw error;
         }
     }
diff --git a/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js b/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js
index 86a09274b7f..8c1d5f50b8a 100644
--- a/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js
+++ b/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js
@@ -1,14 +1,26 @@
 const IJobScheduler = require('../../interfaces/IJobScheduler');
 const {isValidCron} = require('cron-validator');
 
+/**
+ * @typedef {import('../../interfaces/IJobScheduler')} IJobScheduler
+ * @typedef {import('@pulse/runner')} PulseRunner
+ * @typedef {import('@logger/interface')} Logger
+ * 
+ * @typedef {Object} ScheduleConfig
+ * @property {'schedule'|'once'|'now'} type - Type of schedule
+ * @property {string|Date} [value] - Cron expression for 'schedule' type or Date for 'once' type
+ */
+
 /**
  * Pulse implementation of job scheduler
+ * Handles scheduling, updating, and immediate execution of jobs using Pulse runner
+ * @implements {IJobScheduler}
  */
 class PulseJobScheduler extends IJobScheduler {
     /**
      * Creates a new PulseJobScheduler instance
-     * @param {Object} pulseRunner The Pulse runner instance
-     * @param {Object} logger Logger instance
+     * @param {PulseRunner} pulseRunner - The Pulse runner instance for job scheduling
+     * @param {Logger} logger - Logger instance for operational logging
      */
     constructor(pulseRunner, logger) {
         super();
@@ -17,10 +29,10 @@ class PulseJobScheduler extends IJobScheduler {
     }
 
     /**
-     * Validates schedule configuration
+     * Validates schedule configuration structure and values
      * @private
-     * @param {Object} config Schedule configuration
-     * @throws {Error} If configuration is invalid
+     * @param {ScheduleConfig} config - Schedule configuration to validate
+     * @throws {Error} If configuration is invalid or missing required fields
      */
     #validateScheduleConfig(config) {
         if (!config || typeof config !== 'object') {
@@ -42,15 +54,20 @@ class PulseJobScheduler extends IJobScheduler {
     }
 
     /**
-     * Schedules a job to run
-     * @param {string} name Job name
-     * @param {Object} scheduleConfig Schedule configuration
-     * @param {Object} [data] Optional data to pass to the job
-     * @returns {Promise<void>} A promise that resolves once the job is scheduled
+     * Schedules a job to run based on provided configuration
+     * @param {string} name - Unique identifier for the job
+     * @param {ScheduleConfig} scheduleConfig - Configuration defining when the job should run
+     * @param {Object} [data={}] - Optional payload to pass to the job during execution
+     * @returns {Promise<void>} Resolves when job is successfully scheduled
+     * @throws {Error} If scheduling fails or configuration is invalid
      */
     async schedule(name, scheduleConfig, data = {}) {
         try {
             this.#validateScheduleConfig(scheduleConfig);
+            this.log.d(`Attempting to schedule job '${name}' with type: ${scheduleConfig.type}`, {
+                scheduleConfig,
+                data
+            });
 
             switch (scheduleConfig.type) {
             case 'schedule':
@@ -72,22 +89,33 @@ class PulseJobScheduler extends IJobScheduler {
                 break;
             }
 
-            this.log.d(`Job ${name} scheduled successfully with type: ${scheduleConfig.type}`);
+            this.log.i(`Successfully scheduled job '${name}'`, {
+                type: scheduleConfig.type,
+                value: scheduleConfig.value,
+                hasData: Object.keys(data).length > 0
+            });
         }
         catch (error) {
-            this.log.e(`Failed to schedule job ${name}:`, error);
+            this.log.e(`Failed to schedule job '${name}'`, {
+                error: error.message,
+                scheduleConfig,
+                stack: error.stack
+            });
             throw error;
         }
     }
 
     /**
-     * Updates a job's schedule
-     * @param {string} jobName Name of the job
-     * @param {Object} schedule New schedule configuration
-     * @returns {Promise<void>} A promise that resolves once the schedule is updated
+     * Updates an existing job's schedule with new configuration
+     * @param {string} jobName - Name of the job to update
+     * @param {ScheduleConfig} schedule - New schedule configuration
+     * @returns {Promise<void>} Resolves when schedule is successfully updated
+     * @throws {Error} If update fails or new configuration is invalid
      */
     async updateSchedule(jobName, schedule) {
         try {
+            this.log.d(`Attempting to update schedule for job '${jobName}'`, { schedule });
+
             this.#validateScheduleConfig(schedule);
 
             // First remove the existing job
@@ -114,26 +142,38 @@ class PulseJobScheduler extends IJobScheduler {
                 break;
             }
 
-            this.log.i(`Schedule updated for job ${jobName}`);
+            this.log.i(`Successfully updated schedule for job '${jobName}'`, {
+                type: schedule.type,
+                value: schedule.value
+            });
         }
         catch (error) {
-            this.log.e(`Failed to update schedule for job ${jobName}:`, error);
+            this.log.e(`Failed to update schedule for job '${jobName}'`, {
+                error: error.message,
+                schedule,
+                stack: error.stack
+            });
             throw error;
         }
     }
 
     /**
-     * Runs a job immediately
-     * @param {string} jobName Name of the job
-     * @returns {Promise<void>} A promise that resolves when the job is triggered
+     * Triggers immediate execution of a job
+     * @param {string} jobName - Name of the job to execute
+     * @returns {Promise<void>} Resolves when job is successfully triggered
+     * @throws {Error} If immediate execution fails
      */
     async runJobNow(jobName) {
         try {
+            this.log.d(`Attempting to trigger immediate execution of job '${jobName}'`);
             await this.pulseRunner.now({ name: jobName });
-            this.log.i(`Job ${jobName} triggered for immediate execution`);
+            this.log.i(`Successfully triggered immediate execution of job '${jobName}'`);
         }
         catch (error) {
-            this.log.e(`Failed to run job ${jobName} immediately:`, error);
+            this.log.e(`Failed to trigger immediate execution of job '${jobName}'`, {
+                error: error.message,
+                stack: error.stack
+            });
             throw error;
         }
     }
diff --git a/jobServer/jobRunner/index.js b/jobServer/jobRunner/index.js
index 7e6c624646c..ddf0bcd4f3a 100644
--- a/jobServer/jobRunner/index.js
+++ b/jobServer/jobRunner/index.js
@@ -1,29 +1,68 @@
+/**
+ * @typedef {import('./PulseJobRunner.js').PulseJobRunner} PulseJobRunner
+ * @typedef {import('./BaseJobRunner.js').BaseJobRunner} BaseJobRunner
+ * @typedef {import('mongodb').Db} Mongodb
+ * @typedef {import('./types/Logger').Logger} Logger
+ */
+
+/**
+ * @typedef {Object} JobRunnerConfig
+ * @property {number} [pollInterval=1000] - Interval between job checks in milliseconds
+ * @property {number} [maxConcurrent=5] - Maximum number of concurrent jobs
+ * @property {string} [queueName='default'] - Name of the job queue to process
+ */
+
 const PulseJobRunner = require('./PulseJobRunner.js');
 
 /**
- * JobRunner implementation types
+ * Available JobRunner implementation types
+ * @readonly
  * @enum {string}
  */
 const RUNNER_TYPES = {
+    /** Pulse-based job runner implementation */
     PULSE: 'pulse'
 };
 
 /**
- * Job Runner factory
+ * Creates a new job runner instance based on the specified type
  * 
- * @param {Object} db The database connection
- * @param {string} [type='pulse'] The type of runner to create
- * @param {Object} [config={}] Configuration specific to the runner implementation
- * @param {function} Logger - Logger constructor
+ * @param {Mongodb} db - Mongoose database connection
+ * @param {RUNNER_TYPES} [type=RUNNER_TYPES.PULSE] - The type of runner to create
+ * @param {JobRunnerConfig} [config={}] - Configuration specific to the runner implementation
+ * @param {Logger} Logger - Logger Constructor
  * @returns {BaseJobRunner} An instance of BaseJobRunner with specific implementation
  * @throws {Error} If an invalid runner type is specified
+ * @throws {Error} If required dependencies are missing
+ * 
+ * @example
+ * const runner = createJobRunner(db, RUNNER_TYPES.PULSE, {
+ *   pollInterval: 5000,
+ *   maxConcurrent: 10,
+ *   queueName: 'high-priority'
+ * }, logger);
+ * await runner.start();
  */
 function createJobRunner(db, type = RUNNER_TYPES.PULSE, config = {}, Logger) {
-    switch (type.toLowerCase()) {
-    case RUNNER_TYPES.PULSE:
-        return new PulseJobRunner(db, config, Logger);
-    default:
-        throw new Error(`Invalid runner type: ${type}. Must be one of: ${Object.values(RUNNER_TYPES).join(', ')}`);
+    if (!db) {
+        throw new Error('Database connection is required');
+    }
+
+    const log = Logger('jobs:runner:factory');
+
+    log.i('Creating job runner', { type, config });
+
+    try {
+        switch (type.toLowerCase()) {
+        case RUNNER_TYPES.PULSE:
+            return new PulseJobRunner(db, config, Logger);
+        default:
+            throw new Error(`Invalid runner type: ${type}. Must be one of: ${Object.values(RUNNER_TYPES).join(', ')}`);
+        }
+    }
+    catch (error) {
+        log.e('Failed to create job runner', { type, config, error });
+        throw error;
     }
 }
 
diff --git a/jobServer/jobRunner/interfaces/IJobExecutor.js b/jobServer/jobRunner/interfaces/IJobExecutor.js
index 5757cc756c8..cc74622d98e 100644
--- a/jobServer/jobRunner/interfaces/IJobExecutor.js
+++ b/jobServer/jobRunner/interfaces/IJobExecutor.js
@@ -1,38 +1,52 @@
 /**
- * Interface for job execution operations
+ * @typedef {Object} RetryConfig
+ * @property {number} attempts - Maximum number of retry attempts
+ * @property {number} delay - Delay between retries in milliseconds
+ */
+
+/**
+ * Interface for job execution operations.
+ * Provides methods to manage and configure job execution within the system.
+ * @interface
  */
 class IJobExecutor {
     /**
-     * Creates and defines a new job
-     * @param {string} jobName The name of the job
-     * @param {Function} JobClass The job class to create
+     * Creates and defines a new job in the execution system
+     * @param {string} jobName - Unique identifier for the job
+     * @param {Constructor} JobClass - Constructor for the job implementation
+     * @throws {Error} When method is not implemented by concrete class
+     * @returns {Promise<void>} Resolves when job is created and defined
      */
     async createJob(/* jobName, JobClass */) {
         throw new Error('Method not implemented');
     }
 
     /**
-     * Enable a job
-     * @param {string} jobName Name of the job to enable
+     * Enables a previously created job for execution
+     * @param {string} jobName - Name of the job to enable
+     * @throws {Error} When method is not implemented by concrete class
+     * @returns {Promise<void>} Resolves when job is enabled
      */
     async enableJob(/* jobName */) {
         throw new Error('Method not implemented');
     }
 
     /**
-     * Disable a job
-     * @param {string} jobName Name of the job to disable
+     * Disables an active job from execution
+     * @param {string} jobName - Name of the job to disable
+     * @throws {Error} When method is not implemented by concrete class
+     * @returns {Promise<void>} Resolves when job is disabled
      */
     async disableJob(/* jobName */) {
         throw new Error('Method not implemented');
     }
 
     /**
-     * Configure job retry settings
-     * @param {string} jobName Name of the job
-     * @param {Object} retryConfig Retry configuration
-     * @param {number} retryConfig.attempts Number of retry attempts
-     * @param {number} retryConfig.delay Delay between retries in ms
+     * Configures retry behavior for a specific job
+     * @param {string} jobName - Name of the job to configure
+     * @param {RetryConfig} retryConfig - Configuration for retry behavior
+     * @throws {Error} When method is not implemented by concrete class
+     * @returns {Promise<void>} Resolves when retry configuration is applied
      */
     async configureRetry(/* jobName, retryConfig */) {
         throw new Error('Method not implemented');
diff --git a/jobServer/jobRunner/interfaces/IJobLifecycle.js b/jobServer/jobRunner/interfaces/IJobLifecycle.js
index 92b1bd83222..513e2b2211e 100644
--- a/jobServer/jobRunner/interfaces/IJobLifecycle.js
+++ b/jobServer/jobRunner/interfaces/IJobLifecycle.js
@@ -1,17 +1,30 @@
 /**
- * Interface for job lifecycle operations
+ * Interface for managing job lifecycle operations.
+ * Handles the initialization, execution, and cleanup of job runners.
+ * 
+ * @interface
  */
 class IJobLifecycle {
     /**
-     * Starts the job runner
-     * @param {Object.<string, Function>} jobClasses Object containing job classes keyed by job name
+     * Starts the job runner and initializes all registered jobs
+     * 
+     * @typedef {Object} JobClass
+     * @property {Function} new Creates a new instance of the job
+     * @property {Function} execute Method that runs the job logic
+     * 
+     * @param {Object.<string, JobClass>} jobClasses - Map of job names to their implementing classes
+     * @throws {Error} When initialization fails
+     * @returns {Promise<void>} Resolves when all jobs are initialized and runner is ready
      */
     async start(/* jobClasses */) {
         throw new Error('Method not implemented');
     }
 
     /**
-     * Closes the job runner and cleans up resources
+     * Gracefully shuts down the job runner and performs cleanup
+     * 
+     * @throws {Error} When cleanup fails
+     * @returns {Promise<void>} Resolves when all jobs are stopped and resources are released
      */
     async close() {
         throw new Error('Method not implemented');
diff --git a/jobServer/jobRunner/interfaces/IJobScheduler.js b/jobServer/jobRunner/interfaces/IJobScheduler.js
index 9046ec7ee30..9d0eaf0999f 100644
--- a/jobServer/jobRunner/interfaces/IJobScheduler.js
+++ b/jobServer/jobRunner/interfaces/IJobScheduler.js
@@ -1,31 +1,49 @@
+/**
+ * @typedef {Object} ScheduleConfig
+ * @property {'once'|'schedule'|'now'} type - Type of schedule execution
+ * @property {string|Date} [value] - Cron expression (for 'schedule') or Date object (for 'once')
+ */
+
+/**
+ * @typedef {Object} JobData
+ * @property {*} [payload] - Custom data payload for the job
+ * @property {Object} [metadata] - Additional metadata for job execution
+ */
+
 /**
  * Interface for job scheduling operations
+ * Handles scheduling, updating, and immediate execution of jobs
+ * @interface
  */
 class IJobScheduler {
     /**
      * Schedules a job based on its configuration
-     * @param {String} name The name of the job
-     * @param {Object} scheduleConfig Schedule configuration object
-     * @param {('once'|'schedule'|'now')} scheduleConfig.type Type of schedule
-     * @param {string|Date} [scheduleConfig.value] Cron string or Date object
-     * @param {Object} [data] Data to pass to the job
+     * @param {string} name - Unique identifier for the job
+     * @param {ScheduleConfig} scheduleConfig - Schedule configuration
+     * @param {JobData} [data] - Optional data to pass to the job
+     * @throws {Error} If scheduling fails
+     * @returns {Promise<void>} Resolves when scheduling is complete`
      */
     async schedule(/* name, scheduleConfig, data */) {
         throw new Error('Method not implemented');
     }
 
     /**
-     * Update job schedule
-     * @param {string} jobName Name of the job
-     * @param {string|Date} schedule New schedule (cron string or date)
+     * Update existing job schedule
+     * @param {string} jobName - Name of the job to update
+     * @param {string|Date} schedule - New schedule (cron expression or date)
+     * @throws {Error} If job not found or update fails
+     * @returns {Promise<void>} Resolves when scheduling is complete
      */
     async updateSchedule(/* jobName, schedule */) {
         throw new Error('Method not implemented');
     }
 
     /**
-     * Runs a job now
-     * @param {string} jobName Name of the job to run
+     * Executes a job immediately, bypassing its schedule
+     * @param {string} jobName - Name of the job to run
+     * @throws {Error} If job not found or execution fails
+     * @returns {Promise<void>} Resolves when job is executed
      */
     async runJobNow(/* jobName */) {
         throw new Error('Method not implemented');

From 372ddd6fc0089f1d813f44ba1a0a772b7fd42b58 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Wed, 15 Jan 2025 23:49:22 +0530
Subject: [PATCH 25/90] codacy fix

---
 jobServer/jobRunner/PulseJobRunner.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jobServer/jobRunner/PulseJobRunner.js b/jobServer/jobRunner/PulseJobRunner.js
index 1478cf33bca..0c1fb169719 100644
--- a/jobServer/jobRunner/PulseJobRunner.js
+++ b/jobServer/jobRunner/PulseJobRunner.js
@@ -47,7 +47,7 @@ class PulseJobRunner extends BaseJobRunner {
         const log = Logger('jobs:runner:pulse');
         log.d('Initializing PulseJobRunner');
 
-        if (!db || !config || !Logger) {
+        if (!db || !config) {
             log.e('Missing required dependencies');
             throw new Error('Missing required dependencies for PulseJobRunner');
         }

From a3acc44c8527062c7d0b563f57d6fbc25755d734 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Thu, 16 Jan 2025 11:03:59 +0530
Subject: [PATCH 26/90] add design pattern conventions

---
 jobServer/README.md | 95 ++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 80 insertions(+), 15 deletions(-)

diff --git a/jobServer/README.md b/jobServer/README.md
index 122615509cb..e71b726c787 100644
--- a/jobServer/README.md
+++ b/jobServer/README.md
@@ -10,19 +10,20 @@ A flexible, extensible job scheduling and execution system built on MongoDB with
 4. [Collections](#collections)
 5. [Basic Usage](#basic-usage)
 6. [Architecture](#architecture)
-7. [Server Configuration](#server-configuration)
-8. [Job Implementation Guide](#job-implementation-guide)
-9. [Lock Extension & Progress Reporting](#lock-extension--progress-reporting)
-10. [Job Configuration Management](#job-configuration-management)
-11. [Parallel Processing](#parallel-processing)
-12. [File Structure](#file-structure)
-13. [Interface Contracts](#interface-contracts)
-14. [Implementing New Runners](#implementing-new-runners)
-15. [Error Handling & Monitoring](#error-handling--monitoring)
-16. [Best Practices](#best-practices)
-17. [Troubleshooting Guide](#troubleshooting-guide)
-18. [Monitoring & Metrics](#monitoring--metrics)
-19. [BullMQ Implementation Guide](#bullmq-implementation-guide)
+7. [Design Patterns](#design-patterns)
+8. [Server Configuration](#server-configuration)
+9. [Job Implementation Guide](#job-implementation-guide)
+10. [Lock Extension & Progress Reporting](#lock-extension--progress-reporting)
+11. [Job Configuration Management](#job-configuration-management)
+12. [Parallel Processing](#parallel-processing)
+13. [File Structure](#file-structure)
+14. [Interface Contracts](#interface-contracts)
+15. [Implementing New Runners](#implementing-new-runners)
+16. [Error Handling & Monitoring](#error-handling--monitoring)
+17. [Best Practices](#best-practices)
+18. [Troubleshooting Guide](#troubleshooting-guide)
+19. [Monitoring & Metrics](#monitoring--metrics)
+20. [BullMQ Implementation Guide](#bullmq-implementation-guide)
 
 ## Overview
 
@@ -172,6 +173,70 @@ JobExecutor JobScheduler JobLifecycle
    - Handles actual job execution
    - Manages scheduling and state
 
+## Design Patterns
+
+The Job Server Module employs several design patterns to maintain flexibility, testability, and extensibility:
+
+### Core Patterns
+
+1. **Interface Segregation**
+   - Job operations split into focused interfaces (IJobExecutor, IJobScheduler, IJobLifecycle)
+   - Enables targeted implementation of specific job aspects
+   - Reduces coupling between components
+   ```javascript
+   // Example: Separate interfaces for different concerns
+   interface IJobExecutor { /* job execution methods */ }
+   interface IJobScheduler { /* scheduling methods */ }
+   interface IJobLifecycle { /* lifecycle methods */ }
+   ```
+
+2. **Composition over Inheritance**
+   - BaseJobRunner composes functionality from specialized interfaces
+   - Runner implementations combine executor, scheduler, and lifecycle components
+   - Allows flexible mixing of different implementations
+   ```javascript
+   class BaseJobRunner {
+       constructor(scheduler, executor, lifecycle) {
+           this.scheduler = scheduler;
+           this.executor = executor;
+           this.lifecycle = lifecycle;
+       }
+   }
+   ```
+
+3. **Dependency Injection**
+   - Components receive dependencies through constructors
+   - Facilitates testing and configuration
+   - Enables runtime selection of implementations
+   ```javascript
+   const server = await JobServer.create(common, Logger, pluginManager, {
+       runner: {
+           type: 'pulse',
+           config: { /* ... */ }
+       }
+   });
+   ```
+
+4. **Factory Pattern**
+   - Runner implementations created through factory methods
+   - Centralizes runner instantiation logic
+   - Supports multiple runner types (Pulse, BullMQ)
+   ```javascript
+   // Example: Runner factory
+   const RUNNER_TYPES = {
+       PULSE: 'pulse',
+       BULL: 'bullmq'
+   };
+   ```
+
+
+### Benefits
+
+- **Extensibility**: New runners can be added without modifying existing code
+- **Testability**: Components can be tested in isolation with mock implementations
+- **Flexibility**: Runtime configuration of job processing behavior
+- **Maintainability**: Clear separation of concerns and modular design
+
 ## Server Configuration
 
 ### Pulse Runner Configuration
@@ -860,8 +925,7 @@ const connection = new Redis({
 ```
 
 2. **Basic Structure**
-```
-jobRunner/impl/bullmq/
+```jobRunner/impl/bullmq/
 ├── BullMQJobExecutor.js
 ├── BullMQJobLifecycle.js
 ├── BullMQJobScheduler.js
@@ -930,3 +994,4 @@ class BullMQJobLifecycle extends IJobLifecycle {
     }
 }
 ```
+

From e103818b371a050951f075a6692652d60e0f9832 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Thu, 16 Jan 2025 11:28:23 +0530
Subject: [PATCH 27/90] wip mark in readme

---
 jobServer/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jobServer/README.md b/jobServer/README.md
index e71b726c787..c83b0d57e28 100644
--- a/jobServer/README.md
+++ b/jobServer/README.md
@@ -909,7 +909,7 @@ class LoggingJob extends Job {
 }
 ```
 
-## BullMQ Implementation Guide
+## BullMQ Implementation Guide [WIP]
 
 ### Setup Requirements
 

From 4445d5a1cdfbd229d6afb4538261fdfcf9e834d0 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Thu, 16 Jan 2025 12:05:58 +0530
Subject: [PATCH 28/90] Add simple example

---
 jobServer/example/SimpleExample.js | 43 ++++++++++++++++++++++++++++++
 1 file changed, 43 insertions(+)
 create mode 100644 jobServer/example/SimpleExample.js

diff --git a/jobServer/example/SimpleExample.js b/jobServer/example/SimpleExample.js
new file mode 100644
index 00000000000..9dc9785f7d1
--- /dev/null
+++ b/jobServer/example/SimpleExample.js
@@ -0,0 +1,43 @@
+const job = require("../../jobServer");
+
+/**
+ * Simple example job with only required methods.
+ * @extends {job.Job}
+ */
+class SimpleExample extends job.Job {
+    /**
+     * Get the schedule configuration for the job.
+     * @required
+     * @returns {Object} Schedule configuration object
+     */
+    getSchedule() {
+        return {
+            type: 'schedule',
+            value: '0 0 * * *' // Runs daily at midnight
+        };
+    }
+
+    /**
+     * Main job execution method.
+     * @required
+     * @param {Db} db Database connection
+     * @param {Function} done Callback to signal job completion
+     * @param {Function} progress Progress reporting function
+     */
+    async run(db, done, progress) {
+        try {
+            this.logger.i("Starting simple job");
+
+            // Your job logic here
+            await progress(1, 1, "Task completed");
+
+            done(null, { success: true });
+        }
+        catch (error) {
+            this.logger.e("Job failed:", error);
+            done(error);
+        }
+    }
+}
+
+module.exports = SimpleExample;
\ No newline at end of file

From 97ed75384c6dd6a6fce019d5a402947cf5d8d9fa Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Thu, 16 Jan 2025 12:08:03 +0530
Subject: [PATCH 29/90] sane defaults for Job

---
 jobServer/Job.js | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/jobServer/Job.js b/jobServer/Job.js
index 1410965b1ae..4699c862e42 100644
--- a/jobServer/Job.js
+++ b/jobServer/Job.js
@@ -211,7 +211,7 @@ class Job {
     async _run(db, job, done) {
         this.logger.d(`[Job:${this.jobName}] Starting execution`, {
             database: db?._cly_debug?.db,
-            jobId: job?.attrs._id,
+            jobId: job?.attrs?._id,
             jobName: this.jobName
         });
 
@@ -317,9 +317,9 @@ class Job {
      */
     getRetryConfig() {
         return {
-            enabled: true,
+            enabled: false,
             attempts: 3,
-            delay: 2000 // 2 seconds
+            delay: 5 * 60 * 1000 // 5 minutes
         };
     }
 

From b6a542118bef4f7fccef5bcfc65f8a4126974ff4 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Thu, 16 Jan 2025 12:09:30 +0530
Subject: [PATCH 30/90] sane defaults for global

---
 jobServer/config.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/jobServer/config.js b/jobServer/config.js
index 9941659c934..ceef4454397 100644
--- a/jobServer/config.js
+++ b/jobServer/config.js
@@ -15,11 +15,11 @@
  * @property {string} db.collection - MongoDB collection name for storing jobs
  */
 const DEFAULT_PULSE_CONFIG = {
-    processEvery: '3 seconds',
+    processEvery: '60 seconds',
     maxConcurrency: 1,
     defaultConcurrency: 1,
     lockLimit: 1,
-    defaultLockLimit: 1,
+    defaultLockLimit: 3,
     defaultLockLifetime: 55 * 60 * 1000, // 55 minutes
     sort: { nextRunAt: 1, priority: -1 },
     disableAutoIndex: false,

From 46c77aabf0f4bd5e925dffd1c5d2ffda7752ea4c Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Thu, 16 Jan 2025 15:58:50 +0530
Subject: [PATCH 31/90] Prevent job scheduler collision

---
 api/parts/jobs/scanner.js | 34 ++++++++++++++++++++++++++++++----
 1 file changed, 30 insertions(+), 4 deletions(-)

diff --git a/api/parts/jobs/scanner.js b/api/parts/jobs/scanner.js
index dad13d46845..771e234ed97 100644
--- a/api/parts/jobs/scanner.js
+++ b/api/parts/jobs/scanner.js
@@ -2,7 +2,30 @@
 
 const log = require('../../utils/log.js')('jobs:scanner'),
     manager = require('../../../plugins/pluginManager.js'),
-    fs = require('fs');
+    fs = require('fs'),
+    {Job, IPCJob, IPCFaçadeJob, TransientJob} = require('./job.js');
+
+/**
+ * Validates if a job class has the required methods
+ * @param {Function} JobClass - The job class to validate
+ * @returns {boolean} - True if valid, throws error if invalid
+ */
+const validateJobClass = (JobClass) => {
+    // Check if it's a class/constructor
+    if (typeof JobClass !== 'function') {
+        throw new Error('Job must be a class constructor');
+    }
+
+    // Check if it inherits from one of the valid base classes
+    if (!(JobClass.prototype instanceof Job ||
+          JobClass.prototype instanceof IPCJob ||
+          JobClass.prototype instanceof IPCFaçadeJob ||
+          JobClass.prototype instanceof TransientJob)) {
+        throw new Error('Job class must extend Job, IPCJob, IPCFaçadeJob, or TransientJob');
+    }
+
+    return true;
+};
 
 module.exports = (db, filesObj, classesObj) => {
     return new Promise((resolve, reject) => {
@@ -52,9 +75,12 @@ module.exports = (db, filesObj, classesObj) => {
                 (arr || []).forEach(job => {
                     try {
                         let name = job.category + ':' + job.name;
-                        filesObj[name] = job.file;
-                        classesObj[name] = require(job.file);
-                        log.d('Found job %j at %j', name, job.file);
+                        const JobClass = require(job.file);
+                        if (validateJobClass(JobClass)) {
+                            filesObj[name] = job.file;
+                            classesObj[name] = JobClass;
+                            log.d('Found valid job %j at %j', name, job.file);
+                        }
                     }
                     catch (e) {
                         log.e('Error when loading job %s: %j ', job.file, e, e.stack);

From 67f258890bbaf760df84b22d3d933080a89a5d53 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Thu, 23 Jan 2025 23:29:10 +0530
Subject: [PATCH 32/90] migrate crashes jobs

---
 plugins/crashes/api/api.js                      |  6 +++---
 .../crashes/api/jobs/cleanup_custom_field.js    | 17 +++++++++++++++--
 2 files changed, 18 insertions(+), 5 deletions(-)

diff --git a/plugins/crashes/api/api.js b/plugins/crashes/api/api.js
index 891f1d652c2..3c0fef4bd23 100644
--- a/plugins/crashes/api/api.js
+++ b/plugins/crashes/api/api.js
@@ -59,9 +59,9 @@ plugins.setConfigs("crashes", {
             }
         });
 
-        setTimeout(() => {
-            require('../../../api/parts/jobs').job('crashes:cleanup_custom_field').replace().schedule('at 01:01 am ' + 'every 1 day');
-        }, 10000);
+        // setTimeout(() => {
+        // require('../../../api/parts/jobs').job('crashes:cleanup_custom_field').replace().schedule('at 01:01 am ' + 'every 1 day');
+        // }, 10000);
     });
     var ranges = ["ram", "bat", "disk", "run", "session"];
     var segments = ["os_version", "os_name", "manufacture", "device", "resolution", "app_version", "cpu", "opengl", "orientation", "view", "browser"];
diff --git a/plugins/crashes/api/jobs/cleanup_custom_field.js b/plugins/crashes/api/jobs/cleanup_custom_field.js
index c6309e321a3..dddc17aebd1 100644
--- a/plugins/crashes/api/jobs/cleanup_custom_field.js
+++ b/plugins/crashes/api/jobs/cleanup_custom_field.js
@@ -1,11 +1,24 @@
-const job = require('../../../../api/parts/jobs/job.js');
+// const job = require('../../../../api/parts/jobs/job.js');
+const Job = require('./../../../../jobServer/Job');
 const log = require('../../../../api/utils/log.js')('job:crashes:cleanup_custom_field');
 const pluginManager = require('../../../pluginManager.js');
 
 const { cleanupCustomField, DEFAULT_MAX_CUSTOM_FIELD_KEYS } = require('../parts/custom_field.js');
 
 /** class CleanupCustomFieldJob */
-class CleanupCustomFieldJob extends job.Job {
+class CleanupCustomFieldJob extends Job {
+
+    /**
+     * Get the schedule configuration for this job
+     * @returns {GetScheduleConfig} schedule configuration
+     */
+    getSchedule() {
+        return {
+            type: "schedule",
+            value: "1 1 * * *" // every day at 1:01
+        };
+    }
+
     /** function run
      * @param {object} countlyDb - db connection object
      * @param {function} doneJob - function to call when finishing Job

From d6eb77fb92be93f707fff22014cbc0edaf4e07c1 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Thu, 23 Jan 2025 23:29:16 +0530
Subject: [PATCH 33/90] migrate hooks jobs

---
 plugins/hooks/api/jobs/schedule.js            | 21 +++++++++++++++----
 plugins/hooks/api/parts/triggers/scheduled.js | 12 +++++------
 2 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/plugins/hooks/api/jobs/schedule.js b/plugins/hooks/api/jobs/schedule.js
index d5bbf464afc..7a041bce3ad 100644
--- a/plugins/hooks/api/jobs/schedule.js
+++ b/plugins/hooks/api/jobs/schedule.js
@@ -1,14 +1,27 @@
 'use strict';
-const job = require('../../../../api/parts/jobs/job.js'),
-    plugins = require('../../../pluginManager.js'),
-    log = require('../../../../api/utils/log.js')('hooks:monitor');
+// const job = require('../../../../api/parts/jobs/job.js'),
+const Job = require('../../../../jobServer/Job');
+const plugins = require('../../../pluginManager.js');
+const log = require('../../../../api/utils/log.js')('hooks:monitor');
 
 /**
  * @class
  * @classdesc Class MonitorJob is Hooks Monitor Job extend from Countly Job
  * @extends Job
  */
-class ScheduleJob extends job.Job {
+class ScheduleJob extends Job {
+
+    /**
+     * Get the schedule configuration for this job
+     * @returns {GetScheduleConfig} schedule configuration
+     */
+    getSchedule() {
+        return {
+            type: "schedule",
+            value: "1 * * * *" // every 1 hour on the 1st min
+        };
+    }
+
     /**
     * run task
     * @param {object} db - db object
diff --git a/plugins/hooks/api/parts/triggers/scheduled.js b/plugins/hooks/api/parts/triggers/scheduled.js
index 99e598fed49..b40e2f948f9 100644
--- a/plugins/hooks/api/parts/triggers/scheduled.js
+++ b/plugins/hooks/api/parts/triggers/scheduled.js
@@ -2,7 +2,7 @@ const plugins = require('../../../../pluginManager.js');
 const common = require('../../../../../api/utils/common.js');
 const utils = require('../../utils.js');
 const log = common.log("hooks:api:schedule");
-const JOB = require('../../../../../api/parts/jobs');
+// const JOB = require('../../../../../api/parts/jobs');
 const later = require('later');
 const moment = require('moment-timezone');
 
@@ -98,11 +98,11 @@ class ScheduledTrigger {
 }
 
 
-plugins.register("/master", function() {
-    setTimeout(() => {
-        JOB.job('hooks:schedule', {type: 'ScheduledTrigger'}).replace().schedule("every 1 hour on the 1st min");
-    }, 10000);
-});
+// plugins.register("/master", function() {
+//     setTimeout(() => {
+//         JOB.job('hooks:schedule', {type: 'ScheduledTrigger'}).replace().schedule("every 1 hour on the 1st min");
+//     }, 10000);
+// });
 
 
 module.exports = ScheduledTrigger;

From 4a45e76fef3e6a887810c86701e89e3a077a2b04 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Thu, 23 Jan 2025 23:29:22 +0530
Subject: [PATCH 34/90] migrate logger jobs

---
 plugins/logger/api/api.js        | 16 ++++++++--------
 plugins/logger/api/jobs/clear.js | 17 +++++++++++++++--
 2 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/plugins/logger/api/api.js b/plugins/logger/api/api.js
index 4e36c16430b..d1a6e5cadf5 100644
--- a/plugins/logger/api/api.js
+++ b/plugins/logger/api/api.js
@@ -4,7 +4,7 @@ var exported = {},
     automaticStateManager = require('./helpers/automaticStateManager'),
     log = require('../../../api/utils/log.js')('logger:api'),
     { validateRead } = require('../../../api/utils/rights.js');
-const JOB = require('../../../api/parts/jobs');
+// const JOB = require('../../../api/parts/jobs');
 const MAX_NUMBER_OF_LOG_ENTRIES = 1000;
 const FEATURE_NAME = 'logger';
 
@@ -21,13 +21,13 @@ plugins.setConfigs("logger", {
 });
 
 (function() {
-    plugins.register("/master", function() {
-        setTimeout(() => {
-            JOB.job('logger:clear', { max: MAX_NUMBER_OF_LOG_ENTRIES })
-                .replace()
-                .schedule("every 5 minutes");
-        }, 10000);
-    });
+    // plugins.register("/master", function() {
+    //     setTimeout(() => {
+    //         JOB.job('logger:clear', { max: MAX_NUMBER_OF_LOG_ENTRIES })
+    //             .replace()
+    //             .schedule("every 5 minutes");
+    //     }, 10000);
+    // });
 
     plugins.register("/permissions/features", function(ob) {
         ob.features.push(FEATURE_NAME);
diff --git a/plugins/logger/api/jobs/clear.js b/plugins/logger/api/jobs/clear.js
index ddf9376c217..cb117945d58 100644
--- a/plugins/logger/api/jobs/clear.js
+++ b/plugins/logger/api/jobs/clear.js
@@ -2,7 +2,8 @@
  * @typedef {import("mongodb").Db} Database
  * @typedef {import("mongodb").ObjectId} ObjectId
  */
-const JOB = require("../../../../api/parts/jobs/job.js");
+// const JOB = require("../../../../api/parts/jobs/job.js");
+const Job = require("../../../../jobServer/Job");
 const log = require("../../../../api/utils/log.js")("job:logger:clear");
 
 const DEFAULT_MAX_ENTRIES = 1000;
@@ -10,7 +11,19 @@ const DEFAULT_MAX_ENTRIES = 1000;
 /**
  * clears logs
  */
-class ClearJob extends JOB.Job {
+class ClearJob extends Job {
+
+    /**
+     * Get the schedule configuration for this job
+     * @returns {GetScheduleConfig} schedule configuration
+     */
+    getSchedule() {
+        return {
+            type: "schedule",
+            value: "*/5 * * * *" // every 5 minutes
+        };
+    }
+
     /**
      * Cleans up the logs{APPID} collection
      * @param {Database} db mongodb database instance

From b91fe81e9f563b026289f98eb0b238d7a8a62726 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Thu, 23 Jan 2025 23:29:30 +0530
Subject: [PATCH 35/90] migrate reports jobs

---
 plugins/reports/api/api.js       | 12 ++++++------
 plugins/reports/api/jobs/send.js | 25 +++++++++++++++++++------
 2 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/plugins/reports/api/api.js b/plugins/reports/api/api.js
index 38dfe3487d5..fdbbef28218 100644
--- a/plugins/reports/api/api.js
+++ b/plugins/reports/api/api.js
@@ -14,12 +14,12 @@ const FEATURE_NAME = 'reports';
         ob.features.push(FEATURE_NAME);
     });
 
-    plugins.register("/master", function() {
-        // Allow configs to load & scanner to find all jobs classes
-        setTimeout(() => {
-            require('../../../api/parts/jobs').job('reports:send').replace().schedule("every 1 hour starting on the 0 min");
-        }, 10000);
-    });
+    // plugins.register("/master", function() {
+    //     // Allow configs to load & scanner to find all jobs classes
+    //     setTimeout(() => {
+    //         require('../../../api/parts/jobs').job('reports:send').replace().schedule("every 1 hour starting on the 0 min");
+    //     }, 10000);
+    // });
 
     /**
      * @api {get} /o/reports/all Get reports data 
diff --git a/plugins/reports/api/jobs/send.js b/plugins/reports/api/jobs/send.js
index 0b893861086..52dcbe681e1 100644
--- a/plugins/reports/api/jobs/send.js
+++ b/plugins/reports/api/jobs/send.js
@@ -1,16 +1,29 @@
 'use strict';
 
-const job = require('../../../../api/parts/jobs/job.js'),
-    log = require('../../../../api/utils/log.js')('job:reports');
-var plugins = require('../../../pluginManager.js'),
-    async = require("async"),
-    reports = require("../reports");
+// const job = require('../../../../api/parts/jobs/job.js'),
+const Job = require('../../../../jobServer/Job');
+const log = require('../../../../api/utils/log.js')('job:reports');
+const plugins = require('../../../pluginManager.js');
+const async = require("async");
+const reports = require("../reports");
 /**
  * @class
  * @classdesc Class ReportsJob is report Job extend from Countly Job
  * @extends Job
  */
-class ReportsJob extends job.Job {
+class ReportsJob extends Job {
+
+    /**
+     * Get the schedule configuration for this job
+     * @returns {GetScheduleConfig} schedule configuration
+     */
+    getSchedule() {
+        return {
+            type: "schedule",
+            value: "0 * * * *" // every hour
+        };
+    }
+
     /**
     * run task
     * @param {object} countlyDb - db object

From 84fba2806cc4e32b00e86719a6fc98a6da75d504 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Thu, 23 Jan 2025 23:29:34 +0530
Subject: [PATCH 36/90] migrate stats jobs

---
 plugins/server-stats/api/api.js        | 12 +++++------
 plugins/server-stats/api/jobs/stats.js | 30 +++++++++++++++++++-------
 2 files changed, 28 insertions(+), 14 deletions(-)

diff --git a/plugins/server-stats/api/api.js b/plugins/server-stats/api/api.js
index abd498dd544..a096f5c903b 100644
--- a/plugins/server-stats/api/api.js
+++ b/plugins/server-stats/api/api.js
@@ -18,12 +18,12 @@ const internalEventsSkipped = ["[CLY]_orientation"];
         ob.features.push(FEATURE_NAME);
     });
 
-    plugins.register("/master", function() {
-        // Allow configs to load & scanner to find all jobs classes
-        setTimeout(() => {
-            require('../../../api/parts/jobs').job('server-stats:stats').replace().schedule('every 1 day');
-        }, 10000);
-    });
+    // plugins.register("/master", function() {
+    //     // Allow configs to load & scanner to find all jobs classes
+    //     setTimeout(() => {
+    //         require('../../../api/parts/jobs').job('server-stats:stats').replace().schedule('every 1 day');
+    //     }, 10000);
+    // });
 
     /**
      * @param {string} events - events to be mapped
diff --git a/plugins/server-stats/api/jobs/stats.js b/plugins/server-stats/api/jobs/stats.js
index d112fc1913c..647f6c96bad 100644
--- a/plugins/server-stats/api/jobs/stats.js
+++ b/plugins/server-stats/api/jobs/stats.js
@@ -1,12 +1,12 @@
 'use strict';
 
-const job = require('../../../../api/parts/jobs/job.js'),
-    tracker = require('../../../../api/parts/mgmt/tracker.js'),
-    log = require('../../../../api/utils/log.js')('job:stats'),
-    config = require("../../../../frontend/express/config.js"),
-    pluginManager = require('../../../pluginManager.js'),
-    moment = require('moment-timezone'),
-    request = require('countly-request')(pluginManager.getConfig("security"));
+// const job = require('../../../../api/parts/jobs/job.js');
+const Job = require('../../../../jobServer/Job');
+const tracker = require('../../../../api/parts/mgmt/tracker.js');
+const log = require('../../../../api/utils/log.js')('job:stats');
+const config = require("../../../../frontend/express/config.js");
+const pluginManager = require('../../../pluginManager.js');
+const moment = require('moment-timezone');
 
 const promisedLoadConfigs = function(db) {
     return new Promise((resolve) => {
@@ -17,7 +17,19 @@ const promisedLoadConfigs = function(db) {
 };
 
 /** Representing a StatsJob. Inherits api/parts/jobs/job.js (job.Job) */
-class StatsJob extends job.Job {
+class StatsJob extends Job {
+
+    /**
+     * Get the schedule configuration for this job
+     * @returns {GetScheduleConfig} schedule configuration
+     */
+    getSchedule() {
+        return {
+            type: "schedule",
+            value: "0 3 * * *" // Every day at 2:00 AM
+        };
+    }
+
     /**
     * Inherits api/parts/jobs/job.js, please review for detailed description
     * @param {string|Object} name - Name for job
@@ -67,6 +79,8 @@ class StatsJob extends job.Job {
                             var usersData = [];
 
                             await promisedLoadConfigs(db);
+                            // getConfig for security after loadConfigs
+                            const request = require('countly-request')(pluginManager.getConfig("security"));
 
                             let domain = '';
 

From 774822c324d93ef6c36e08bf119a36c675cad5ac Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Thu, 23 Jan 2025 23:29:39 +0530
Subject: [PATCH 37/90] migrate views jobs

---
 plugins/views/api/api.js              | 12 ++++++------
 plugins/views/api/jobs/cleanupMeta.js | 17 +++++++++++++++--
 2 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/plugins/views/api/api.js b/plugins/views/api/api.js
index 8d5d9cdbca6..0dcb8764ae0 100644
--- a/plugins/views/api/api.js
+++ b/plugins/views/api/api.js
@@ -33,12 +33,12 @@ const escapedViewSegments = { "name": true, "segment": true, "height": true, "wi
         common.dbUniqueMap.users.push("vc");
     });
 
-    plugins.register("/master", function() {
-        // Allow configs to load & scanner to find all jobs classes
-        setTimeout(() => {
-            require('../../../api/parts/jobs').job('views:cleanupMeta')?.replace()?.schedule("every 1 day");
-        }, 3000);
-    });
+    // plugins.register("/master", function() {
+    //     // Allow configs to load & scanner to find all jobs classes
+    //     setTimeout(() => {
+    //         require('../../../api/parts/jobs').job('views:cleanupMeta')?.replace()?.schedule("every 1 day");
+    //     }, 3000);
+    // });
 
     plugins.register("/i/user_merge", function(ob) {
         var newAppUser = ob.newAppUser;
diff --git a/plugins/views/api/jobs/cleanupMeta.js b/plugins/views/api/jobs/cleanupMeta.js
index 939655a05ff..f44321e57f2 100644
--- a/plugins/views/api/jobs/cleanupMeta.js
+++ b/plugins/views/api/jobs/cleanupMeta.js
@@ -1,4 +1,5 @@
-const job = require('../../../../api/parts/jobs/job.js');
+// const job = require('../../../../api/parts/jobs/job.js');
+const Job = require('../../../../jobServer/Job');
 const log = require('../../../../api/utils/log.js')('job:views:cleanup_meta');
 var Promise = require("bluebird");
 
@@ -8,7 +9,19 @@ const viewsUtils = require("../parts/viewsUtils.js");
 /**
  * Class for running job
  */
-class CleanupMetaJob extends job.Job {
+class CleanupMetaJob extends Job {
+
+    /**
+     * Get the schedule configuration for this job
+     * @returns {GetScheduleConfig} schedule configuration
+     */
+    getSchedule() {
+        return {
+            type: "schedule",
+            value: "0 4 * * *" // every day at 4am
+        };
+    }
+
     /**
      * Get's called to run the job
      * @param {mongoDatabase} countlyDb ref to countlyDb

From 254a52c7bb9f6cadbe853cd9017d37580c682a16 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Thu, 23 Jan 2025 23:30:26 +0530
Subject: [PATCH 38/90] disable api.js jobs

---
 api/api.js | 58 +++++++++++++++++++++++++++++-------------------------
 1 file changed, 31 insertions(+), 27 deletions(-)

diff --git a/api/api.js b/api/api.js
index df712833b8b..31f3404d393 100644
--- a/api/api.js
+++ b/api/api.js
@@ -4,7 +4,7 @@ const formidable = require('formidable');
 const os = require('os');
 const countlyConfig = require('./config', 'dont-enclose');
 const plugins = require('../plugins/pluginManager.js');
-const jobs = require('./parts/jobs');
+// const jobs = require('./parts/jobs');
 const log = require('./utils/log.js')('core:api');
 const common = require('./utils/common.js');
 const {processRequest} = require('./utils/requestProcessor');
@@ -123,22 +123,26 @@ plugins.connectToAllDatabases().then(function() {
     /**
     * Set Plugins Logs Config
     */
-    plugins.setConfigs('logs', {
-        debug: (countlyConfig.logging && countlyConfig.logging.debug) ? countlyConfig.logging.debug.join(', ') : '',
-        info: (countlyConfig.logging && countlyConfig.logging.info) ? countlyConfig.logging.info.join(', ') : '',
-        warn: (countlyConfig.logging && countlyConfig.logging.warn) ? countlyConfig.logging.warn.join(', ') : '',
-        error: (countlyConfig.logging && countlyConfig.logging.error) ? countlyConfig.logging.error.join(', ') : '',
-        default: (countlyConfig.logging && countlyConfig.logging.default) ? countlyConfig.logging.default : 'warn',
-    }, undefined, () => {
-        const cfg = plugins.getConfig('logs'), msg = {
-            cmd: 'log',
-            config: cfg
-        };
-        if (process.send) {
-            process.send(msg);
+    plugins.setConfigs('logs',
+        {
+            debug: (countlyConfig.logging && countlyConfig.logging.debug) ? countlyConfig.logging.debug.join(', ') : '',
+            info: (countlyConfig.logging && countlyConfig.logging.info) ? countlyConfig.logging.info.join(', ') : '',
+            warn: (countlyConfig.logging && countlyConfig.logging.warn) ? countlyConfig.logging.warn.join(', ') : '',
+            error: (countlyConfig.logging && countlyConfig.logging.error) ? countlyConfig.logging.error.join(', ') : '',
+            default: (countlyConfig.logging && countlyConfig.logging.default) ? countlyConfig.logging.default : 'warn',
+        },
+        undefined,
+        () => {
+            const cfg = plugins.getConfig('logs'), msg = {
+                cmd: 'log',
+                config: cfg
+            };
+            if (process.send) {
+                process.send(msg);
+            }
+            require('./utils/log.js').ipcHandler(msg);
         }
-        require('./utils/log.js').ipcHandler(msg);
-    });
+    );
 
     /**
     * Initialize Plugins
@@ -305,17 +309,17 @@ plugins.connectToAllDatabases().then(function() {
         plugins.dispatch("/master", {});
 
         // Allow configs to load & scanner to find all jobs classes
-        setTimeout(() => {
-            jobs.job('api:topEvents').replace().schedule('at 00:01 am ' + 'every 1 day');
-            jobs.job('api:ping').replace().schedule('every 1 day');
-            jobs.job('api:clear').replace().schedule('every 1 day');
-            jobs.job('api:clearTokens').replace().schedule('every 1 day');
-            jobs.job('api:clearAutoTasks').replace().schedule('every 1 day');
-            jobs.job('api:task').replace().schedule('every 5 minutes');
-            jobs.job('api:userMerge').replace().schedule('every 10 minutes');
-            jobs.job("api:ttlCleanup").replace().schedule("every 1 minute");
-            //jobs.job('api:appExpire').replace().schedule('every 1 day');
-        }, 10000);
+        // setTimeout(() => {
+        // jobs.job('api:topEvents').replace().schedule('at 00:01 am ' + 'every 1 day'); // PORTED
+        // jobs.job('api:ping').replace().schedule('every 1 day'); // PORTED
+        // jobs.job('api:clear').replace().schedule('every 1 day'); // REMOVED
+        // jobs.job('api:clearTokens').replace().schedule('every 1 day'); // PORTED
+        // jobs.job('api:clearAutoTasks').replace().schedule('every 1 day'); // PORTED
+        // jobs.job('api:task').replace().schedule('every 5 minutes'); // PORTED
+        // jobs.job('api:userMerge').replace().schedule('every 10 minutes'); // PORTED
+        // jobs.job("api:ttlCleanup").replace().schedule("every 1 minute"); // PORTED
+        //jobs.job('api:appExpire').replace().schedule('every 1 day'); // Deprecated
+        // }, 10000);
 
         //Record as restarted
 

From dbb0981a4b8f498dde5a2e9d22335a8f678bf64f Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Thu, 23 Jan 2025 23:30:39 +0530
Subject: [PATCH 39/90] migrate core jobs

---
 api/jobs/clearAutoTasks.js | 17 ++++++++++++--
 api/jobs/clearTokens.js    | 19 ++++++++++++---
 api/jobs/ping.js           | 47 ++++++++++++++++++++++++++------------
 api/jobs/task.js           | 23 +++++++++++++++----
 api/jobs/topEvents.js      | 16 +++++++++++--
 api/jobs/ttlCleanup.js     | 17 ++++++++++++--
 api/jobs/userMerge.js      | 25 +++++++++++++++-----
 7 files changed, 129 insertions(+), 35 deletions(-)

diff --git a/api/jobs/clearAutoTasks.js b/api/jobs/clearAutoTasks.js
index f2bf6e929ac..8174626a96b 100644
--- a/api/jobs/clearAutoTasks.js
+++ b/api/jobs/clearAutoTasks.js
@@ -1,8 +1,10 @@
 "use strict";
 
-const job = require("../parts/jobs/job.js");
+// const job = require("../parts/jobs/job.js");
 const log = require('../utils/log.js')('job:clearAutoTasks');
 const taskManager = require('../utils/taskmanager');
+const Job = require("../../jobServer/Job");
+
 /**
  * clear task record in db with  task id
  * @param {string} taskId - the id of task in db.
@@ -20,7 +22,18 @@ const clearTaskRecord = (taskId) => {
 };
 
 /** Class for job of clearing auto tasks created long time ago **/
-class ClearAutoTasks extends job.Job {
+class ClearAutoTasks extends Job {
+    /**
+     * Get the schedule configuration for this job
+     * @returns {GetScheduleConfig} schedule configuration
+     */
+    getSchedule() {
+        return {
+            type: "schedule",
+            value: "0 2 * * *" // Every day at 2:00 AM
+        };
+    }
+
     /**
      * Run the job
      * @param {Db} db connection
diff --git a/api/jobs/clearTokens.js b/api/jobs/clearTokens.js
index 2a3932c2313..9587479acc7 100644
--- a/api/jobs/clearTokens.js
+++ b/api/jobs/clearTokens.js
@@ -1,10 +1,23 @@
 'use strict';
 
-const job = require('../parts/jobs/job.js'),
-    authorize = require('../utils/authorizer.js');
+// const job = require('../parts/jobs/job.js'),
+const authorize = require('../utils/authorizer.js');
+const Job = require("../../jobServer/Job");
 
 /** Class for job of clearing tokens **/
-class CleanTokensJob extends job.Job {
+class CleanTokensJob extends Job {
+
+    /**
+     * Get the schedule configuration for this job
+     * @returns {GetScheduleConfig} schedule configuration
+     */
+    getSchedule() {
+        return {
+            type: "schedule",
+            value: "30 2 * * *" // Every day at 2:30 AM
+        };
+    }
+
     /**
      * Run the job
      * @param {Db} db connection
diff --git a/api/jobs/ping.js b/api/jobs/ping.js
index 984e82c566c..f9c81a5780d 100644
--- a/api/jobs/ping.js
+++ b/api/jobs/ping.js
@@ -1,28 +1,45 @@
 'use strict';
 
-const job = require('../parts/jobs/job.js'),
-    log = require('../utils/log.js')('job:ping'),
-    countlyConfig = require("../../frontend/express/config.js"),
-    versionInfo = require('../../frontend/express/version.info'),
-    plugins = require('../../plugins/pluginManager.js'),
-    request = require('countly-request')(plugins.getConfig("security"));
+// const job = require('../parts/jobs/job.js');
+const log = require('../utils/log.js')('job:ping');
+const countlyConfig = require("../../frontend/express/config.js");
+const versionInfo = require('../../frontend/express/version.info');
+const plugins = require('../../plugins/pluginManager.js');
+
+
+const Job = require("../../jobServer/Job");
 
 
 /** Class for the job of pinging servers **/
-class PingJob extends job.Job {
+class PingJob extends Job {
+
+    /**
+     * Get the schedule configuration for this job
+     * @returns {GetScheduleConfig} schedule configuration
+     */
+    getSchedule() {
+        return {
+            type: "schedule",
+            value: "0 1 * * *" // Every day at 1:00 AM
+        };
+    }
+
     /**
      * Run the ping job
      * @param {Db} db connection
      * @param {done} done callback
      */
     run(db, done) {
-        request({strictSSL: false, uri: (process.env.COUNTLY_CONFIG_PROTOCOL || "http") + "://" + (process.env.COUNTLY_CONFIG_HOSTNAME || "localhost") + (countlyConfig.path || "") + "/configs"}, function() {});
-        var countlyConfigOrig = JSON.parse(JSON.stringify(countlyConfig));
-        var url = "https://count.ly/configurations/ce/tracking";
-        if (versionInfo.type !== "777a2bf527a18e0fffe22fb5b3e322e68d9c07a6") {
-            url = "https://count.ly/configurations/ee/tracking";
-        }
+
         plugins.loadConfigs(db, function() {
+            const request = require('countly-request')(plugins.getConfig("security"));
+            request({strictSSL: false, uri: (process.env.COUNTLY_CONFIG_PROTOCOL || "http") + "://" + (process.env.COUNTLY_CONFIG_HOSTNAME || "localhost") + (countlyConfig.path || "") + "/configs"}, function() {});
+            var countlyConfigOrig = JSON.parse(JSON.stringify(countlyConfig));
+            var url = "https://count.ly/configurations/ce/tracking";
+            if (versionInfo.type !== "777a2bf527a18e0fffe22fb5b3e322e68d9c07a6") {
+                url = "https://count.ly/configurations/ee/tracking";
+            }
+
             const offlineMode = plugins.getConfig("api").offline_mode;
             const { countly_tracking } = plugins.getConfig('frontend');
             if (!offlineMode) {
@@ -56,12 +73,12 @@ class PingJob extends job.Job {
                                 let domain = plugins.getConfig('api').domain;
 
                                 try {
-                                    // try to extract hostname from full domain url
+                                // try to extract hostname from full domain url
                                     const urlObj = new URL(domain);
                                     domain = urlObj.hostname;
                                 }
                                 catch (_) {
-                                    // do nothing, domain from config will be used as is
+                                // do nothing, domain from config will be used as is
                                 }
 
                                 request({
diff --git a/api/jobs/task.js b/api/jobs/task.js
index 06b6893a7e7..adf4d2cc294 100644
--- a/api/jobs/task.js
+++ b/api/jobs/task.js
@@ -1,9 +1,10 @@
 'use strict';
 
-const job = require('../parts/jobs/job.js'),
-    log = require('../utils/log.js')('api:task'),
-    asyncjs = require("async"),
-    plugins = require('../../plugins/pluginManager.js');
+// const job = require('../parts/jobs/job.js');
+const Job = require("../../jobServer/Job");
+const log = require('../utils/log.js')('api:task');
+const asyncjs = require("async");
+const plugins = require('../../plugins/pluginManager.js');
 const common = require('../utils/common.js');
 const taskmanager = require('../utils/taskmanager.js');
 
@@ -14,7 +15,19 @@ common.processRequest = processRequest;
 /**
  *  Task Monitor Job extend from Countly Job
  */
-class MonitorJob extends job.Job {
+class MonitorJob extends Job {
+
+    /**
+     * Get the schedule configuration for this job
+     * @returns {GetScheduleConfig} schedule configuration
+     */
+    getSchedule() {
+        return {
+            type: "schedule",
+            value: "*/5 * * * *" // Every 5 minutes
+        };
+    }
+
     /**
      * Run the job
      * @param {Db} db connection
diff --git a/api/jobs/topEvents.js b/api/jobs/topEvents.js
index 44913509632..10d6161ffe4 100644
--- a/api/jobs/topEvents.js
+++ b/api/jobs/topEvents.js
@@ -1,4 +1,5 @@
-const job = require("../parts/jobs/job.js");
+// const job = require("../parts/jobs/job.js");
+const Job = require("../../jobServer/Job");
 const crypto = require("crypto");
 const Promise = require("bluebird");
 const countlyApi = {
@@ -14,7 +15,7 @@ const common = require('../utils/common.js');
 const log = require('../utils/log.js')('job:topEvents');
 
 /** Class for job of top events widget **/
-class TopEventsJob extends job.Job {
+class TopEventsJob extends Job {
 
     /**
      * TopEvents initialize function
@@ -215,6 +216,17 @@ class TopEventsJob extends job.Job {
         this.init();
         done();
     }
+
+    /**
+     * Get schedule
+     * @returns {GetScheduleConfig} schedule
+     */
+    getSchedule() {
+        return {
+            type: "schedule",
+            value: "1 0 * * *" // every day at 00:01
+        };
+    }
 }
 
 /**
diff --git a/api/jobs/ttlCleanup.js b/api/jobs/ttlCleanup.js
index 1c168d38b21..5de0df26dbd 100644
--- a/api/jobs/ttlCleanup.js
+++ b/api/jobs/ttlCleanup.js
@@ -1,12 +1,25 @@
 const plugins = require("../../plugins/pluginManager.js");
 const common = require('../utils/common');
-const job = require("../parts/jobs/job.js");
+// const job = require("../parts/jobs/job.js");
 const log = require("../utils/log.js")("job:ttlCleanup");
+const Job = require("../../jobServer/Job");
 
 /**
  * Class for job of cleaning expired records inside ttl collections
  */
-class TTLCleanup extends job.Job {
+class TTLCleanup extends Job {
+
+    /**
+     * Get the schedule configuration for this job
+     * @returns {GetScheduleConfig} schedule configuration
+     */
+    getSchedule() {
+        return {
+            type: "schedule",
+            value: "* * * * *" // Every minute
+        };
+    }
+
     /**
      * Run the job
      */
diff --git a/api/jobs/userMerge.js b/api/jobs/userMerge.js
index 17d3a8022b7..871a2b1ceff 100644
--- a/api/jobs/userMerge.js
+++ b/api/jobs/userMerge.js
@@ -1,10 +1,11 @@
 'use strict';
 
-const job = require('../parts/jobs/job.js'),
-    plugins = require('../../plugins/pluginManager.js'),
-    log = require('../utils/log.js')('job:userMerge');
-var Promise = require("bluebird");
-var usersApi = require('../parts/mgmt/app_users.js');
+// const job = require('../parts/jobs/job.js');
+const Job = require("../../jobServer/Job");
+const plugins = require('../../plugins/pluginManager.js');
+const log = require('../utils/log.js')('job:userMerge');
+const Promise = require("bluebird");
+const usersApi = require('../parts/mgmt/app_users.js');
 
 
 var getMergeDoc = function(data) {
@@ -223,7 +224,19 @@ var handleMerges = function(db, callback) {
     });
 };
 /** Class for the user mergind job **/
-class UserMergeJob extends job.Job {
+class UserMergeJob extends Job {
+
+    /**
+     * Get the schedule configuration for this job
+     * @returns {GetScheduleConfig} schedule configuration
+     */
+    getSchedule() {
+        return {
+            type: "schedule",
+            value: "*/5 * * * *" // Every 5 minutes
+        };
+    }
+
     /**
      * Run the job
      * @param {Db} db connection

From b87ae763b16c13c2166831df5eaad320db94d135 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Thu, 23 Jan 2025 23:30:58 +0530
Subject: [PATCH 40/90] prevent double loading of jobs in old and new

---
 api/parts/jobs/scanner.js | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/api/parts/jobs/scanner.js b/api/parts/jobs/scanner.js
index 771e234ed97..2050a0eb1c7 100644
--- a/api/parts/jobs/scanner.js
+++ b/api/parts/jobs/scanner.js
@@ -83,7 +83,12 @@ module.exports = (db, filesObj, classesObj) => {
                         }
                     }
                     catch (e) {
-                        log.e('Error when loading job %s: %j ', job.file, e, e.stack);
+                        if (e.message === "Job class must extend Job, IPCJob, IPCFaçadeJob, or TransientJob") {
+                            // do nothing
+                        }
+                        else {
+                            log.e('Error when loading job %s: %j ', job.file, e, e.stack);
+                        }
                     }
                 });
             });

From b7edf022505820e051ebda3ca18b78e36e37ed4c Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Thu, 23 Jan 2025 23:31:32 +0530
Subject: [PATCH 41/90] fixes for runnow and priority/retry configs

---
 jobServer/jobRunner/impl/pulse/PulseJobExecutor.js  | 4 ++--
 jobServer/jobRunner/impl/pulse/PulseJobScheduler.js | 3 ++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js b/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js
index a55c3bc1fd1..2be672c1279 100644
--- a/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js
+++ b/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js
@@ -60,7 +60,7 @@ class PulseJobExecutor extends IJobExecutor {
                     concurrency,
                     lockLifetime,
                     shouldSaveResult: true,
-                    attempts: retryConfig?.enabled ? retryConfig.attempts : 1,
+                    attempts: retryConfig?.enabled ? retryConfig.attempts : 0,
                     backoff: retryConfig?.enabled ? {
                         type: 'exponential',
                         delay: retryConfig.delay
@@ -200,7 +200,7 @@ class PulseJobExecutor extends IJobExecutor {
             [JOB_PRIORITIES.HIGHEST]: JobPriority.highest
         };
 
-        if (!priority || !priorityMap[priority]) {
+        if (!priority || priorityMap[priority] === undefined) {
             this.log.w(`Invalid priority "${priority}", defaulting to normal`);
             return JobPriority.normal;
         }
diff --git a/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js b/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js
index 8c1d5f50b8a..951494bd4cd 100644
--- a/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js
+++ b/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js
@@ -162,11 +162,12 @@ class PulseJobScheduler extends IJobScheduler {
      * @param {string} jobName - Name of the job to execute
      * @returns {Promise<void>} Resolves when job is successfully triggered
      * @throws {Error} If immediate execution fails
+     * @note Implement data passing if needed
      */
     async runJobNow(jobName) {
         try {
             this.log.d(`Attempting to trigger immediate execution of job '${jobName}'`);
-            await this.pulseRunner.now({ name: jobName });
+            await this.pulseRunner.now(jobName, {});
             this.log.i(`Successfully triggered immediate execution of job '${jobName}'`);
         }
         catch (error) {

From 2f331105b5a344b6416a62a1187b9764736f0e56 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Thu, 23 Jan 2025 23:31:52 +0530
Subject: [PATCH 42/90] scan new job every 30 seconds

---
 jobServer/config.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/jobServer/config.js b/jobServer/config.js
index ceef4454397..22a9125b63a 100644
--- a/jobServer/config.js
+++ b/jobServer/config.js
@@ -15,7 +15,7 @@
  * @property {string} db.collection - MongoDB collection name for storing jobs
  */
 const DEFAULT_PULSE_CONFIG = {
-    processEvery: '60 seconds',
+    processEvery: '30 seconds',
     maxConcurrency: 1,
     defaultConcurrency: 1,
     lockLimit: 1,

From 70f5b6af8722e09b22e40483f58dfa2b0900fd48 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Thu, 23 Jan 2025 23:32:48 +0530
Subject: [PATCH 43/90] refactoring and backward compatiblity injections in
 jobserver

---
 jobServer/Job.js        |  28 ++---
 jobServer/JobManager.js |  32 +++--
 jobServer/JobScanner.js |   4 +-
 jobServer/JobServer.js  |  95 +++++----------
 jobServer/index.js      | 255 ++++++++++++++++++++++++++++++++++++----
 5 files changed, 303 insertions(+), 111 deletions(-)

diff --git a/jobServer/Job.js b/jobServer/Job.js
index 4699c862e42..76f2271e617 100644
--- a/jobServer/Job.js
+++ b/jobServer/Job.js
@@ -1,4 +1,4 @@
-const defaultLogger = { d: console.debug, w: console.warn, e: console.error, i: console.info };
+const Logger = require('../api/utils/log.js');
 const { JOB_PRIORITIES } = require('./constants/JobPriorities');
 
 /**
@@ -19,7 +19,7 @@ const { JOB_PRIORITIES } = require('./constants/JobPriorities');
  */
 
 /**
- * @typedef {Object} ScheduleConfig
+ * @typedef {Object} GetScheduleConfig
  * @property {('once'|'schedule'|'now')} type - Type of schedule
  * @property {(string|Date)} [value] - Schedule value (cron expression or Date)
  */
@@ -79,12 +79,10 @@ const { JOB_PRIORITIES } = require('./constants/JobPriorities');
  * }
  */
 class Job {
+
     /** @type {string} Name of the job */
     jobName = Job.name;
 
-    /** @type {Logger} Logger instance */
-    logger;
-
     /** @type {Function|null} Touch method from job runner */
     #touchMethod = null;
 
@@ -98,8 +96,8 @@ class Job {
      * Creates an instance of Job.
      */
     constructor() {
-        this.logger = defaultLogger;
-        this.logger.d(`Job instance "${this.jobName}" created`);
+        this.log = Logger('jobs:job');
+        this.log.d(`Job instance "${this.jobName}" created`);
     }
 
     /**
@@ -114,8 +112,8 @@ class Job {
      * Sets the logger
      * @param {Object} [logger=defaultLogger] The logger instance
      */
-    setLogger(logger = defaultLogger) {
-        this.logger = logger;
+    setLogger(logger = this.log) {
+        this.log = logger;
     }
 
     /**
@@ -160,7 +158,7 @@ class Job {
      * Get the schedule type and timing for the job.
      * This method must be implemented by the child class.
      * 
-     * @returns {Object} Schedule configuration object
+     * @returns {GetScheduleConfig} Schedule configuration object
      * @property {('once'|'schedule'|'now')} type - Type of schedule
      * @property {string|Date} [value] - Schedule value:
      *   - For type='schedule': Cron expression (e.g., '0 * * * *' for hourly)
@@ -209,7 +207,7 @@ class Job {
      * @private
      */
     async _run(db, job, done) {
-        this.logger.d(`[Job:${this.jobName}] Starting execution`, {
+        this.log.d(`[Job:${this.jobName}] Starting execution`, {
             database: db?._cly_debug?.db,
             jobId: job?.attrs?._id,
             jobName: this.jobName
@@ -243,14 +241,14 @@ class Job {
                 }
             });
 
-            this.logger.i(`[Job:${this.jobName}] Completed successfully`, {
+            this.log.i(`[Job:${this.jobName}] Completed successfully`, {
                 result: result || null,
                 duration: `${Date.now() - job?.attrs?.lastRunAt?.getTime()}ms`
             });
             done(null, result);
         }
         catch (error) {
-            this.logger.e(`[Job:${this.jobName}] Execution failed`, {
+            this.log.e(`[Job:${this.jobName}] Execution failed`, {
                 error: error.message,
                 stack: error.stack,
                 duration: `${Date.now() - job?.attrs?.lastRunAt?.getTime()}ms`
@@ -304,7 +302,7 @@ class Job {
             await this.#touchMethod(progress);
         }
 
-        this.logger?.d(`[Job:${this.jobName}] Progress update`, progressData);
+        this.log?.d(`[Job:${this.jobName}] Progress update`, progressData);
     }
 
     /**
@@ -318,7 +316,7 @@ class Job {
     getRetryConfig() {
         return {
             enabled: false,
-            attempts: 3,
+            attempts: 0,
             delay: 5 * 60 * 1000 // 5 minutes
         };
     }
diff --git a/jobServer/JobManager.js b/jobServer/JobManager.js
index 1f3c675975e..6d5fc536b28 100644
--- a/jobServer/JobManager.js
+++ b/jobServer/JobManager.js
@@ -18,6 +18,11 @@ const JobUtils = require('./JobUtils');
  * @property {Date} [updatedAt] - When the config was last updated
  * @property {string} checksum - Hash of the job implementation
  * @property {Object} defaultConfig - Original job configuration
+ * @property {Object} defaultConfig.schedule - Default schedule configuration
+ * @property {Object} defaultConfig.retry - Default retry configuration
+ * @property {number} defaultConfig.priority - Default job priority
+ * @property {number} defaultConfig.concurrency - Default concurrency limit
+ * @property {number} defaultConfig.lockLifetime - Default lock lifetime in milliseconds
  */
 
 /**
@@ -75,11 +80,24 @@ class JobManager {
         changeStream.on('change', async(change) => {
             this.#log.d('Detected config change:', {
                 operationType: change.operationType,
-                jobName: change.fullDocument?.jobName
+                documentId: change.documentKey?._id,
+                updatedFields: change.updateDescription?.updatedFields
             });
-            if (change.operationType === 'update' || change.operationType === 'insert') {
-                const jobConfig = change.fullDocument;
-                await this.#applyConfig(jobConfig);
+
+            // Dont really need this, as we are only updating the config
+            // if (change.operationType === 'insert') {
+            //     const jobConfig = change.fullDocument;
+            //     await this.#applyConfig(jobConfig);
+            // }
+            // else 
+            if (change.operationType === 'update') {
+                // Fetch the complete document after update
+                const jobConfig = await this.#jobConfigsCollection.findOne({
+                    _id: change.documentKey._id
+                });
+                if (jobConfig) {
+                    await this.#applyConfig({...change?.updateDescription?.updatedFields, jobName: jobConfig.jobName});
+                }
             }
         });
     }
@@ -123,8 +141,8 @@ class JobManager {
                 await this.#jobRunner.configureRetry(jobName, jobConfig.retry);
             }
 
-            if (typeof jobConfig.enabled === 'boolean') {
-                if (jobConfig.enabled) {
+            if (typeof jobConfig?.enabled === 'boolean') {
+                if (jobConfig?.enabled) {
                     await this.#jobRunner.enableJob(jobName);
                     this.#log.i(`Job ${jobName} enabled via config`);
                 }
@@ -136,7 +154,7 @@ class JobManager {
         }
         catch (error) {
             this.#log.e('Failed to apply job configuration:', {
-                jobName: jobConfig.jobName,
+                jobName: jobConfig?.jobName || "unknown",
                 error: error.message,
                 stack: error.stack
             });
diff --git a/jobServer/JobScanner.js b/jobServer/JobScanner.js
index 616dc988375..53fefb8082b 100644
--- a/jobServer/JobScanner.js
+++ b/jobServer/JobScanner.js
@@ -129,7 +129,7 @@ class JobScanner {
         catch (err) {
             const message = `Failed to read directory ${jobConfig.dir}: ${err.message}`;
             if (jobConfig.required) {
-                this.#log.e(message, err);
+                this.#log.w(message, err);
                 throw new Error(message);
             }
             this.#log.w(message);
@@ -159,7 +159,7 @@ class JobScanner {
             };
         }
         catch (err) {
-            this.#log.e(`Failed to load job ${job.file}: ${err.message}`, err);
+            this.#log.w(`Failed to load job ${job.file}: ${err.message}`, err);
             return null;
         }
     }
diff --git a/jobServer/JobServer.js b/jobServer/JobServer.js
index e73415c2838..7d62ec807d4 100644
--- a/jobServer/JobServer.js
+++ b/jobServer/JobServer.js
@@ -31,13 +31,6 @@ class JobServer {
      */
     #pluginManager;
 
-    /**
-     * The Countly common object
-     * @private
-     * @type {import('../api/utils/common.js')}
-     */
-    #common;
-
     /**
      * The job manager instance
      * @private
@@ -78,31 +71,44 @@ class JobServer {
      */
     #isShuttingDown = false;
 
+    /**
+     * @typedef {Object} DbConnections
+     * @property {MongoDb} countlyDb - Main Countly database connection
+     * @property {MongoDb} drillDb - Drill database connection
+     * @property {MongoDb} outDb - Output database connection
+     */
+    /**
+     * The database connections object
+     * @private
+     * @type {DbConnections}
+     */
+    #dbConnections;
+
     /**
      * Factory method to create and initialize a new JobServer instance.
-     * @param {Object} common - Countly common utilities
      * @param {Logger} Logger - Logger constructor
      * @param {PluginManager} pluginManager - Plugin manager instance
+     * @param {DbConnections} dbConnections - Database connections object
      * @returns {Promise<JobServer>} A fully initialized JobServer instance
      * @throws {Error} If initialization fails
      */
-    static async create(common, Logger, pluginManager) {
-        const process = new JobServer(common, Logger, pluginManager);
+    static async create(Logger, pluginManager, dbConnections) {
+        const process = new JobServer(Logger, pluginManager, dbConnections);
         await process.init();
         return process;
     }
 
     /**
      * Creates a new JobServer instance
-     * @param {Object} common Countly common
-     * @param {function} Logger - Logger constructor
-     * @param {pluginManager} pluginManager - Plugin manager instance
+     * @param {Logger} Logger - Logger constructor
+     * @param {PluginManager} pluginManager - Plugin manager instance
+     * @param {Object} dbConnections - Database connections object
      */
-    constructor(common, Logger, pluginManager) {
-        this.#common = common;
+    constructor(Logger, pluginManager, dbConnections) {
         this.Logger = Logger;
-        this.#log = Logger('jobs:server');
         this.#pluginManager = pluginManager;
+        this.#dbConnections = dbConnections;
+        this.#log = Logger('jobs:server');
     }
 
     /**
@@ -114,16 +120,13 @@ class JobServer {
     async init() {
         try {
             this.#log.d('Initializing job server...');
-            await this.#connectToDb();
 
-            this.#jobManager = new JobManager(this.#db, this.Logger);
-            this.#jobScanner = new JobScanner(this.#db, this.Logger, this.#pluginManager);
+            this.#jobManager = new JobManager(this.#dbConnections.countlyDb, this.Logger);
+            this.#jobScanner = new JobScanner(this.#dbConnections.countlyDb, this.Logger, this.#pluginManager);
 
-            this.#jobConfigsCollection = this.#db.collection(JOBS_CONFIG_COLLECTION);
+            this.#jobConfigsCollection = this.#dbConnections.countlyDb.collection(JOBS_CONFIG_COLLECTION);
             // await this.#jobConfigsCollection.createIndex({ jobName: 1 }, /*{ unique: true }*/);
 
-            this.#setupSignalHandlers();
-
             this.#log.i('Job server initialized successfully');
         }
         catch (error) {
@@ -155,52 +158,16 @@ class JobServer {
         }
         catch (error) {
             this.#log.e('Critical error during server startup: %j', error);
-            await this.#shutdown(1);
+            await this.shutdown(1);
         }
     }
 
-    /**
-     * Connects to the mongo database.
-     * @returns {Promise<void>} A promise that resolves once the connection is established.
-     */
-    async #connectToDb() {
-        try {
-            this.#db = await this.#pluginManager.dbConnection('countly');
-        }
-        catch (e) {
-            this.#log.e('Failed to connect to database:', e);
-            throw e;
-        }
-    }
-
-    /**
-     * Sets up process signal handlers for graceful shutdown and error handling.
-     * @private
-     */
-    #setupSignalHandlers() {
-        process.on('SIGTERM', () => {
-            this.#log.i('Received SIGTERM signal');
-            this.#shutdown();
-        });
-
-        process.on('SIGINT', () => {
-            this.#log.i('Received SIGINT signal');
-            this.#shutdown();
-        });
-
-        process.on('uncaughtException', (error) => {
-            this.#log.e('Uncaught exception in job server: %j', error);
-            this.#shutdown(1);
-        });
-    }
-
     /**
      * Gracefully shuts down the job server, closing connections and stopping jobs.
-     * @private
      * @param {number} [exitCode=0] - Process exit code
      * @returns {Promise<void>} A promise that resolves once the job server is shut down
      */
-    async #shutdown(exitCode = 0) {
+    async shutdown(exitCode = 0) {
         if (this.#isShuttingDown) {
             this.#log.d('Shutdown already in progress, skipping duplicate request');
             return;
@@ -209,9 +176,11 @@ class JobServer {
 
         this.#log.i('Initiating job server shutdown...');
 
-        if (this.#db && typeof this.#db.close === 'function') {
-            this.#log.d('Closing database connection');
-            await this.#db.close();
+        for (const [key, db] of Object.entries(this.#dbConnections)) {
+            if (db && typeof db.close === 'function') {
+                this.#log.d(`Closing ${key} database connection`);
+                await db.close();
+            }
         }
 
         if (!this.#isRunning) {
diff --git a/jobServer/index.js b/jobServer/index.js
index c942a38f769..6f443327fea 100644
--- a/jobServer/index.js
+++ b/jobServer/index.js
@@ -2,16 +2,16 @@
  * @module jobServer
  * @version 2.0
  * @author Countly
- * 
+ *
  * @typedef {import('../api/utils/common.js')} Common
  * @typedef {import('../plugins/pluginManager.js')} PluginManager
  * @typedef {import('./Job')} JobType
  * @typedef {import('./JobServer')} JobServerType
- * 
+ *
  * @note
  * Dependencies like common utilities and plugin manager should only be imported in this entry file
  * and injected into the respective modules via their constructors or create methods.
- * 
+ *
  * @description
  * This module serves as the entry point for Countly's job management system.
  * It provides a robust infrastructure for:
@@ -19,14 +19,14 @@
  * - Managing job lifecycles and states
  * - Handling job dependencies and priorities
  * - Providing process isolation for job execution
- * 
+ *
  * @exports {Object} module.exports
  * @property {JobType} Job - Class for creating and managing individual jobs
  * @property {JobServerType} JobServer - Class for running jobs in a separate process
- * 
+ *
  * @throws {Error} DatabaseConnectionError When database connection fails during initialization
  * @throws {Error} InvalidJobError When job definition is invalid
- * 
+ *
  * @example
  * // Import and create a new job
  * const { Job } = require('./jobs');
@@ -38,43 +38,250 @@
  */
 
 const JobServer = require('./JobServer');
-const Job = require('./Job');
+const Logger = require('../api/utils/log.js');
+const log = new Logger('jobServer:index');
+const CountlyRequest = require("countly-request");
+const {ReadBatcher, WriteBatcher} = require('../api/parts/data/batcher');
 
 // Start the process if this file is run directly
 if (require.main === module) {
     const common = require('../api/utils/common.js');
     const pluginManager = require('../plugins/pluginManager.js');
-    const Logger = common.log;
-    const log = Logger('jobs:server');
+
+    /**
+     * @type {JobServer|null}
+     */
+    let jobServer = null;
 
     log.i('Initializing job server process...');
 
-    JobServer.create(common, Logger, pluginManager)
-        .then(process => {
+
+    /**
+     * Initialize configuration
+     * @returns {Promise<Object>} A promise that resolves to an object containing configuration
+     */
+    const initializeConfig = async() => {
+        try {
+            const config = await pluginManager.getConfig();
+            log.d('Configuration initialized successfully');
+            return config;
+        }
+        catch (error) {
+            log.e('Failed to initialize configuration:', {
+                error: error.message,
+                stack: error.stack
+            });
+            throw new Error('Configuration initialization failed: ' + error.message);
+        }
+    };
+
+    /**
+     * Initialize countly request
+     * @param {Object} config - Configuration object
+     * @returns {Promise<Object>} A promise that resolves to an object containing countly request
+     */
+    const initializeCountlyRequest = async(config) => {
+        try {
+            const countlyRequest = CountlyRequest(config.security);
+            log.d('Countly request initialized successfully');
+            return countlyRequest;
+        }
+        catch (error) {
+            log.e('Failed to initialize countly request:', {
+                error: error.message,
+                stack: error.stack,
+                config: config ? 'present' : 'missing'
+            });
+            throw new Error('Countly request initialization failed: ' + error.message);
+        }
+    };
+
+    /**
+     * Override common db objects with the provided connections
+     * @param {Object} dbConnections - Object containing database connections
+     */
+    const overrideCommonDb = async(dbConnections) => {
+        try {
+            if (!dbConnections || !dbConnections.countlyDb || !dbConnections.drillDb || !dbConnections.outDb) {
+                throw new Error('Invalid database connections provided');
+            }
+            const { countlyDb, drillDb, outDb } = dbConnections;
+            common.db = countlyDb;
+            common.drillDb = drillDb;
+            common.outDb = outDb;
+            log.d('Common db objects overridden successfully');
+        }
+        catch (error) {
+            log.e('Failed to override common db objects:', {
+                error: error.message,
+                stack: error.stack,
+                connections: Object.keys(dbConnections || {})
+            });
+            throw new Error('Common db override failed: ' + error.message);
+        }
+    };
+
+    /**
+     * Override common batcher with the provided connections
+     * @param {Db} commonDb - Object containing database connections
+     */
+    const overrideCommonBatcher = async(commonDb) => {
+        try {
+            common.writeBatcher = new WriteBatcher(commonDb);
+            common.readBatcher = new ReadBatcher(commonDb);
+        }
+        catch (error) {
+            log.e('Failed to override common batcher:', {
+                error: error.message,
+                stack: error.stack
+            });
+        }
+    };
+
+    /**
+     * Initialize database connections
+     * @returns {Promise<DbConnections>} A promise that resolves to an object containing database connections
+     */
+    const initializeDbConnections = async() => {
+        try {
+            log.d('Initializing database connections...');
+            const countlyDb = await pluginManager.dbConnection('countly');
+            log.d('Countly DB connection established');
+
+            const drillDb = await pluginManager.dbConnection('countly_drill');
+            log.d('Drill DB connection established');
+
+            const outDb = await pluginManager.dbConnection('countly_out');
+            log.d('Out DB connection established');
+
+            return {
+                countlyDb,
+                drillDb,
+                outDb,
+            };
+        }
+        catch (error) {
+            log.e('Failed to initialize database connections:', {
+                error: error.message,
+                stack: error.stack
+            });
+            throw new Error('Database connections initialization failed: ' + error.message);
+        }
+    };
+
+    /**
+     * Override common dispatch with a Countly server request
+     * @param {Function} countlyRequest - Countly request function
+     * @param {Object} countlyConfig - Countly config object
+     */
+    const overrideCommonDispatch = async(countlyRequest, countlyConfig) => {
+        try {
+            if (!countlyRequest || !countlyConfig) {
+                throw new Error('Invalid parameters for dispatch override');
+            }
+
+            const protocol = process.env.COUNTLY_CONFIG_PROTOCOL || "http";
+            const hostname = process.env.COUNTLY_CONFIG_HOSTNAME || "localhost";
+            const pathPrefix = countlyConfig.path || "";
+
+            // The base URL of the running Countly server
+            const baseUrl = protocol + "://" + hostname + pathPrefix;
+            log.d('Configuring dispatch override with base URL:', baseUrl);
+
+            /**
+             * Keep the same signature, but perform a request to the running Countly server
+             * @param {string} event - The event name
+             * @param {Object} params - The event parameters
+             * @param {Function} callback - Callback function
+             */
+            common.dispatch = function(event, params, callback) {
+                // Construct the request to your Countly server
+                countlyRequest.post({
+                    url: baseUrl + "/i/plugins/dispatch",
+                    json: { event: event, params: params }
+                }, function(err, res, body) {
+                    if (err) {
+                        log.e("Error dispatching event to Countly server:", {
+                            error: err,
+                            event: event,
+                            params: params,
+                            baseUrl: baseUrl,
+                        });
+                    }
+                    if (typeof callback === 'function') {
+                        return callback(err, res, body);
+                    }
+                    else {
+                        log.e("Callback is not a function");
+                    }
+                });
+            };
+            log.d('Common dispatch successfully overridden');
+        }
+        catch (error) {
+            log.e('Failed to override common dispatch:', {
+                error: error.message,
+                stack: error.stack,
+                config: countlyConfig ? 'present' : 'missing',
+                request: countlyRequest ? 'present' : 'missing'
+            });
+            throw new Error('Dispatch override failed: ' + error.message);
+        }
+    };
+
+
+    initializeConfig()
+        .then(async config => {
+            log.i('Starting job server initialization sequence...');
+
+            const countlyRequest = await initializeCountlyRequest(config);
+            await overrideCommonDispatch(countlyRequest, config);
+
+            const dbConnections = await initializeDbConnections();
+            await overrideCommonDb(dbConnections);
+            await overrideCommonBatcher(dbConnections.countlyDb);
+
+            jobServer = await JobServer.create(Logger, pluginManager, dbConnections);
+
             log.i('Job server successfully created, starting process...');
-            return process.start();
+            return jobServer.start();
         })
         .catch(error => {
             log.e('Critical error during job server initialization:', {
                 error: error.message,
-                stack: error.stack
+                stack: error.stack,
+                type: error.constructor.name,
+                time: new Date().toISOString()
             });
-            process.exit(1);
+            if (require.main === module) {
+                log.e('Exiting process due to critical initialization error');
+                process.exit(1);
+            }
         });
 
-    // Handle process signals for graceful shutdown
-    ['SIGTERM', 'SIGINT'].forEach(signal => {
-        process.on(signal, () => {
-            log.i(`Received ${signal}, initiating graceful shutdown...`);
-            // Allow time for cleanup before force exit
-            setTimeout(() => {
-                log.e('Forced shutdown after timeout');
-                process.exit(1);
-            }, 10000);
+    if (require.main === module) {
+        ['SIGTERM', 'SIGINT'].forEach(signal => {
+            process.on(signal, () => {
+                log.i(`Received ${signal}, initiating graceful shutdown...`);
+                if (jobServer && typeof jobServer.shutdown === 'function') {
+                    jobServer.shutdown();
+                }
+                else {
+                    setTimeout(() => {
+                        log.e('Forced shutdown after timeout');
+                        process.exit(1);
+                    }, 3000);
+                }
+            });
         });
-    });
+    }
 }
 
+/**
+ * Keeping old interface for backward compatibility
+ * @type {Job|{}}
+ */
+const Job = require('./Job');
 module.exports = {
     Job: Job
 };
\ No newline at end of file

From b25f4db7d2a03de8e83e845983fec05baa362910 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Thu, 23 Jan 2025 23:33:02 +0530
Subject: [PATCH 44/90] basic api for jobs

---
 api/utils/requestProcessor.js |  12 +-
 jobServer/api.js              | 572 ++++++++++++++++++++++++++++++++++
 2 files changed, 582 insertions(+), 2 deletions(-)
 create mode 100644 jobServer/api.js

diff --git a/api/utils/requestProcessor.js b/api/utils/requestProcessor.js
index eaa5bffad52..17f71afe129 100644
--- a/api/utils/requestProcessor.js
+++ b/api/utils/requestProcessor.js
@@ -23,7 +23,14 @@ const validateUserForDataWriteAPI = validateUserForWrite;
 const validateUserForGlobalAdmin = validateGlobalAdmin;
 const validateUserForMgmtReadAPI = validateUser;
 const request = require('countly-request')(plugins.getConfig("security"));
-const Handle = require('../../api/parts/jobs/index.js');
+
+try {
+    require('../../jobServer/api');
+    log.i('Job api loaded');
+}
+catch (ex) {
+    log.e('Job api not available');
+}
 
 var loaded_configs_time = 0;
 
@@ -2732,7 +2739,8 @@ const processRequest = (params) => {
                      * 
                     */
                     validateUserForGlobalAdmin(params, async() => {
-                        await Handle.suspendJob(params);
+                        // await Handle.suspendJob(params);
+                        // TODO: UPDATE TO NEW JOB PROCESSOR
                     });
                     break;
                 }
diff --git a/jobServer/api.js b/jobServer/api.js
new file mode 100644
index 00000000000..0908c517f17
--- /dev/null
+++ b/jobServer/api.js
@@ -0,0 +1,572 @@
+const Logger = require('../api/utils/log.js');
+const log = new Logger('jobs:api');
+
+const plugins = require("../plugins/pluginManager");
+const {validateGlobalAdmin} = require("../api/utils/rights");
+const common = require("../api/utils/common");
+const cronstrue = require('cronstrue');
+const moment = require('moment');
+
+/**
+ * Maps job status based on job properties
+ * @param {Object} job - The job object
+ * @param {Date} [job.lockedAt] - When the job was locked
+ * @param {Date} [job.failedAt] - When the job failed
+ * @param {Date} [job.lastFinishedAt] - When the job last finished
+ * @returns {string} Status - "RUNNING", "FAILED", "COMPLETED", or "SCHEDULED"
+ */
+const getJobStatus = (job) => {
+    if (job.lockedAt) {
+        return "RUNNING";
+    }
+    if (job.failedAt) {
+        return "FAILED";
+    }
+    if (job.lastFinishedAt) {
+        return "COMPLETED";
+    }
+    return "SCHEDULED";
+};
+
+/**
+ * Maps job run status based on job properties
+ * @param {Object} job - The job object
+ * @param {Date} [job.failedAt] - When the job failed
+ * @param {Date} [job.lastFinishedAt] - When the job last finished
+ * @returns {string} Status - "failed", "success", or "pending"
+ */
+const getRunStatus = (job) => {
+    if (job.failedAt) {
+        return "failed";
+    }
+    if (job.lastFinishedAt) {
+        return "success";
+    }
+    return "pending";
+};
+
+// Format job duration helper
+const formatJobDuration = (startDate, endDate) => {
+    if (!startDate || !endDate) {
+        return null;
+    }
+    return ((new Date(endDate) - new Date(startDate)) / 1000).toFixed(2);
+};
+
+// Get schedule label helper
+const getScheduleLabel = (schedule) => {
+    if (!schedule) {
+        return '';
+    }
+    try {
+        return cronstrue.toString(schedule);
+    }
+    catch (e) {
+        return schedule;
+    }
+};
+
+/**
+ * Job management API endpoint for controlling job configurations
+ * @name /i/jobs
+ * @example
+ * // Enable a job
+ * POST /i/jobs
+ * {
+ *   "jobName": "cleanupJob",
+ *   "action": "enable"
+ * }
+ * Response: { "code": 200, "message": "Success" }
+ * 
+ * // Disable a job
+ * POST /i/jobs
+ * {
+ *   "jobName": "cleanupJob",
+ *   "action": "disable"
+ * }
+ * Response: { "code": 200, "message": "Success" }
+ * 
+ * // Trigger immediate job execution
+ * POST /i/jobs
+ * {
+ *   "jobName": "cleanupJob",
+ *   "action": "runNow"
+ * }
+ * Response: { "code": 200, "message": "Success" }
+ * 
+ * // Update schedule
+ * POST /i/jobs
+ * {
+ *   "jobName": "cleanupJob",
+ *   "action": "updateSchedule",
+ *   "schedule": "0 0 * * *"  // Runs at midnight every day
+ * }
+ * Response: { "code": 200, "message": "Success" }
+ * 
+ * // Update retry configuration
+ * POST /i/jobs
+ * {
+ *   "jobName": "cleanupJob",
+ *   "action": "updateRetry",
+ *   "retry": {
+ *     "attempts": 3,    // Number of retry attempts
+ *     "delay": 300      // Delay between retries in seconds
+ *   }
+ * }
+ * Response: { "code": 200, "message": "Success" }
+ * 
+ * @returns {Object} Response
+ * @returns {number} Response.code - Status codes:
+ *                                  200: Success
+ *                                  400: Invalid parameters or missing required fields
+ *                                  500: Internal server error
+ * @returns {string} Response.message - Status message describing the result
+ */
+plugins.register('/i/jobs', async function(ob) {
+    validateGlobalAdmin(ob.params, async function() {
+        const {jobName, schedule, retry} = ob.params.qstring || {};
+        const action = ob.params.qstring?.action;
+
+        if (!jobName) {
+            common.returnMessage(ob.params, 400, 'Job name is required');
+            return;
+        }
+
+        const jobsCollection = common.db.collection('jobConfigs');
+        const updateData = { updatedAt: new Date() };
+
+        try {
+            switch (action) {
+            case 'enable':
+                updateData.enabled = true;
+                break;
+            case 'disable':
+                updateData.enabled = false;
+                break;
+            case 'runNow':
+                updateData.runNow = true;
+                break;
+            case 'updateSchedule':
+                if (!schedule) {
+                    common.returnMessage(ob.params, 400, 'Schedule configuration is required');
+                    return;
+                }
+                updateData.schedule = schedule;
+                break;
+            case 'updateRetry':
+                if (!retry) {
+                    common.returnMessage(ob.params, 400, 'Retry configuration is required');
+                    return;
+                }
+                updateData.retry = retry;
+                break;
+            default:
+                common.returnMessage(ob.params, 400, 'Invalid action');
+                return;
+            }
+
+            await jobsCollection.updateOne(
+                { jobName },
+                { $set: updateData }
+            );
+            common.returnMessage(ob.params, 200, 'Success');
+        }
+        catch (error) {
+            log.e('Error in jobs API:', { error: error.message, stack: error.stack });
+            common.returnMessage(ob.params, 500, 'Internal server error');
+        }
+    });
+    return true;
+});
+
+/**
+ * Job query API endpoint for retrieving job information
+ * @name /o/jobs
+ * @example
+ * // Get all jobs with pagination and search
+ * GET /o/jobs
+ * {
+ *   "iDisplayStart": 0,        // Pagination start index
+ *   "iDisplayLength": 50,      // Number of records per page
+ *   "sSearch": "backup",       // Search term for job names
+ *   "iSortCol_0": "nextRunAt", // Column to sort by
+ *   "sSortDir_0": "desc"       // Sort direction: "asc" or "desc"
+ * }
+ * Response: {
+ *   "sEcho": "1",
+ *   "iTotalRecords": 100,
+ *   "iTotalDisplayRecords": 5,
+ *   "aaData": [{
+ *     "job": {
+ *       "name": "backupJob",
+ *       "status": "SCHEDULED",
+ *       "schedule": "0 0 * * *",
+ *       "scheduleLabel": "At 12:00 AM",
+ *       "nextRunAt": "2024-03-20T00:00:00.000Z",
+ *       "lastFinishedAt": "2024-03-19T00:00:00.000Z",
+ *       "lastRunStatus": "success",
+ *       "failReason": null
+ *     },
+ *     "config": {
+ *       "enabled": true,
+ *       "schedule": "0 0 * * *",
+ *       "retry": { "attempts": 3, "delay": 300 }
+ *     }
+ *   }]
+ * }
+ * 
+ * // Get specific job details with run history
+ * GET /o/jobs
+ * {
+ *   "name": "cleanupJob",
+ *   "iDisplayStart": 0,
+ *   "iDisplayLength": 50
+ * }
+ * Response: {
+ *   "sEcho": "1",
+ *   "iTotalRecords": 10,
+ *   "iTotalDisplayRecords": 10,
+ *   "aaData": [{
+ *     "lastRunAt": "2024-03-19T00:00:00.000Z",
+ *     "status": "COMPLETED",
+ *     "lastRunStatus": "success",
+ *     "duration": "45.32",
+ *     "failReason": null,
+ *     "result": { "processedRecords": 1000 },
+ *     "runCount": 1,
+ *     "dataAsString": "{\n  \"processedRecords\": 1000\n}"
+ *   }],
+ *   "jobDetails": {
+ *     "config": {
+ *       "enabled": true,
+ *       "defaultConfig": {
+ *         "schedule": { "value": "0 0 * * *" },
+ *         "retry": { "attempts": 3, "delay": 300 }
+ *       },
+ *       "scheduleLabel": "At 12:00 AM",
+ *       "scheduleOverride": null,
+ *       "retryOverride": null
+ *     },
+ *     "currentState": {
+ *       "status": "SCHEDULED",
+ *       "nextRun": "2024-03-20T00:00:00.000Z",
+ *       "lastRun": "2024-03-19T00:00:00.000Z",
+ *       "lastRunStatus": "success",
+ *       "failReason": null,
+ *       "lastRunDuration": "45.32",
+ *       "finishedCount": 10,
+ *       "totalRuns": 10
+ *     }
+ *   }
+ * }
+ * 
+ * @returns {Object} Response
+ * @returns {string} Response.sEcho - Echo parameter from request for DataTables
+ * @returns {number} Response.iTotalRecords - Total number of records before filtering
+ * @returns {number} Response.iTotalDisplayRecords - Number of records after filtering
+ * @returns {Array<Object>} Response.aaData - Array of job data or job runs
+ * @returns {Object} [Response.jobDetails] - Detailed job information when querying specific job
+ * @returns {Object} Response.jobDetails.config - Job configuration including schedule and retry settings
+ * @returns {Object} Response.jobDetails.currentState - Current job state including next run and statistics
+ */
+plugins.register('/o/jobs', async function(ob) {
+    validateGlobalAdmin(ob.params, async function() {
+        const db = common.db;
+        const jobsCollection = db.collection('pulseJobs');
+        const jobConfigsCollection = db.collection('jobConfigs');
+
+        try {
+            const {
+                name: jobName,
+                iDisplayStart,
+                iDisplayLength,
+                sSearch,
+                iSortCol_0,
+                sSortDir_0
+            } = ob.params.qstring;
+
+            // If jobName is provided, fetch detailed view
+            if (jobName) {
+                // Get job definition and config
+                const [jobConfig, latestJob] = await Promise.all([
+                    jobConfigsCollection.findOne({ jobName }),
+                    jobsCollection.findOne(
+                        { name: jobName, type: { $ne: 'single' } }, // Exclude single runs
+                        { sort: { lastRunAt: -1 } }
+                    )
+                ]);
+
+                // Get individual job runs
+                const jobRuns = await jobsCollection
+                    .find({
+                        name: jobName,
+                        type: 'single' // Only get single run instances
+                    })
+                    .sort({ lastRunAt: -1 })
+                    .skip(Number(iDisplayStart) || 0)
+                    .limit(Number(iDisplayLength) || 50)
+                    .toArray();
+
+                const total = await jobsCollection.countDocuments({
+                    name: jobName,
+                    type: 'single'
+                });
+
+                // Process job runs with more detailed information
+                const processedRuns = jobRuns.map(run => ({
+                    lastRunAt: run.lastRunAt,
+                    status: getJobStatus(run),
+                    lastRunStatus: getRunStatus(run),
+                    duration: formatJobDuration(run.lastRunAt, run.lastFinishedAt),
+                    failReason: run.failReason,
+                    result: run.result,
+                    runCount: run.runCount,
+                    dataAsString: JSON.stringify(run.result || {}, null, 2)
+                }));
+
+                // Structure job details with clear separation
+                const jobDetails = {
+                    config: {
+                        ...jobConfig,
+                        enabled: jobConfig?.enabled ?? true,
+                        defaultConfig: {
+                            schedule: { value: latestJob?.repeatInterval },
+                            retry: {
+                                attempts: latestJob?.attempts,
+                                delay: latestJob?.backoff
+                            }
+                        },
+                        scheduleLabel: getScheduleLabel(jobConfig?.defaultConfig?.schedule?.value),
+                        scheduleOverride: jobConfig?.schedule,
+                        retryOverride: jobConfig?.retry
+                    },
+                    currentState: latestJob ? {
+                        status: getJobStatus(latestJob),
+                        nextRun: latestJob.nextRunAt,
+                        lastRun: latestJob.lastFinishedAt,
+                        lastRunStatus: getRunStatus(latestJob),
+                        failReason: latestJob.failReason,
+                        lastRunDuration: formatJobDuration(latestJob.lastRunAt, latestJob.lastFinishedAt),
+                        finishedCount: latestJob.finishedCount,
+                        totalRuns: latestJob.runCount
+                    } : null
+                };
+
+                common.returnOutput(ob.params, {
+                    sEcho: ob.params.qstring.sEcho,
+                    iTotalRecords: total,
+                    iTotalDisplayRecords: total,
+                    aaData: processedRuns,
+                    jobDetails: jobDetails
+                });
+                return;
+            }
+
+            // Handle list view if no jobName provided
+            const skip = Number(iDisplayStart) || 0;
+            const limit = Number(iDisplayLength) || 50;
+            const search = sSearch || '';
+            const sort = {
+                [iSortCol_0 || 'nextRunAt']: sSortDir_0 === 'desc' ? -1 : 1
+            };
+
+            await handleListView(ob, jobsCollection, jobConfigsCollection, search, sort, skip, limit);
+        }
+        catch (error) {
+            log.e('Error in jobs API:', { error: error.message, stack: error.stack });
+            common.returnMessage(ob.params, 500, 'Internal server error');
+        }
+    });
+    return true;
+});
+
+/**
+ * Handles detailed view for a specific job
+ * @param {Object} ob - Request object
+ * @param {string} jobName - Name of the job
+ * @param {Collection} jobsCollection - MongoDB jobs collection
+ * @param {Collection} jobConfigsCollection - MongoDB job configs collection
+ * @param {number} skip - Number of records to skip
+ * @param {number} limit - Number of records to return
+ * @returns {Promise<void>}
+ */
+async function handleDetailedView(ob, jobName, jobsCollection, jobConfigsCollection, skip, limit) {
+    const query = { name: jobName };
+
+    const [jobRuns, jobConfig, latestJob] = await Promise.all([
+        jobsCollection.find({ ...query, type: "single" })
+            .sort({ nextRunAt: -1 })
+            .skip(skip)
+            .limit(limit)
+            .toArray(),
+        jobConfigsCollection.findOne({ jobName }),
+        jobsCollection.findOne(
+            { name: jobName },
+            { sort: { lastFinishedAt: -1, lastRunAt: -1 } }
+        )
+    ]);
+
+    const total = await jobsCollection.countDocuments({ ...query, type: "single" });
+
+    const processedRuns = jobRuns.map(run => ({
+        name: run.name,
+        status: getJobStatus(run),
+        schedule: run.repeatInterval,
+        scheduleLabel: run.repeatInterval ? getScheduleLabel(run.repeatInterval) : 'Manual Run',
+        next: run.nextRunAt,
+        finished: run.lastFinishedAt,
+        lastRunStatus: getRunStatus(run),
+        failReason: run.failReason,
+        data: run.data,
+        dataAsString: JSON.stringify(run.data, null, 2),
+        duration: formatJobDuration(run.lastRunAt, run.lastFinishedAt),
+        durationInSeconds: formatJobDuration(run.lastRunAt, run.lastFinishedAt) + 's',
+        lastRunAt: run.lastRunAt,
+        nextRunDate: run.nextRunAt ? moment(run.nextRunAt).format('D MMM, YYYY') : '-',
+        nextRunTime: run.nextRunAt ? moment(run.nextRunAt).format('HH:mm:ss') : '-',
+        lastRun: run.lastFinishedAt ? moment(run.lastFinishedAt).fromNow() : '-'
+    }));
+
+    const jobDetails = {
+        config: {
+            ...jobConfig,
+            scheduleLabel: jobConfig?.defaultConfig?.schedule?.value ?
+                getScheduleLabel(jobConfig.defaultConfig.schedule.value) : '-',
+        },
+        currentState: latestJob ? {
+            status: getJobStatus(latestJob),
+            nextRun: latestJob.nextRunAt,
+            lastRun: latestJob.lastFinishedAt,
+            lastRunStatus: getRunStatus(latestJob),
+            failReason: latestJob.failReason,
+            lastRunDuration: formatJobDuration(latestJob.lastRunAt, latestJob.lastFinishedAt)
+        } : null
+    };
+
+    common.returnOutput(ob.params, {
+        sEcho: ob.params.qstring.sEcho,
+        iTotalRecords: total,
+        iTotalDisplayRecords: total,
+        aaData: processedRuns,
+        jobDetails: jobDetails
+    });
+}
+
+/**
+ * Handles list view for all jobs
+ * @param {Object} ob - Request object
+ * @param {Collection} jobsCollection - MongoDB jobs collection
+ * @param {Collection} jobConfigsCollection - MongoDB job configs collection
+ * @param {string} search - Search string to filter jobs
+ * @param {Object} sort - Sort configuration
+ * @param {number} skip - Number of records to skip
+ * @param {number} limit - Number of records to return
+ * @returns {Promise<void>}
+ */
+async function handleListView(ob, jobsCollection, jobConfigsCollection, search, sort, skip, limit) {
+    const query = search ? { name: new RegExp(search, 'i') } : {};
+
+    const pipeline = [
+        { $match: query },
+        {
+            $group: {
+                _id: "$name",
+                doc: { $first: "$$ROOT" },
+                lastFinishedAt: { $max: "$lastFinishedAt" },
+                lastRunAt: { $max: "$lastRunAt" },
+                lastFailedAt: { $max: "$failedAt" },
+                lastStatus: {
+                    $first: {
+                        $cond: [
+                            { $eq: ["$type", "single"] },
+                            {
+                                status: {
+                                    $switch: {
+                                        branches: [
+                                            { case: { $ne: ["$lockedAt", null] }, then: "RUNNING" },
+                                            { case: { $ne: ["$failedAt", null] }, then: "FAILED" },
+                                            { case: { $ne: ["$lastFinishedAt", null] }, then: "COMPLETED" }
+                                        ],
+                                        default: "SCHEDULED"
+                                    }
+                                },
+                                failReason: "$failReason",
+                                lastRunStatus: {
+                                    $switch: {
+                                        branches: [
+                                            { case: { $ne: ["$failedAt", null] }, then: "failed" },
+                                            { case: { $ne: ["$lastFinishedAt", null] }, then: "success" }
+                                        ],
+                                        default: "pending"
+                                    }
+                                }
+                            },
+                            "$$ROOT"
+                        ]
+                    }
+                }
+            }
+        },
+        {
+            $addFields: {
+                "doc.lastFinishedAt": "$lastFinishedAt",
+                "doc.lastRunAt": "$lastRunAt",
+                "doc.failedAt": "$lastFailedAt",
+                "doc.status": "$lastStatus.status",
+                "doc.failReason": "$lastStatus.failReason",
+                "doc.lastRunStatus": "$lastStatus.lastRunStatus"
+            }
+        },
+        { $replaceRoot: { newRoot: "$doc" }},
+        { $sort: sort },
+        { $skip: skip },
+        { $limit: limit }
+    ];
+
+    const [jobs, configs, total] = await Promise.all([
+        jobsCollection.aggregate(pipeline).toArray(),
+        jobConfigsCollection.find({}).toArray(),
+        jobsCollection.distinct('name', query).then(names => names.length)
+    ]);
+
+    const configMap = configs.reduce((map, config) => {
+        map[config.jobName] = config;
+        return map;
+    }, {});
+
+    const processedJobs = jobs.map(job => {
+        const config = configMap[job.name] || {
+            enabled: true,
+            defaultConfig: {
+                schedule: { value: job.repeatInterval },
+                retry: { attempts: job.attempts, delay: job.backoff }
+            }
+        };
+
+        return {
+            job: {
+                name: job.name,
+                status: getJobStatus(job),
+                schedule: config.defaultConfig.schedule.value,
+                scheduleLabel: getScheduleLabel(config.defaultConfig.schedule.value),
+                nextRunAt: job.nextRunAt,
+                lastFinishedAt: job.lastFinishedAt,
+                lastRunStatus: getRunStatus(job),
+                failReason: job.failReason,
+                total
+            },
+            config: {
+                enabled: config.enabled,
+                schedule: config.defaultConfig.schedule.value,
+                retry: config.defaultConfig.retry
+            }
+        };
+    });
+
+    common.returnOutput(ob.params, {
+        sEcho: ob.params.qstring.sEcho,
+        iTotalRecords: total,
+        iTotalDisplayRecords: total,
+        aaData: processedJobs
+    });
+}
\ No newline at end of file

From ceaf8b934a8943ab2ff38411d283174588c4677b Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Thu, 23 Jan 2025 23:33:16 +0530
Subject: [PATCH 45/90] init jobs fe

---
 .../core/jobs/javascripts/countly.views.js    | 346 +++++++++++++-----
 .../core/jobs/templates/jobs-details.html     | 154 +++++---
 .../public/core/jobs/templates/jobs.html      |  94 ++++-
 3 files changed, 444 insertions(+), 150 deletions(-)

diff --git a/frontend/express/public/core/jobs/javascripts/countly.views.js b/frontend/express/public/core/jobs/javascripts/countly.views.js
index 1bfc8c5ad48..630ac7540d9 100644
--- a/frontend/express/public/core/jobs/javascripts/countly.views.js
+++ b/frontend/express/public/core/jobs/javascripts/countly.views.js
@@ -1,36 +1,34 @@
-/*global countlyAuth, countlyCommon, app, countlyVue, CV, countlyGlobal, CountlyHelpers, $ */
+/*global countlyAuth, countlyCommon, app, countlyVue, CV, countlyGlobal, CountlyHelpers, $, moment */
 
 (function() {
     var getColor = function(row) {
-        if (row.status === "SCHEDULED") {
+        if (row.status === "RUNNING") {
+            return "green";
+        }
+        else if (row.status === "SCHEDULED" && row.enabled) {
             return "yellow";
         }
-        else if (row.status === "SUSPENDED") {
+        else if (row.status === "SUSPENDED" || !row.enabled) {
             return "gray";
         }
-        else if (row.status === "CANCELLED") {
+        else if (row.status === "CANCELLED" || row.lastRunStatus === "failed") {
             return "red";
         }
-        else if (row.status === "RUNNING") {
-            return "green";
-        }
     };
     var updateScheduleRow = function(row) {
-        var index;
-        row.nextRunDate = countlyCommon.getDate(row.next);
-        row.nextRunTime = countlyCommon.getTime(row.next);
-        row.lastRun = countlyCommon.formatTimeAgo(row.finished);
-        row.scheduleLabel = row.schedule || "";
-        index = row.scheduleLabel.indexOf("starting on");
-        if (index > (-1)) {
-            row.scheduleLabel = row.schedule.substring(0, index).trim();
-            row.scheduleDetail = row.schedule.substring(index).trim();
-        }
-        if (row.schedule && row.schedule.startsWith("at")) {
-            index = row.schedule.indexOf("every");
-            row.scheduleDetail = row.schedule.substring(0, index).trim();
-            row.scheduleLabel = row.schedule.substring(index).trim();
+        // Format dates using moment
+        row.nextRunDate = row.nextRunAt ? moment(row.nextRunAt).format('YYYY-MM-DD') : '';
+        row.nextRunTime = row.nextRunAt ? moment(row.nextRunAt).format('HH:mm:ss') : '';
+        row.lastRun = row.lastFinishedAt ? moment(row.lastFinishedAt).fromNow() : '';
+
+        // Handle schedule display
+        if (row.config && row.config.defaultConfig && row.config.defaultConfig.schedule) {
+            row.schedule = row.config.defaultConfig.schedule.value;
+            row.configuredSchedule = row.config.schedule;
+            row.scheduleOverridden = row.configuredSchedule &&
+                                    row.configuredSchedule !== row.schedule;
         }
+
     };
     var JobsView = countlyVue.views.create({
         template: CV.T('/core/jobs/templates/jobs.html'),
@@ -42,25 +40,46 @@
                     self.loaded = false;
                     return {
                         type: "GET",
-                        url: countlyCommon.API_URL + "/o",
+                        url: countlyCommon.API_URL + "/o/jobs",
                         data: {
                             app_id: countlyCommon.ACTIVE_APP_ID,
-                            method: 'jobs'
+                            iDisplayStart: 0,
+                            iDisplayLength: 50
                         }
                     };
                 },
                 onReady: function(context, rows) {
                     self.loaded = true;
-                    var row;
-                    for (var i = 0; i < rows.length; i++) {
-                        row = rows[i];
+                    var processedRows = [];
+                    for (var i = 0; i < rows.aaData.length; i++) {
+                        var row = rows.aaData[i].job;
+                        var config = rows.aaData[i].config;
+
+                        // Process job status - now directly from API
+                        row.enabled = config.enabled;
+                        row.config = config;
+
+                        // Schedule handling
+                        row.schedule = config.schedule;
+                        row.scheduleLabel = row.scheduleLabel || '';
+
                         updateScheduleRow(row);
+                        processedRows.push(row);
                     }
-                    return rows;
+                    return processedRows;
                 }
             }));
             return {
                 loaded: true,
+                saving: false,
+                scheduleDialogVisible: false,
+                selectedJobConfig: {
+                    name: '',
+                    schedule: '',
+                    defaultSchedule: '',
+                    scheduleLabel: '',
+                    enabled: true
+                },
                 tableStore: tableStore,
                 remoteTableDataSource: countlyVue.vuex.getServerDataSource(tableStore, "jobsTable")
             };
@@ -71,6 +90,33 @@
             },
         },
         methods: {
+            formatDateTime(date) {
+                return date ? moment(date).format('D MMM, YYYY HH:mm:ss') : '-';
+            },
+            getStatusColor(details) {
+                if (!details.config.enabled) {
+                    return 'gray';
+                }
+                if (details.currentState.status === 'RUNNING') {
+                    return 'blue';
+                }
+                if (details.currentState.status === 'FAILED') {
+                    return 'red';
+                }
+                if (details.currentState.status === 'COMPLETED') {
+                    return 'green';
+                }
+                return 'yellow';
+            },
+            getRunStatusColor(status) {
+                if (status === 'success') {
+                    return 'green';
+                }
+                if (status === 'failed') {
+                    return 'red';
+                }
+                return 'gray';
+            },
             refresh: function(force) {
                 if (this.loaded || force) {
                     this.loaded = false;
@@ -82,94 +128,186 @@
             },
             getColor: getColor,
             handleCommand: function(command, row) {
-                if (row.rowId) {
+                if (row.name) {
                     var self = this;
-                    if (command === "change-job-status") {
-                        const suspend = row.status !== "SUSPENDED" ? true : false;
-                        var notifyType = "ok";
-                        $.ajax({
-                            type: "GET",
-                            url: countlyCommon.API_URL + "/o",
-                            data: {
-                                app_id: countlyCommon.ACTIVE_APP_ID,
-                                method: 'suspend_job',
-                                id: row.rowId,
-                                suspend: suspend
-                            },
-                            contentType: "application/json",
-                            success: function(res) {
-                                if (res.result) {
-                                    self.refresh(true);
-                                }
-                                else {
-                                    notifyType = "error";
-                                }
+                    if (command === 'schedule') {
+                        this.selectedJobConfig = {
+                            name: row.name,
+                            schedule: row.configuredSchedule || row.schedule,
+                            defaultSchedule: row.schedule,
+                            enabled: row.enabled
+                        };
+                        this.scheduleDialogVisible = true;
+                        return;
+                    }
+
+                    var data = {
+                        app_id: countlyCommon.ACTIVE_APP_ID,
+                        jobName: row.name,
+                        action: command
+                    };
+
+                    // You can switch to type: "POST" if desired;
+                    // server code accepts query params so GET also works.
+                    $.ajax({
+                        type: "GET",
+                        url: countlyCommon.API_URL + "/i/jobs",
+                        data: data,
+                        success: function(res) {
+                            if (res.result === "Success") {
+                                self.refresh(true);
                                 CountlyHelpers.notify({
-                                    type: notifyType,
-                                    message: res.message
+                                    type: "ok",
+                                    message: CV.i18n("jobs." + command + "-success")
                                 });
-                            },
-                            error: function(err) {
+                            }
+                            else {
                                 CountlyHelpers.notify({
                                     type: "error",
-                                    message: err.responseJSON.error
+                                    message: res.result
                                 });
                             }
+                        },
+                        error: function(err) {
+                            CountlyHelpers.notify({
+                                type: "error",
+                                message: err.responseJSON?.result || "Error"
+                            });
+                        }
+                    });
+                }
+            },
+            saveSchedule: function() {
+                var self = this;
+                self.saving = true;
+
+                $.ajax({
+                    type: "GET",
+                    url: countlyCommon.API_URL + "/i/jobs",
+                    data: {
+                        app_id: countlyCommon.ACTIVE_APP_ID,
+                        jobName: this.selectedJobConfig.name,
+                        action: 'updateSchedule',
+                        schedule: this.selectedJobConfig.schedule
+                    },
+                    success: function() {
+                        self.saving = false;
+                        self.scheduleDialogVisible = false;
+                        self.refresh(true);
+                        CountlyHelpers.notify({
+                            type: "ok",
+                            message: CV.i18n("jobs.schedule-updated")
+                        });
+                    },
+                    error: function(err) {
+                        self.saving = false;
+                        CountlyHelpers.notify({
+                            type: "error",
+                            message: err.responseJSON?.result || "Error"
                         });
                     }
-                }
+                });
             },
         }
     });
 
-    var JobDetailView = countlyVue.views.create({
-        template: CV.T('/core/jobs/templates/jobs-details.html'),
+    var JobDetailsView = countlyVue.views.BaseView.extend({
+        template: "#jobs-details-template",
         data: function() {
-            var self = this;
-            var tableStore = countlyVue.vuex.getLocalStore(countlyVue.vuex.ServerDataTable("jobsTable", {
-                columns: ['name', "schedule", "next", "finished", "status", "total"],
-                onRequest: function() {
-                    self.loaded = false;
-                    return {
-                        type: "GET",
-                        url: countlyCommon.API_URL + "/o",
-                        data: {
-                            app_id: countlyCommon.ACTIVE_APP_ID,
-                            method: 'jobs',
-                            name: self.job_name
-                        }
-                    };
-                },
-                onReady: function(context, rows) {
-                    self.loaded = true;
-                    var row;
-                    for (var i = 0; i < rows.length; i++) {
-                        row = rows[i];
-                        row.dataAsString = JSON.stringify(row.data, null, 2);
-                        row.durationInSeconds = (row.duration / 1000) + 's';
-                        updateScheduleRow(row);
-                    }
-                    return rows;
-                }
-            }));
             return {
-                job_name: this.$route.params.job_name,
-                loaded: true,
-                tableStore: tableStore,
-                remoteTableDataSource: countlyVue.vuex.getServerDataSource(tableStore, "jobsTable")
+                job_name: this.$route.params.jobName,
+                jobDetails: null,
+                jobRuns: [],
+                isLoading: false,
+                jobRunColumns: [
+                    { prop: "lastRunAt", label: CV.i18n('jobs.run-time'), sortable: true },
+                    { prop: "status", label: CV.i18n('jobs.status'), sortable: true },
+                    { prop: "duration", label: CV.i18n('jobs.duration'), sortable: true },
+                    { prop: "result", label: CV.i18n('jobs.result') }
+                ]
             };
         },
+        computed: {
+            hasOverrides: function() {
+                return this.jobDetails &&
+                       (this.jobDetails.config.scheduleOverride ||
+                        this.jobDetails.config.retryOverride);
+            }
+        },
         methods: {
-            refresh: function(force) {
-                if (this.loaded || force) {
-                    this.loaded = false;
-                    this.tableStore.dispatch("fetchJobsTable");
+            fetchJobDetails: function() {
+                var self = this;
+                self.isLoading = true;
+
+                CV.$.ajax({
+                    type: "GET",
+                    url: countlyCommon.API_PARTS.data.r + "/jobs",
+                    data: {
+                        "app_id": countlyCommon.ACTIVE_APP_ID,
+                        "name": self.job_name,
+                        "iDisplayStart": 0,
+                        "iDisplayLength": 50
+                    },
+                    dataType: "json",
+                    success: function(response) {
+                        // API now returns jobDetails directly
+                        self.jobDetails = response.jobDetails;
+
+                        // Process job runs from aaData
+                        self.jobRuns = (response.aaData || []).map(function(run) {
+                            return {
+                                lastRunAt: run.lastRunAt,
+                                status: run.status,
+                                duration: run.duration,
+                                result: run.result,
+                                failReason: run.failReason,
+                                dataAsString: run.dataAsString
+                            };
+                        });
+
+                        self.isLoading = false;
+                    },
+                    error: function() {
+                        self.isLoading = false;
+                        CountlyHelpers.notify({
+                            title: CV.i18n("common.error"),
+                            message: CV.i18n("jobs.details-fetch-error"),
+                            type: "error"
+                        });
+                    }
+                });
+            },
+            formatDateTime: function(date) {
+                return date ? moment(date).format('D MMM, YYYY HH:mm:ss') : '-';
+            },
+            calculateDuration: function(run) {
+                if (!run.lastRunAt || !run.lastFinishedAt) {
+                    return '-';
                 }
+                return ((new Date(run.lastFinishedAt) - new Date(run.lastRunAt)) / 1000).toFixed(2);
             },
-            navigate: function(id) {
-                app.navigate("#/manage/jobs/" + id);
+            getStatusColor: function(jobDetails) {
+                if (!jobDetails.config.enabled) {
+                    return "grey";
+                }
+                switch (jobDetails.currentState.status) {
+                case "RUNNING": return "blue";
+                case "FAILED": return "red";
+                case "COMPLETED": return "green";
+                default: return "yellow";
+                }
             },
-            getColor: getColor
+            getRunStatusColor: function(run) {
+                switch (run.status) {
+                case "success": return "green";
+                case "failed": return "red";
+                case "running": return "blue";
+                default: return "yellow";
+                }
+            }
+        },
+        mounted: function() {
+            this.fetchJobDetails();
         }
     });
 
@@ -182,7 +320,7 @@
 
     var getDetailedView = function() {
         return new countlyVue.views.BackboneWrapper({
-            component: JobDetailView,
+            component: JobDetailsView,
             vuex: [] //empty array if none
         });
     };
@@ -192,10 +330,20 @@
             this.renderWhenReady(getMainView());
         });
 
-        app.route("/manage/jobs/:name", "manageJobName", function(name) {
-            var view = getDetailedView();
-            view.params = {job_name: name};
-            this.renderWhenReady(view);
+        app.route("/manage/jobs/:jobName", 'jobs-details', function(jobName) {
+            var jobDetailsView = new countlyVue.views.BackboneWrapper({
+                component: JobDetailsView,
+                templates: [
+                    {
+                        namespace: "jobs",
+                        mapping: {
+                            "details-template": "/core/jobs/templates/jobs-details.html"
+                        }
+                    }
+                ]
+            });
+            jobDetailsView.params = { jobName: jobName };
+            this.renderWhenReady(jobDetailsView);
         });
     }
 })();
\ No newline at end of file
diff --git a/frontend/express/public/core/jobs/templates/jobs-details.html b/frontend/express/public/core/jobs/templates/jobs-details.html
index 4062a534513..637e9cbc94e 100644
--- a/frontend/express/public/core/jobs/templates/jobs-details.html
+++ b/frontend/express/public/core/jobs/templates/jobs-details.html
@@ -1,49 +1,121 @@
 <div v-bind:class="[componentId]">
-    <cly-header
-        :title="job_name"
-    >
+    <cly-header :title="job_name">
         <template v-slot:header-top>
             <cly-back-link :title="i18n('plugins.back')"></cly-back-link>
         </template>
     </cly-header>
     <cly-main>
-		<cly-section>
-			<cly-datatable-n :data-source="remoteTableDataSource" :default-sort="{prop: 'next', order: 'descending'}">
-                <template v-slot="scope">
-                    <el-table-column sortable="custom" prop="status" :label="i18n('jobs.job-status')">
-                        <template slot-scope="scope">   
-                            <cly-status-tag :text="scope.row.status" :color="getColor(scope.row)"></cly-status-tag>
-                        </template>
-                    </el-table-column>
-                    <el-table-column sortable="custom" prop="schedule" :label="i18n('jobs.job-schedule')">
-                        <template slot-scope="scope">
-                            <p>{{scope.row.scheduleLabel}}</p>
-                            <p style="color:#A7AEB8; font-size: 12px;">{{scope.row.scheduleDetail}}</p>
-                        </template>
-                    </el-table-column>
-                    <el-table-column sortable="custom" prop="next" :label="i18n('jobs.job-next-run')">
-                        <template slot-scope="scope">
-                            <p>{{scope.row.nextRunDate}}</p>
-                            <p style="color:#A7AEB8; font-size: 12px;">{{scope.row.nextRunTime}}</p>
-                        </template>
-                    </el-table-column>
-                    <el-table-column sortable="custom" prop="finished" :label="i18n('jobs.job-last-run')">
-                        <template slot-scope="scope">
-                            <p v-html="scope.row.lastRun"></p>
-                        </template>
-                    </el-table-column>    
-                    <el-table-column prop="dataAsString" :label="i18n('jobs.job-data')">
-                        <template slot-scope="scope">
-                            <pre class="has-ellipsis" v-tooltip="scope.row.dataAsString">{{scope.row.dataAsString}}</pre>
-                        </template>
-                    </el-table-column>
-                    <el-table-column sortable="custom" prop="duration" :label="i18n('jobs.run-duration')">
-                        <template slot-scope="scope">
-                            <p v-html="scope.row.durationInSeconds"></p>
-                        </template>
-                    </el-table-column>
-                </template>
-            </cly-datatable-n>
-		</cly-section>
+        <template v-if="jobDetails">
+            <!-- Job Overview Section -->
+            <cly-section>
+                <div class="bu-columns">
+                    <div class="bu-column">
+                        <h4 class="bu-is-size-4 bu-mb-4">Job Configuration</h4>
+                        <div class="bu-box">
+                            <!-- Base Configuration -->
+                            <div class="bu-mb-4">
+                                <h5 class="bu-is-size-5 bu-mb-2">Base Configuration</h5>
+                                <p class="bu-mb-2">
+                                    <strong>Status:</strong>
+                                    <cly-status-tag
+                                        :text="jobDetails.config.enabled ? jobDetails.currentState.status : 'DISABLED'"
+                                        :color="getStatusColor(jobDetails)"
+                                    >
+                                    </cly-status-tag>
+                                </p>
+                                <p class="bu-mb-2">
+                                    <strong>Default Schedule:</strong>
+                                    {{jobDetails.config.defaultConfig.schedule.value}}
+                                </p>
+                                <p class="bu-mb-2 bu-has-text-grey">
+                                    {{jobDetails.config.scheduleLabel}}
+                                </p>
+                            </div>
+
+                            <!-- Configuration Overrides -->
+                            <div class="bu-mb-4" v-if="hasOverrides">
+                                <h5 class="bu-is-size-5 bu-mb-2">Active Overrides</h5>
+                                <p class="bu-mb-2" v-if="jobDetails.config.scheduleOverride">
+                                    <strong>Schedule Override:</strong>
+                                    <el-tag size="small" type="warning">{{jobDetails.config.scheduleOverride}}</el-tag>
+                                </p>
+                                <p class="bu-mb-2" v-if="jobDetails.config.retryOverride">
+                                    <strong>Retry Override:</strong>
+                                    <br>
+                                    Attempts: {{jobDetails.config.retryOverride.attempts}}
+                                    <br>
+                                    Delay: {{jobDetails.config.retryOverride.delay}}ms
+                                </p>
+                            </div>
+
+                            <!-- Current State -->
+                            <div class="bu-mb-4">
+                                <h5 class="bu-is-size-5 bu-mb-2">Current State</h5>
+                                <p class="bu-mb-2">
+                                    <strong>Last Run:</strong>
+                                    {{jobDetails.currentState.lastRun}}
+                                </p>
+                                <p class="bu-mb-2">
+                                    <strong>Next Run:</strong>
+                                    {{jobDetails.currentState.nextRun}}
+                                </p>
+                                <p class="bu-mb-2" v-if="jobDetails.currentState.failReason">
+                                    <strong>Last Failure:</strong>
+                                    <span class="bu-has-text-danger">{{jobDetails.currentState.failReason}}</span>
+                                </p>
+                            </div>
+                        </div>
+                    </div>
+                </div>
+            </cly-section>
+
+            <!-- Job History Section -->
+            <cly-section>
+                <h4 class="bu-is-size-4 bu-mb-4">Job History</h4>
+                <cly-datatable-n
+                    :rows="jobRuns"
+                    :columns="jobRunColumns"
+                    :force-loading="isLoading"
+                >
+                    <template v-slot="scope">
+                        <el-table-column prop="lastRunAt" :label="i18n('jobs.run-time')" sortable="custom">
+                            <template v-slot="scope">
+                                {{formatDateTime(scope.row.lastRunAt)}}
+                            </template>
+                        </el-table-column>
+                        
+                        <el-table-column prop="status" :label="i18n('jobs.status')" sortable="custom">
+                            <template v-slot="scope">
+                                <cly-status-tag
+                                    :text="scope.row.status"
+                                    :color="getRunStatusColor(scope.row)"
+                                >
+                                </cly-status-tag>
+                                <el-tooltip
+                                    v-if="scope.row.failReason"
+                                    :content="scope.row.failReason"
+                                    placement="top"
+                                >
+                                    <i class="ion-alert-circled bu-ml-2" style="color: #d23f00"></i>
+                                </el-tooltip>
+                            </template>
+                        </el-table-column>
+
+                        <el-table-column prop="duration" :label="i18n('jobs.duration')" sortable="custom">
+                            <template v-slot="scope">
+                                {{scope.row.duration}}s
+                            </template>
+                        </el-table-column>
+
+                        <el-table-column prop="result" :label="i18n('jobs.result')">
+                            <template v-slot="scope">
+                                <pre v-if="scope.row.dataAsString" class="bu-p-2 bu-has-background-grey-lighter bu-is-family-code">{{scope.row.dataAsString}}</pre>
+                                <span v-else>-</span>
+                            </template>
+                        </el-table-column>
+                    </template>
+                </cly-datatable-n>
+            </cly-section>
+        </template>
     </cly-main>
 </div>
\ No newline at end of file
diff --git a/frontend/express/public/core/jobs/templates/jobs.html b/frontend/express/public/core/jobs/templates/jobs.html
index 0d0585488fc..6dfeb7410c9 100644
--- a/frontend/express/public/core/jobs/templates/jobs.html
+++ b/frontend/express/public/core/jobs/templates/jobs.html
@@ -5,7 +5,13 @@
     </cly-header>
     <cly-main>
 		<cly-section data-test-id="table-jobs">
-			<cly-datatable-n test-id="datatable-jobs" :data-source="remoteTableDataSource" v-on:row-click="goTo" :isClickable="true" :default-sort="{prop: 'name', order: 'ascending'}">
+			<cly-datatable-n 
+                test-id="datatable-jobs" 
+                :data-source="remoteTableDataSource" 
+                v-on:row-click="goTo" 
+                :isClickable="true" 
+                :default-sort="{prop: 'name', order: 'ascending'}"
+            >
                 <template v-slot="scope">
                     <el-table-column sortable="custom" prop="name" :label="i18n('jobs.job-name')" type="clickable">
                         <template slot-scope="scope">
@@ -17,23 +23,43 @@
                     <el-table-column sortable="custom" prop="status" :label="i18n('jobs.job-status')">
                         <template slot-scope="scope">   
                             <span :data-test-id="'datatable-jobs-status-' + scope.$index">
-                                 <cly-status-tag :text="scope.row.status" :color="getColor(scope.row)"></cly-status-tag>
+                                <cly-status-tag 
+                                    :text="scope.row.config.enabled ? scope.row.status : 'DISABLED'" 
+                                    :color="getColor(scope.row)">
+                                </cly-status-tag>
+                                <el-tooltip v-if="scope.row.failReason" :content="scope.row.failReason" placement="top">
+                                    <i class="ion-alert-circled bu-ml-2" style="color: #D23F00;"></i>
+                                </el-tooltip>
                             </span>
                         </template>
                     </el-table-column>
-                    <el-table-column sortable="custom" prop="schedule" :label="i18n('jobs.job-schedule')">
+                    <el-table-column sortable="custom" prop="scheduleLabel" :label="i18n('jobs.job-schedule')">
                         <template slot-scope="scope">
-                            <p :data-test-id="'datatable-jobs-schedule-' + scope.$index">{{scope.row.scheduleLabel}}</p>
-                            <p :data-test-id="'datatable-jobs-schedule-detail-' + scope.$index" style="color:#A7AEB8; font-size: 12px;">{{scope.row.scheduleDetail}}</p>
+                            <p :data-test-id="'datatable-jobs-schedule-' + scope.$index">
+                                {{scope.row.scheduleLabel}}
+                                <el-tag 
+                                    v-if="scope.row.scheduleOverridden" 
+                                    size="mini" 
+                                    type="warning"
+                                >
+                                    Overridden
+                                </el-tag>
+                            </p>
+                            <p 
+                                :data-test-id="'datatable-jobs-schedule-detail-' + scope.$index" 
+                                style="color:#A7AEB8; font-size: 12px;"
+                            >
+                               {{scope.row.configuredSchedule || scope.row.schedule}}
+                            </p>
                         </template>
                     </el-table-column>
-                    <el-table-column sortable="custom" prop="next" :label="i18n('jobs.job-next-run')">
+                    <el-table-column sortable="custom" prop="nextRunAt" :label="i18n('jobs.job-next-run')">
                         <template slot-scope="scope">
                             <p :data-test-id="'datatable-jobs-next-run-date-' + scope.$index">{{scope.row.nextRunDate}}</p>
                             <p :data-test-id="'datatable-jobs-next-run-time-' + scope.$index" style="color:#A7AEB8; font-size: 12px;">{{scope.row.nextRunTime}}</p>
                         </template>
                     </el-table-column>
-                    <el-table-column sortable="custom" prop="finished" :label="i18n('jobs.job-last-run')">
+                    <el-table-column sortable="custom" prop="lastFinishedAt" :label="i18n('jobs.job-last-run')">
                         <template slot-scope="scope">
                             <p v-html="scope.row.lastRun" :data-test-id="'datatable-jobs-last-run-' + scope.$index"></p>
                         </template>
@@ -43,10 +69,58 @@
                             <p :data-test-id="'datatable-jobs-total-' + scope.$index">{{scope.row.total}}</p>
                         </template>
                     </el-table-column>
+                    <el-table-column sortable="custom" prop="lastRunStatus" :label="i18n('jobs.last-run-status')">
+                        <template slot-scope="scope">
+                            <span :data-test-id="'datatable-jobs-last-run-status-' + scope.$index">
+                                <cly-status-tag 
+                                    :text="scope.row.lastRunStatus" 
+                                    :color="scope.row.lastRunStatus === 'success' ? 'green' : (scope.row.lastRunStatus === 'failed' ? 'red' : 'gray')"
+                                >
+                                </cly-status-tag>
+                            </span>
+                        </template>
+                    </el-table-column>
                     <el-table-column align="center" type="options">
                         <template v-slot="scope">
-                            <cly-more-options v-if="scope.row.status !== 'RUNNING' && canSuspendJob && scope.row.hover" @command="handleCommand($event, scope.row)" placement="bottom-end" :test-id="'datatable-jobs-' + scope.$index">
-                                <el-dropdown-item command="change-job-status">{{scope.row.status === 'SUSPENDED' ? i18n('jobs.schedule') : i18n('jobs.suspend')}}</el-dropdown-item>
+                            <!-- Dialog can be placed outside if desired, but this works. -->
+                            <el-dialog
+                                :title="i18n('jobs.schedule-configuration')"
+                                :visible.sync="scheduleDialogVisible"
+                                width="30%">
+                                <div class="bu-columns">
+                                    <div class="bu-column">
+                                        <div class="config-item">
+                                            <label>Schedule:</label>
+                                            <el-input 
+                                                v-model="selectedJobConfig.schedule"
+                                                :placeholder="selectedJobConfig.defaultSchedule">
+                                            </el-input>
+                                            <div class="bu-mt-2 bu-is-size-7 bu-has-text-grey">
+                                                {{i18n('jobs.schedule-hint')}}
+                                            </div>
+                                            <div v-if="selectedJobConfig.scheduleLabel" class="bu-mt-2 bu-is-size-7">
+                                                {{selectedJobConfig.scheduleLabel}}
+                                            </div>
+                                        </div>
+                                    </div>
+                                </div>
+                                <span slot="footer" class="dialog-footer">
+                                    <el-button @click="scheduleDialogVisible = false">{{i18n('common.cancel')}}</el-button>
+                                    <el-button type="primary" @click="saveSchedule" :loading="saving">{{i18n('common.save')}}</el-button>
+                                </span>
+                            </el-dialog>
+                            
+                            <!-- Removed 'scope.row.hover' and changed .job to row itself -->
+                            <cly-more-options 
+                                v-if="canSuspendJob" 
+                                @command="handleCommand($event, scope.row)" 
+                                placement="bottom-end" 
+                                :test-id="'datatable-jobs-' + scope.$index"
+                            >
+                                <el-dropdown-item command="enable" v-if="!scope.row.config.enabled">{{i18n('jobs.enable')}}</el-dropdown-item>
+                                <el-dropdown-item command="disable" v-if="scope.row.config.enabled">{{i18n('jobs.disable')}}</el-dropdown-item>
+                                <el-dropdown-item command="schedule">{{i18n('jobs.change-schedule')}}</el-dropdown-item>
+                                <el-dropdown-item command="runNow">{{i18n('jobs.run-now')}}</el-dropdown-item>
                             </cly-more-options>
                         </template>
                     </el-table-column>
@@ -54,4 +128,4 @@
             </cly-datatable-n>
 		</cly-section>
     </cly-main>
-</div>
\ No newline at end of file
+</div>

From 0edd2a77ddfd5ff9e6f1a5223e4dc15a33a7efa0 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Fri, 24 Jan 2025 00:20:42 +0530
Subject: [PATCH 46/90] refactor job view

---
 .../core/jobs/javascripts/countly.views.js    | 168 +++--
 .../core/jobs/templates/jobs-details.html     | 137 +++-
 .../public/core/jobs/templates/jobs.html      | 239 ++++--
 jobServer/api.js                              | 693 +++++++-----------
 4 files changed, 655 insertions(+), 582 deletions(-)

diff --git a/frontend/express/public/core/jobs/javascripts/countly.views.js b/frontend/express/public/core/jobs/javascripts/countly.views.js
index 630ac7540d9..cb1cdfe577d 100644
--- a/frontend/express/public/core/jobs/javascripts/countly.views.js
+++ b/frontend/express/public/core/jobs/javascripts/countly.views.js
@@ -1,7 +1,11 @@
 /*global countlyAuth, countlyCommon, app, countlyVue, CV, countlyGlobal, CountlyHelpers, $, moment */
 
 (function() {
+    /**
+     * Helper function to map the job status to a color tag
+     */
     var getColor = function(row) {
+        // row is the merged job object
         if (row.status === "RUNNING") {
             return "green";
         }
@@ -14,61 +18,84 @@
         else if (row.status === "CANCELLED" || row.lastRunStatus === "failed") {
             return "red";
         }
+        return "gray";
     };
+
+    /**
+     * Helper to update row display fields (like nextRunDate, nextRunTime, lastRun)
+     */
     var updateScheduleRow = function(row) {
-        // Format dates using moment
         row.nextRunDate = row.nextRunAt ? moment(row.nextRunAt).format('YYYY-MM-DD') : '';
         row.nextRunTime = row.nextRunAt ? moment(row.nextRunAt).format('HH:mm:ss') : '';
         row.lastRun = row.lastFinishedAt ? moment(row.lastFinishedAt).fromNow() : '';
 
-        // Handle schedule display
+        // If the row has .config.defaultConfig.schedule.value, use it as its "default schedule"
         if (row.config && row.config.defaultConfig && row.config.defaultConfig.schedule) {
             row.schedule = row.config.defaultConfig.schedule.value;
             row.configuredSchedule = row.config.schedule;
-            row.scheduleOverridden = row.configuredSchedule &&
-                                    row.configuredSchedule !== row.schedule;
+            row.scheduleOverridden = row.configuredSchedule && (row.configuredSchedule !== row.schedule);
         }
-
     };
+
+    /**
+     * Main view for listing jobs
+     */
     var JobsView = countlyVue.views.create({
-        template: CV.T('/core/jobs/templates/jobs.html'),
+        template: CV.T('/core/jobs/templates/jobs.html'), // your HTML template path
         data: function() {
             var self = this;
-            var tableStore = countlyVue.vuex.getLocalStore(countlyVue.vuex.ServerDataTable("jobsTable", {
-                columns: ['name', "schedule", "next", "finished", "status", "total"],
-                onRequest: function() {
-                    self.loaded = false;
-                    return {
-                        type: "GET",
-                        url: countlyCommon.API_URL + "/o/jobs",
-                        data: {
-                            app_id: countlyCommon.ACTIVE_APP_ID,
-                            iDisplayStart: 0,
-                            iDisplayLength: 50
-                        }
-                    };
-                },
-                onReady: function(context, rows) {
-                    self.loaded = true;
-                    var processedRows = [];
-                    for (var i = 0; i < rows.aaData.length; i++) {
-                        var row = rows.aaData[i].job;
-                        var config = rows.aaData[i].config;
 
-                        // Process job status - now directly from API
-                        row.enabled = config.enabled;
-                        row.config = config;
+            // Create a local vuex store for the server data table
+            var tableStore = countlyVue.vuex.getLocalStore(
+                countlyVue.vuex.ServerDataTable("jobsTable", {
+                    // columns: ['name', "schedule", "next", "finished", "status", "total"],
+                    columns: [
+                        "name",
+                        "status",
+                        "scheduleLabel",
+                        "nextRunAt",
+                        "lastFinishedAt",
+                        "lastRunStatus",
+                        "total"
+                    ],
+
+                    onRequest: function() {
+                        // Called before making the request
+                        self.loaded = false;
+                        return {
+                            type: "GET",
+                            url: countlyCommon.API_URL + "/o/jobs", // no ?name= param => list mode
+                            data: {
+                                app_id: countlyCommon.ACTIVE_APP_ID,
+                                iDisplayStart: 0,
+                                iDisplayLength: 50
+                            }
+                        };
+                    },
+                    onReady: function(context, rows) {
+                        // Called when request completes successfully
+                        self.loaded = true;
+
+                        // rows.aaData is an array: [ { job: {...}, config: {...} }, ... ]
+                        // We merge job + config into a single row object
+                        var processedRows = [];
+                        for (var i = 0; i < rows.length; i++) {
+                            var mergedJob = rows[i].job; // from "job"
+                            var config = rows[i].config; // from "config"
+
+                            mergedJob.enabled = config.enabled;
+                            mergedJob.config = config;
 
-                        // Schedule handling
-                        row.schedule = config.schedule;
-                        row.scheduleLabel = row.scheduleLabel || '';
+                            // Do any schedule display updates, etc.
+                            updateScheduleRow(mergedJob);
 
-                        updateScheduleRow(row);
-                        processedRows.push(row);
+                            processedRows.push(mergedJob);
+                        }
+                        return processedRows;
                     }
-                    return processedRows;
-                }
-            }));
+                })
+            );
+
             return {
                 loaded: true,
                 saving: false,
@@ -85,15 +112,19 @@
             };
         },
         computed: {
+            /**
+             * Whether the current user can enable/disable jobs
+             */
             canSuspendJob: function() {
                 return countlyGlobal.member.global_admin || countlyGlobal.admin_apps[countlyCommon.ACTIVE_APP_ID];
             },
         },
         methods: {
-            formatDateTime(date) {
+            formatDateTime: function(date) {
                 return date ? moment(date).format('D MMM, YYYY HH:mm:ss') : '-';
             },
-            getStatusColor(details) {
+            getStatusColor: function(details) {
+                // Not strictly used in the listing, but you can keep it for reference
                 if (!details.config.enabled) {
                     return 'gray';
                 }
@@ -109,6 +140,7 @@
                 return 'yellow';
             },
             getRunStatusColor(status) {
+                // For run status
                 if (status === 'success') {
                     return 'green';
                 }
@@ -123,14 +155,21 @@
                     this.tableStore.dispatch("fetchJobsTable");
                 }
             },
+            /**
+             * Navigates to job details page
+             */
             goTo: function(row) {
                 app.navigate("#/manage/jobs/" + row.name, true);
             },
             getColor: getColor,
+            /**
+             * Called from the row’s more options, e.g. "enable", "disable", "schedule", "runNow"
+             */
             handleCommand: function(command, row) {
                 if (row.name) {
                     var self = this;
                     if (command === 'schedule') {
+                        // Show the schedule dialog
                         this.selectedJobConfig = {
                             name: row.name,
                             schedule: row.configuredSchedule || row.schedule,
@@ -141,16 +180,15 @@
                         return;
                     }
 
+                    // For enable, disable, runNow, etc. => /i/jobs
                     var data = {
                         app_id: countlyCommon.ACTIVE_APP_ID,
                         jobName: row.name,
                         action: command
                     };
 
-                    // You can switch to type: "POST" if desired;
-                    // server code accepts query params so GET also works.
                     $.ajax({
-                        type: "GET",
+                        type: "GET", // or POST if your server expects that
                         url: countlyCommon.API_URL + "/i/jobs",
                         data: data,
                         success: function(res) {
@@ -177,6 +215,9 @@
                     });
                 }
             },
+            /**
+             * Called when user clicks "Save" on the schedule dialog
+             */
             saveSchedule: function() {
                 var self = this;
                 self.saving = true;
@@ -211,6 +252,9 @@
         }
     });
 
+    /**
+     * Detailed view for a single job
+     */
     var JobDetailsView = countlyVue.views.BaseView.extend({
         template: "#jobs-details-template",
         data: function() {
@@ -219,6 +263,7 @@
                 jobDetails: null,
                 jobRuns: [],
                 isLoading: false,
+                // columns for the run history table
                 jobRunColumns: [
                     { prop: "lastRunAt", label: CV.i18n('jobs.run-time'), sortable: true },
                     { prop: "status", label: CV.i18n('jobs.status'), sortable: true },
@@ -228,13 +273,19 @@
             };
         },
         computed: {
+            /**
+             * Check if there are any scheduleOverride or retryOverride in the config
+             */
             hasOverrides: function() {
                 return this.jobDetails &&
-                       (this.jobDetails.config.scheduleOverride ||
-                        this.jobDetails.config.retryOverride);
+                    (this.jobDetails.config?.scheduleOverride ||
+                        this.jobDetails.config?.retryOverride);
             }
         },
         methods: {
+            /**
+             * Fetches jobDetails + normal docs from /o/jobs?name=<jobName>
+             */
             fetchJobDetails: function() {
                 var self = this;
                 self.isLoading = true;
@@ -250,10 +301,10 @@
                     },
                     dataType: "json",
                     success: function(response) {
-                        // API now returns jobDetails directly
+                        // jobDetails => the main scheduled doc + overrides
                         self.jobDetails = response.jobDetails;
 
-                        // Process job runs from aaData
+                        // aaData => the array of normal run docs
                         self.jobRuns = (response.aaData || []).map(function(run) {
                             return {
                                 lastRunAt: run.lastRunAt,
@@ -281,13 +332,17 @@
                 return date ? moment(date).format('D MMM, YYYY HH:mm:ss') : '-';
             },
             calculateDuration: function(run) {
+                // (optional) if you want dynamic calculations
                 if (!run.lastRunAt || !run.lastFinishedAt) {
                     return '-';
                 }
                 return ((new Date(run.lastFinishedAt) - new Date(run.lastRunAt)) / 1000).toFixed(2);
             },
+            /**
+             * Map jobDetails.currentState.status to a color
+             */
             getStatusColor: function(jobDetails) {
-                if (!jobDetails.config.enabled) {
+                if (!jobDetails.config?.enabled) {
                     return "grey";
                 }
                 switch (jobDetails.currentState.status) {
@@ -297,6 +352,9 @@
                 default: return "yellow";
                 }
             },
+            /**
+             * Map each run’s status to a color
+             */
             getRunStatusColor: function(run) {
                 switch (run.status) {
                 case "success": return "green";
@@ -307,24 +365,34 @@
             }
         },
         mounted: function() {
+            // On load, fetch data
             this.fetchJobDetails();
         }
     });
 
+    /**
+     * Wrap the JobsView as a Countly Backbone view
+     */
     var getMainView = function() {
         return new countlyVue.views.BackboneWrapper({
             component: JobsView,
-            vuex: [] //empty array if none
+            vuex: [] // empty array if none
         });
     };
 
+    /**
+     * Wrap the JobDetailsView as a Countly Backbone view
+     */
     var getDetailedView = function() {
         return new countlyVue.views.BackboneWrapper({
             component: JobDetailsView,
-            vuex: [] //empty array if none
+            vuex: [] // empty array if none
         });
     };
 
+    /**
+     * Define routes for #/manage/jobs and #/manage/jobs/:jobName
+     */
     if (countlyAuth.validateGlobalAdmin()) {
         app.route("/manage/jobs", "manageJobs", function() {
             this.renderWhenReady(getMainView());
@@ -346,4 +414,4 @@
             this.renderWhenReady(jobDetailsView);
         });
     }
-})();
\ No newline at end of file
+})();
diff --git a/frontend/express/public/core/jobs/templates/jobs-details.html b/frontend/express/public/core/jobs/templates/jobs-details.html
index 637e9cbc94e..9fdacb62120 100644
--- a/frontend/express/public/core/jobs/templates/jobs-details.html
+++ b/frontend/express/public/core/jobs/templates/jobs-details.html
@@ -1,67 +1,102 @@
+<!-- jobs-details.html -->
 <div v-bind:class="[componentId]">
     <cly-header :title="job_name">
         <template v-slot:header-top>
             <cly-back-link :title="i18n('plugins.back')"></cly-back-link>
         </template>
     </cly-header>
+
     <cly-main>
+        <!-- Wait until jobDetails is loaded -->
         <template v-if="jobDetails">
             <!-- Job Overview Section -->
             <cly-section>
                 <div class="bu-columns">
                     <div class="bu-column">
-                        <h4 class="bu-is-size-4 bu-mb-4">Job Configuration</h4>
+                        <h4 class="bu-is-size-4 bu-mb-4">
+                            {{ i18n('jobs.job-configuration') }}
+                        </h4>
+
                         <div class="bu-box">
                             <!-- Base Configuration -->
                             <div class="bu-mb-4">
-                                <h5 class="bu-is-size-5 bu-mb-2">Base Configuration</h5>
+                                <h5 class="bu-is-size-5 bu-mb-2">
+                                    {{ i18n('jobs.base-configuration') }}
+                                </h5>
                                 <p class="bu-mb-2">
-                                    <strong>Status:</strong>
+                                    <strong>{{ i18n('jobs.status') }}:</strong>
                                     <cly-status-tag
-                                        :text="jobDetails.config.enabled ? jobDetails.currentState.status : 'DISABLED'"
+                                        :text="jobDetails.config?.enabled
+                                               ? jobDetails.currentState?.status
+                                               : 'DISABLED'"
                                         :color="getStatusColor(jobDetails)"
                                     >
                                     </cly-status-tag>
                                 </p>
                                 <p class="bu-mb-2">
-                                    <strong>Default Schedule:</strong>
-                                    {{jobDetails.config.defaultConfig.schedule.value}}
+                                    <strong>{{ i18n('jobs.default-schedule') }}:</strong>
+                                    {{ jobDetails.config?.defaultConfig?.schedule?.value }}
                                 </p>
                                 <p class="bu-mb-2 bu-has-text-grey">
-                                    {{jobDetails.config.scheduleLabel}}
+                                    {{ jobDetails.config?.scheduleLabel }}
                                 </p>
                             </div>
 
                             <!-- Configuration Overrides -->
-                            <div class="bu-mb-4" v-if="hasOverrides">
-                                <h5 class="bu-is-size-5 bu-mb-2">Active Overrides</h5>
-                                <p class="bu-mb-2" v-if="jobDetails.config.scheduleOverride">
-                                    <strong>Schedule Override:</strong>
-                                    <el-tag size="small" type="warning">{{jobDetails.config.scheduleOverride}}</el-tag>
+                            <div
+                                class="bu-mb-4"
+                                v-if="hasOverrides"
+                            >
+                                <h5 class="bu-is-size-5 bu-mb-2">
+                                    {{ i18n('jobs.active-overrides') }}
+                                </h5>
+                                <p
+                                    class="bu-mb-2"
+                                    v-if="jobDetails.config?.scheduleOverride"
+                                >
+                                    <strong>{{ i18n('jobs.schedule-override') }}:</strong>
+                                    <el-tag
+                                        size="small"
+                                        type="warning"
+                                    >
+                                        {{ jobDetails.config.scheduleOverride }}
+                                    </el-tag>
                                 </p>
-                                <p class="bu-mb-2" v-if="jobDetails.config.retryOverride">
-                                    <strong>Retry Override:</strong>
+                                <p
+                                    class="bu-mb-2"
+                                    v-if="jobDetails.config?.retryOverride"
+                                >
+                                    <strong>{{ i18n('jobs.retry-override') }}:</strong>
                                     <br>
-                                    Attempts: {{jobDetails.config.retryOverride.attempts}}
+                                    {{ i18n('jobs.attempts') }}:
+                                    {{ jobDetails.config.retryOverride.attempts }}
                                     <br>
-                                    Delay: {{jobDetails.config.retryOverride.delay}}ms
+                                    {{ i18n('jobs.delay') }}:
+                                    {{ jobDetails.config.retryOverride.delay }}ms
                                 </p>
                             </div>
 
                             <!-- Current State -->
                             <div class="bu-mb-4">
-                                <h5 class="bu-is-size-5 bu-mb-2">Current State</h5>
+                                <h5 class="bu-is-size-5 bu-mb-2">
+                                    {{ i18n('jobs.current-state') }}
+                                </h5>
                                 <p class="bu-mb-2">
-                                    <strong>Last Run:</strong>
-                                    {{jobDetails.currentState.lastRun}}
+                                    <strong>{{ i18n('jobs.last-run') }}:</strong>
+                                    {{ jobDetails.currentState?.lastRun }}
                                 </p>
                                 <p class="bu-mb-2">
-                                    <strong>Next Run:</strong>
-                                    {{jobDetails.currentState.nextRun}}
+                                    <strong>{{ i18n('jobs.next-run') }}:</strong>
+                                    {{ jobDetails.currentState?.nextRun }}
                                 </p>
-                                <p class="bu-mb-2" v-if="jobDetails.currentState.failReason">
-                                    <strong>Last Failure:</strong>
-                                    <span class="bu-has-text-danger">{{jobDetails.currentState.failReason}}</span>
+                                <p
+                                    class="bu-mb-2"
+                                    v-if="jobDetails.currentState?.failReason"
+                                >
+                                    <strong>{{ i18n('jobs.last-failure') }}:</strong>
+                                    <span class="bu-has-text-danger">
+                                        {{ jobDetails.currentState.failReason }}
+                                    </span>
                                 </p>
                             </div>
                         </div>
@@ -69,22 +104,35 @@ <h5 class="bu-is-size-5 bu-mb-2">Current State</h5>
                 </div>
             </cly-section>
 
-            <!-- Job History Section -->
+            <!-- Job History Section (table of type='normal' runs) -->
             <cly-section>
-                <h4 class="bu-is-size-4 bu-mb-4">Job History</h4>
+                <h4 class="bu-is-size-4 bu-mb-4">
+                    {{ i18n('jobs.job-history') }}
+                </h4>
+
                 <cly-datatable-n
                     :rows="jobRuns"
                     :columns="jobRunColumns"
                     :force-loading="isLoading"
                 >
                     <template v-slot="scope">
-                        <el-table-column prop="lastRunAt" :label="i18n('jobs.run-time')" sortable="custom">
+                        <!-- Column: lastRunAt -->
+                        <el-table-column
+                            prop="lastRunAt"
+                            :label="i18n('jobs.run-time')"
+                            sortable="custom"
+                        >
                             <template v-slot="scope">
-                                {{formatDateTime(scope.row.lastRunAt)}}
+                                {{ formatDateTime(scope.row.lastRunAt) }}
                             </template>
                         </el-table-column>
-                        
-                        <el-table-column prop="status" :label="i18n('jobs.status')" sortable="custom">
+
+                        <!-- Column: status -->
+                        <el-table-column
+                            prop="status"
+                            :label="i18n('jobs.status')"
+                            sortable="custom"
+                        >
                             <template v-slot="scope">
                                 <cly-status-tag
                                     :text="scope.row.status"
@@ -96,20 +144,37 @@ <h4 class="bu-is-size-4 bu-mb-4">Job History</h4>
                                     :content="scope.row.failReason"
                                     placement="top"
                                 >
-                                    <i class="ion-alert-circled bu-ml-2" style="color: #d23f00"></i>
+                                    <i
+                                        class="ion-alert-circled bu-ml-2"
+                                        style="color: #d23f00"
+                                    ></i>
                                 </el-tooltip>
                             </template>
                         </el-table-column>
 
-                        <el-table-column prop="duration" :label="i18n('jobs.duration')" sortable="custom">
+                        <!-- Column: duration -->
+                        <el-table-column
+                            prop="duration"
+                            :label="i18n('jobs.duration')"
+                            sortable="custom"
+                        >
                             <template v-slot="scope">
-                                {{scope.row.duration}}s
+                                {{ scope.row.duration }}s
                             </template>
                         </el-table-column>
 
-                        <el-table-column prop="result" :label="i18n('jobs.result')">
+                        <!-- Column: result -->
+                        <el-table-column
+                            prop="result"
+                            :label="i18n('jobs.result')"
+                        >
                             <template v-slot="scope">
-                                <pre v-if="scope.row.dataAsString" class="bu-p-2 bu-has-background-grey-lighter bu-is-family-code">{{scope.row.dataAsString}}</pre>
+                                <pre
+                                    v-if="scope.row.dataAsString"
+                                    class="bu-p-2 bu-has-background-grey-lighter bu-is-family-code"
+                                >
+{{ scope.row.dataAsString }}
+                                </pre>
                                 <span v-else>-</span>
                             </template>
                         </el-table-column>
@@ -118,4 +183,4 @@ <h4 class="bu-is-size-4 bu-mb-4">Job History</h4>
             </cly-section>
         </template>
     </cly-main>
-</div>
\ No newline at end of file
+</div>
diff --git a/frontend/express/public/core/jobs/templates/jobs.html b/frontend/express/public/core/jobs/templates/jobs.html
index 6dfeb7410c9..b9d46df961e 100644
--- a/frontend/express/public/core/jobs/templates/jobs.html
+++ b/frontend/express/public/core/jobs/templates/jobs.html
@@ -1,131 +1,236 @@
+<!-- jobs.html -->
 <div v-bind:class="[componentId]">
-    <cly-header
-        :title="i18n('sidebar.management.jobs')"
-    >
+    <cly-header :title="i18n('sidebar.management.jobs')">
     </cly-header>
+
     <cly-main>
-		<cly-section data-test-id="table-jobs">
-			<cly-datatable-n 
-                test-id="datatable-jobs" 
-                :data-source="remoteTableDataSource" 
-                v-on:row-click="goTo" 
-                :isClickable="true" 
-                :default-sort="{prop: 'name', order: 'ascending'}"
+        <cly-section data-test-id="table-jobs">
+            <!--
+                cly-datatable-n uses :data-source="remoteTableDataSource"
+                which is tied to our ServerDataTable in views.js.
+                We define columns using el-table-column inside <template v-slot="scope">.
+            -->
+            <cly-datatable-n
+                test-id="datatable-jobs"
+                :data-source="remoteTableDataSource"
+                v-on:row-click="goTo"
+                :isClickable="true"
+                :default-sort="{ prop: 'name', order: 'ascending' }"
             >
                 <template v-slot="scope">
-                    <el-table-column sortable="custom" prop="name" :label="i18n('jobs.job-name')" type="clickable">
-                        <template slot-scope="scope">
+                    <!-- Job Name Column -->
+                    <el-table-column
+                        prop="name"
+                        :label="i18n('jobs.job-name')"
+                        sortable="custom"
+                        type="clickable"
+                    >
+                        <template v-slot="scope">
                             <span :data-test-id="'datatable-jobs-name-' + scope.$index">
                                 {{ scope.row.name }}
                             </span>
                         </template>
                     </el-table-column>
-                    <el-table-column sortable="custom" prop="status" :label="i18n('jobs.job-status')">
-                        <template slot-scope="scope">   
+
+                    <!-- Job Status Column -->
+                    <el-table-column
+                        prop="status"
+                        :label="i18n('jobs.job-status')"
+                        sortable="custom"
+                    >
+                        <template v-slot="scope">
                             <span :data-test-id="'datatable-jobs-status-' + scope.$index">
-                                <cly-status-tag 
-                                    :text="scope.row.config.enabled ? scope.row.status : 'DISABLED'" 
-                                    :color="getColor(scope.row)">
-                                </cly-status-tag>
-                                <el-tooltip v-if="scope.row.failReason" :content="scope.row.failReason" placement="top">
+                                <cly-status-tag
+                                    :text="scope.row.config.enabled ? scope.row.status : 'DISABLED'"
+                                    :color="getColor(scope.row)"
+                                ></cly-status-tag>
+
+                                <!-- Show error tooltip if there's a failReason -->
+                                <el-tooltip
+                                    v-if="scope.row.failReason"
+                                    :content="scope.row.failReason"
+                                    placement="top"
+                                >
                                     <i class="ion-alert-circled bu-ml-2" style="color: #D23F00;"></i>
                                 </el-tooltip>
                             </span>
                         </template>
                     </el-table-column>
-                    <el-table-column sortable="custom" prop="scheduleLabel" :label="i18n('jobs.job-schedule')">
-                        <template slot-scope="scope">
+
+                    <!-- Job Schedule (Cron) Column -->
+                    <el-table-column
+                        prop="scheduleLabel"
+                        :label="i18n('jobs.job-schedule')"
+                        sortable="custom"
+                    >
+                        <template v-slot="scope">
                             <p :data-test-id="'datatable-jobs-schedule-' + scope.$index">
-                                {{scope.row.scheduleLabel}}
-                                <el-tag 
-                                    v-if="scope.row.scheduleOverridden" 
-                                    size="mini" 
+                                {{ scope.row.scheduleLabel }}
+                                <el-tag
+                                    v-if="scope.row.scheduleOverridden"
+                                    size="mini"
                                     type="warning"
                                 >
                                     Overridden
                                 </el-tag>
                             </p>
-                            <p 
-                                :data-test-id="'datatable-jobs-schedule-detail-' + scope.$index" 
-                                style="color:#A7AEB8; font-size: 12px;"
+                            <p
+                                :data-test-id="'datatable-jobs-schedule-detail-' + scope.$index"
+                                class="bu-has-text-grey bu-is-size-7"
                             >
-                               {{scope.row.configuredSchedule || scope.row.schedule}}
+                                {{ scope.row.configuredSchedule || scope.row.schedule }}
                             </p>
                         </template>
                     </el-table-column>
-                    <el-table-column sortable="custom" prop="nextRunAt" :label="i18n('jobs.job-next-run')">
-                        <template slot-scope="scope">
-                            <p :data-test-id="'datatable-jobs-next-run-date-' + scope.$index">{{scope.row.nextRunDate}}</p>
-                            <p :data-test-id="'datatable-jobs-next-run-time-' + scope.$index" style="color:#A7AEB8; font-size: 12px;">{{scope.row.nextRunTime}}</p>
+
+                    <!-- Next Run Column -->
+                    <el-table-column
+                        prop="nextRunAt"
+                        :label="i18n('jobs.job-next-run')"
+                        sortable="custom"
+                    >
+                        <template v-slot="scope">
+                            <p :data-test-id="'datatable-jobs-next-run-date-' + scope.$index">
+                                {{ scope.row.nextRunDate }}
+                            </p>
+                            <p
+                                :data-test-id="'datatable-jobs-next-run-time-' + scope.$index"
+                                class="bu-has-text-grey bu-is-size-7"
+                            >
+                                {{ scope.row.nextRunTime }}
+                            </p>
                         </template>
                     </el-table-column>
-                    <el-table-column sortable="custom" prop="lastFinishedAt" :label="i18n('jobs.job-last-run')">
-                        <template slot-scope="scope">
-                            <p v-html="scope.row.lastRun" :data-test-id="'datatable-jobs-last-run-' + scope.$index"></p>
+
+                    <!-- Last Run Column -->
+                    <el-table-column
+                        prop="lastFinishedAt"
+                        :label="i18n('jobs.job-last-run')"
+                        sortable="custom"
+                    >
+                        <template v-slot="scope">
+                            <p
+                                v-html="scope.row.lastRun"
+                                :data-test-id="'datatable-jobs-last-run-' + scope.$index"
+                            ></p>
                         </template>
                     </el-table-column>
-                    <el-table-column sortable="custom" prop="total" :label="i18n('jobs.job-total-scheduled')">
-                        <template slot-scope="scope">
-                            <p :data-test-id="'datatable-jobs-total-' + scope.$index">{{scope.row.total}}</p>
+
+                    <!-- Total Column (if needed) -->
+                    <el-table-column
+                        prop="total"
+                        :label="i18n('jobs.job-total-scheduled')"
+                        sortable="custom"
+                    >
+                        <template v-slot="scope">
+                            <p :data-test-id="'datatable-jobs-total-' + scope.$index">
+                                {{ scope.row.total }}
+                            </p>
                         </template>
                     </el-table-column>
-                    <el-table-column sortable="custom" prop="lastRunStatus" :label="i18n('jobs.last-run-status')">
-                        <template slot-scope="scope">
+
+                    <!-- Last Run Status Column -->
+                    <el-table-column
+                        prop="lastRunStatus"
+                        :label="i18n('jobs.last-run-status')"
+                        sortable="custom"
+                    >
+                        <template v-slot="scope">
                             <span :data-test-id="'datatable-jobs-last-run-status-' + scope.$index">
-                                <cly-status-tag 
-                                    :text="scope.row.lastRunStatus" 
-                                    :color="scope.row.lastRunStatus === 'success' ? 'green' : (scope.row.lastRunStatus === 'failed' ? 'red' : 'gray')"
+                                <cly-status-tag
+                                    :text="scope.row.lastRunStatus"
+                                    :color="scope.row.lastRunStatus === 'success'
+                                        ? 'green'
+                                        : (scope.row.lastRunStatus === 'failed' ? 'red' : 'gray')"
                                 >
                                 </cly-status-tag>
                             </span>
                         </template>
                     </el-table-column>
-                    <el-table-column align="center" type="options">
+
+                    <!-- Action Buttons Column -->
+                    <el-table-column
+                        align="center"
+                        type="options"
+                    >
                         <template v-slot="scope">
-                            <!-- Dialog can be placed outside if desired, but this works. -->
+                            <!--
+                                This is your schedule dialog.
+                                It's placed here so you can reuse the same dialog instance
+                                for each row, opening it via 'handleCommand'.
+                            -->
                             <el-dialog
                                 :title="i18n('jobs.schedule-configuration')"
                                 :visible.sync="scheduleDialogVisible"
-                                width="30%">
+                                width="30%"
+                            >
                                 <div class="bu-columns">
                                     <div class="bu-column">
                                         <div class="config-item">
-                                            <label>Schedule:</label>
-                                            <el-input 
+                                            <label>{{ i18n('jobs.schedule') }}:</label>
+                                            <el-input
                                                 v-model="selectedJobConfig.schedule"
-                                                :placeholder="selectedJobConfig.defaultSchedule">
-                                            </el-input>
+                                                :placeholder="selectedJobConfig.defaultSchedule"
+                                            ></el-input>
+
                                             <div class="bu-mt-2 bu-is-size-7 bu-has-text-grey">
-                                                {{i18n('jobs.schedule-hint')}}
+                                                {{ i18n('jobs.schedule-hint') }}
                                             </div>
-                                            <div v-if="selectedJobConfig.scheduleLabel" class="bu-mt-2 bu-is-size-7">
-                                                {{selectedJobConfig.scheduleLabel}}
+
+                                            <div
+                                                v-if="selectedJobConfig.scheduleLabel"
+                                                class="bu-mt-2 bu-is-size-7"
+                                            >
+                                                {{ selectedJobConfig.scheduleLabel }}
                                             </div>
                                         </div>
                                     </div>
                                 </div>
+
                                 <span slot="footer" class="dialog-footer">
-                                    <el-button @click="scheduleDialogVisible = false">{{i18n('common.cancel')}}</el-button>
-                                    <el-button type="primary" @click="saveSchedule" :loading="saving">{{i18n('common.save')}}</el-button>
+                                    <el-button @click="scheduleDialogVisible = false">
+                                        {{ i18n('common.cancel') }}
+                                    </el-button>
+                                    <el-button
+                                        type="primary"
+                                        @click="saveSchedule"
+                                        :loading="saving"
+                                    >
+                                        {{ i18n('common.save') }}
+                                    </el-button>
                                 </span>
                             </el-dialog>
-                            
-                            <!-- Removed 'scope.row.hover' and changed .job to row itself -->
-                            <cly-more-options 
-                                v-if="canSuspendJob" 
-                                @command="handleCommand($event, scope.row)" 
-                                placement="bottom-end" 
+
+                            <!-- The 'More Options' dropdown -->
+                            <cly-more-options
+                                v-if="canSuspendJob"
+                                @command="handleCommand($event, scope.row)"
+                                placement="bottom-end"
                                 :test-id="'datatable-jobs-' + scope.$index"
                             >
-                                <el-dropdown-item command="enable" v-if="!scope.row.config.enabled">{{i18n('jobs.enable')}}</el-dropdown-item>
-                                <el-dropdown-item command="disable" v-if="scope.row.config.enabled">{{i18n('jobs.disable')}}</el-dropdown-item>
-                                <el-dropdown-item command="schedule">{{i18n('jobs.change-schedule')}}</el-dropdown-item>
-                                <el-dropdown-item command="runNow">{{i18n('jobs.run-now')}}</el-dropdown-item>
+                                <el-dropdown-item
+                                    command="enable"
+                                    v-if="!scope.row.config.enabled"
+                                >
+                                    {{ i18n('jobs.enable') }}
+                                </el-dropdown-item>
+                                <el-dropdown-item
+                                    command="disable"
+                                    v-if="scope.row.config.enabled"
+                                >
+                                    {{ i18n('jobs.disable') }}
+                                </el-dropdown-item>
+                                <el-dropdown-item command="schedule">
+                                    {{ i18n('jobs.change-schedule') }}
+                                </el-dropdown-item>
+                                <el-dropdown-item command="runNow">
+                                    {{ i18n('jobs.run-now') }}
+                                </el-dropdown-item>
                             </cly-more-options>
                         </template>
                     </el-table-column>
                 </template>
             </cly-datatable-n>
-		</cly-section>
+        </cly-section>
     </cly-main>
 </div>
diff --git a/jobServer/api.js b/jobServer/api.js
index 0908c517f17..6b97ae53e26 100644
--- a/jobServer/api.js
+++ b/jobServer/api.js
@@ -1,3 +1,5 @@
+/* api.js */
+
 const Logger = require('../api/utils/log.js');
 const log = new Logger('jobs:api');
 
@@ -7,15 +9,16 @@ const common = require("../api/utils/common");
 const cronstrue = require('cronstrue');
 const moment = require('moment');
 
+// ----------------------------------
+// Helper Functions
+// ----------------------------------
+
 /**
- * Maps job status based on job properties
- * @param {Object} job - The job object
- * @param {Date} [job.lockedAt] - When the job was locked
- * @param {Date} [job.failedAt] - When the job failed
- * @param {Date} [job.lastFinishedAt] - When the job last finished
- * @returns {string} Status - "RUNNING", "FAILED", "COMPLETED", or "SCHEDULED"
+ * Determine job status from the doc's timestamps
+ * @param {Object} job Document from pulseJobs
+ * @returns {String} "RUNNING", "FAILED", "COMPLETED", or "SCHEDULED"
  */
-const getJobStatus = (job) => {
+function getJobStatus(job) {
     if (job.lockedAt) {
         return "RUNNING";
     }
@@ -26,16 +29,14 @@ const getJobStatus = (job) => {
         return "COMPLETED";
     }
     return "SCHEDULED";
-};
+}
 
 /**
- * Maps job run status based on job properties
- * @param {Object} job - The job object
- * @param {Date} [job.failedAt] - When the job failed
- * @param {Date} [job.lastFinishedAt] - When the job last finished
- * @returns {string} Status - "failed", "success", or "pending"
+ * Determine run status for a single job doc
+ * @param {Object} job
+ * @returns {String} "failed", "success", or "pending"
  */
-const getRunStatus = (job) => {
+function getRunStatus(job) {
     if (job.failedAt) {
         return "failed";
     }
@@ -43,18 +44,28 @@ const getRunStatus = (job) => {
         return "success";
     }
     return "pending";
-};
+}
 
-// Format job duration helper
-const formatJobDuration = (startDate, endDate) => {
+/**
+ * Returns job duration in seconds (string), or null if start/end not present
+ * @param {Date} startDate
+ * @param {Date} endDate
+ * @returns {String|null} e.g. "2.34"
+ */
+function formatJobDuration(startDate, endDate) {
     if (!startDate || !endDate) {
         return null;
     }
-    return ((new Date(endDate) - new Date(startDate)) / 1000).toFixed(2);
-};
+    const ms = new Date(endDate) - new Date(startDate);
+    return (ms / 1000).toFixed(2);
+}
 
-// Get schedule label helper
-const getScheduleLabel = (schedule) => {
+/**
+ * Returns a human-friendly label for a cron expression using cronstrue, or the raw expression if invalid
+ * @param {String} schedule
+ * @returns {String}
+ */
+function getScheduleLabel(schedule) {
     if (!schedule) {
         return '';
     }
@@ -64,67 +75,135 @@ const getScheduleLabel = (schedule) => {
     catch (e) {
         return schedule;
     }
-};
+}
+
+/**
+ * Build the "jobDetails" object for a scheduled doc
+ * @param {Object} scheduledDoc doc from pulseJobs, type='single'
+ * @param {Object} jobConfig doc from jobConfigs
+ * @returns {Object}
+ */
+function buildJobDetails(scheduledDoc, jobConfig) {
+    if (!scheduledDoc && !jobConfig) {
+        return {};
+    }
+
+    const enabled = jobConfig?.enabled ?? true;
+
+    // Combine default config from the scheduled doc + overrides from jobConfigs
+    const details = {
+        config: {
+            ...(jobConfig || {}),
+
+            // We ensure we have defaultConfig if jobConfig is missing it
+            defaultConfig: {
+                schedule: { value: scheduledDoc?.repeatInterval },
+                retry: {
+                    attempts: scheduledDoc?.attempts,
+                    delay: scheduledDoc?.backoff
+                }
+            },
+
+            enabled,
+            // The "override" fields
+            scheduleOverride: jobConfig?.schedule || null,
+            retryOverride: jobConfig?.retry || null,
+            // The human-friendly label from the scheduled doc's cron
+            scheduleLabel: getScheduleLabel(
+                jobConfig?.defaultConfig?.schedule?.value ||
+                scheduledDoc?.repeatInterval
+            )
+        },
+        currentState: {}
+    };
+
+    // If we have a scheduledDoc, fill out currentState
+    if (scheduledDoc) {
+        details.currentState = {
+            status: getJobStatus(scheduledDoc),
+            nextRun: scheduledDoc.nextRunAt,
+            lastRun: scheduledDoc.lastFinishedAt,
+            lastRunStatus: getRunStatus(scheduledDoc),
+            failReason: scheduledDoc.failReason,
+            lastRunDuration: formatJobDuration(scheduledDoc.lastRunAt, scheduledDoc.lastFinishedAt),
+            // Additional fields if needed
+            finishedCount: scheduledDoc.finishedCount,
+            totalRuns: scheduledDoc.runCount
+        };
+    }
+
+    return details;
+}
 
 /**
- * Job management API endpoint for controlling job configurations
- * @name /i/jobs
- * @example
- * // Enable a job
- * POST /i/jobs
- * {
- *   "jobName": "cleanupJob",
- *   "action": "enable"
- * }
- * Response: { "code": 200, "message": "Success" }
- * 
- * // Disable a job
- * POST /i/jobs
- * {
- *   "jobName": "cleanupJob",
- *   "action": "disable"
- * }
- * Response: { "code": 200, "message": "Success" }
- * 
- * // Trigger immediate job execution
- * POST /i/jobs
- * {
- *   "jobName": "cleanupJob",
- *   "action": "runNow"
- * }
- * Response: { "code": 200, "message": "Success" }
- * 
- * // Update schedule
- * POST /i/jobs
- * {
- *   "jobName": "cleanupJob",
- *   "action": "updateSchedule",
- *   "schedule": "0 0 * * *"  // Runs at midnight every day
- * }
- * Response: { "code": 200, "message": "Success" }
- * 
- * // Update retry configuration
- * POST /i/jobs
- * {
- *   "jobName": "cleanupJob",
- *   "action": "updateRetry",
- *   "retry": {
- *     "attempts": 3,    // Number of retry attempts
- *     "delay": 300      // Delay between retries in seconds
- *   }
- * }
- * Response: { "code": 200, "message": "Success" }
- * 
- * @returns {Object} Response
- * @returns {number} Response.code - Status codes:
- *                                  200: Success
- *                                  400: Invalid parameters or missing required fields
- *                                  500: Internal server error
- * @returns {string} Response.message - Status message describing the result
+ * Process a "normal" (type='normal') doc for job details table
+ * @param {Object} doc
+ * @returns {Object}
  */
+function processRunDoc(doc) {
+    return {
+        lastRunAt: doc.lastRunAt,
+        status: getJobStatus(doc),
+        duration: formatJobDuration(doc.lastRunAt, doc.lastFinishedAt),
+        result: doc.result,
+        failReason: doc.failReason,
+        dataAsString: JSON.stringify(doc.result || {}, null, 2)
+    };
+}
+
+/**
+ * Build a job row for the listing page
+ * @param {Object} mainDoc doc from pipeline ($first)
+ * @param {Object} jobConfig from jobConfigs
+ * @param {Object} groupItem aggregator object (with lastRunAt, lastFinishedAt, etc.)
+ * @returns {Object} { job: {...}, config: {...} }
+ */
+function buildListViewJob(mainDoc, jobConfig, groupItem) {
+    // If there's no doc, return something minimal
+    if (!mainDoc) {
+        return { job: {}, config: {} };
+    }
+
+    // Merge in config overrides
+    const finalConfig = {
+        enabled: jobConfig?.enabled ?? true,
+
+        // The default config is taken from the doc
+        defaultConfig: {
+            schedule: { value: mainDoc.repeatInterval },
+            retry: { attempts: mainDoc.attempts, delay: mainDoc.backoff }
+        },
+
+        // The user overrides
+        schedule: jobConfig?.schedule,
+        retry: jobConfig?.retry
+    };
+
+    const jobObj = {
+        name: mainDoc.name,
+        status: getJobStatus(mainDoc),
+        schedule: finalConfig.defaultConfig.schedule.value,
+        scheduleLabel: getScheduleLabel(finalConfig.defaultConfig.schedule.value),
+        nextRunAt: mainDoc.nextRunAt,
+        lastFinishedAt: groupItem.lastFinishedAt,
+        lastRunStatus: getRunStatus(mainDoc),
+        failReason: mainDoc.failReason,
+        // "total" can be included if you want. Here it's arbitrary:
+        total: 0
+    };
+
+    return {
+        job: jobObj,
+        config: finalConfig
+    };
+}
+
+// ----------------------------------
+// /i/jobs endpoint
+// ----------------------------------
 plugins.register('/i/jobs', async function(ob) {
     validateGlobalAdmin(ob.params, async function() {
-        const {jobName, schedule, retry} = ob.params.qstring || {};
+        const { jobName, schedule, retry } = ob.params.qstring || {};
         const action = ob.params.qstring?.action;
 
         if (!jobName) {
@@ -165,110 +244,25 @@ plugins.register('/i/jobs', async function(ob) {
                 return;
             }
 
+            // Update or insert if missing
             await jobsCollection.updateOne(
                 { jobName },
-                { $set: updateData }
+                { $set: updateData },
+                { upsert: true }
             );
             common.returnMessage(ob.params, 200, 'Success');
         }
         catch (error) {
-            log.e('Error in jobs API:', { error: error.message, stack: error.stack });
+            log.e('Error in /i/jobs:', { error: error.message, stack: error.stack });
             common.returnMessage(ob.params, 500, 'Internal server error');
         }
     });
     return true;
 });
 
-/**
- * Job query API endpoint for retrieving job information
- * @name /o/jobs
- * @example
- * // Get all jobs with pagination and search
- * GET /o/jobs
- * {
- *   "iDisplayStart": 0,        // Pagination start index
- *   "iDisplayLength": 50,      // Number of records per page
- *   "sSearch": "backup",       // Search term for job names
- *   "iSortCol_0": "nextRunAt", // Column to sort by
- *   "sSortDir_0": "desc"       // Sort direction: "asc" or "desc"
- * }
- * Response: {
- *   "sEcho": "1",
- *   "iTotalRecords": 100,
- *   "iTotalDisplayRecords": 5,
- *   "aaData": [{
- *     "job": {
- *       "name": "backupJob",
- *       "status": "SCHEDULED",
- *       "schedule": "0 0 * * *",
- *       "scheduleLabel": "At 12:00 AM",
- *       "nextRunAt": "2024-03-20T00:00:00.000Z",
- *       "lastFinishedAt": "2024-03-19T00:00:00.000Z",
- *       "lastRunStatus": "success",
- *       "failReason": null
- *     },
- *     "config": {
- *       "enabled": true,
- *       "schedule": "0 0 * * *",
- *       "retry": { "attempts": 3, "delay": 300 }
- *     }
- *   }]
- * }
- * 
- * // Get specific job details with run history
- * GET /o/jobs
- * {
- *   "name": "cleanupJob",
- *   "iDisplayStart": 0,
- *   "iDisplayLength": 50
- * }
- * Response: {
- *   "sEcho": "1",
- *   "iTotalRecords": 10,
- *   "iTotalDisplayRecords": 10,
- *   "aaData": [{
- *     "lastRunAt": "2024-03-19T00:00:00.000Z",
- *     "status": "COMPLETED",
- *     "lastRunStatus": "success",
- *     "duration": "45.32",
- *     "failReason": null,
- *     "result": { "processedRecords": 1000 },
- *     "runCount": 1,
- *     "dataAsString": "{\n  \"processedRecords\": 1000\n}"
- *   }],
- *   "jobDetails": {
- *     "config": {
- *       "enabled": true,
- *       "defaultConfig": {
- *         "schedule": { "value": "0 0 * * *" },
- *         "retry": { "attempts": 3, "delay": 300 }
- *       },
- *       "scheduleLabel": "At 12:00 AM",
- *       "scheduleOverride": null,
- *       "retryOverride": null
- *     },
- *     "currentState": {
- *       "status": "SCHEDULED",
- *       "nextRun": "2024-03-20T00:00:00.000Z",
- *       "lastRun": "2024-03-19T00:00:00.000Z",
- *       "lastRunStatus": "success",
- *       "failReason": null,
- *       "lastRunDuration": "45.32",
- *       "finishedCount": 10,
- *       "totalRuns": 10
- *     }
- *   }
- * }
- * 
- * @returns {Object} Response
- * @returns {string} Response.sEcho - Echo parameter from request for DataTables
- * @returns {number} Response.iTotalRecords - Total number of records before filtering
- * @returns {number} Response.iTotalDisplayRecords - Number of records after filtering
- * @returns {Array<Object>} Response.aaData - Array of job data or job runs
- * @returns {Object} [Response.jobDetails] - Detailed job information when querying specific job
- * @returns {Object} Response.jobDetails.config - Job configuration including schedule and retry settings
- * @returns {Object} Response.jobDetails.currentState - Current job state including next run and statistics
- */
+// ----------------------------------
+// /o/jobs endpoint
+// ----------------------------------
 plugins.register('/o/jobs', async function(ob) {
     validateGlobalAdmin(ob.params, async function() {
         const db = common.db;
@@ -285,72 +279,40 @@ plugins.register('/o/jobs', async function(ob) {
                 sSortDir_0
             } = ob.params.qstring;
 
-            // If jobName is provided, fetch detailed view
-            if (jobName) {
-                // Get job definition and config
-                const [jobConfig, latestJob] = await Promise.all([
-                    jobConfigsCollection.findOne({ jobName }),
-                    jobsCollection.findOne(
-                        { name: jobName, type: { $ne: 'single' } }, // Exclude single runs
-                        { sort: { lastRunAt: -1 } }
-                    )
-                ]);
-
-                // Get individual job runs
-                const jobRuns = await jobsCollection
-                    .find({
-                        name: jobName,
-                        type: 'single' // Only get single run instances
-                    })
-                    .sort({ lastRunAt: -1 })
-                    .skip(Number(iDisplayStart) || 0)
-                    .limit(Number(iDisplayLength) || 50)
-                    .toArray();
+            // Basic pagination & sorting
+            const skip = Number(iDisplayStart) || 0;
+            const limit = Number(iDisplayLength) || 50;
+            const sortColumn = iSortCol_0 || 'nextRunAt';
+            const sortDir = (sSortDir_0 === 'desc') ? -1 : 1;
 
-                const total = await jobsCollection.countDocuments({
+            // If a specific job name is provided -> detail view
+            if (jobName) {
+                // The "scheduled" doc (type: 'single')
+                const scheduledDoc = await jobsCollection.findOne({
                     name: jobName,
                     type: 'single'
                 });
 
-                // Process job runs with more detailed information
-                const processedRuns = jobRuns.map(run => ({
-                    lastRunAt: run.lastRunAt,
-                    status: getJobStatus(run),
-                    lastRunStatus: getRunStatus(run),
-                    duration: formatJobDuration(run.lastRunAt, run.lastFinishedAt),
-                    failReason: run.failReason,
-                    result: run.result,
-                    runCount: run.runCount,
-                    dataAsString: JSON.stringify(run.result || {}, null, 2)
-                }));
+                // The "normal" docs (type: 'normal')
+                const query = { name: jobName, type: 'normal' };
+                const normalDocsCursor = jobsCollection.find(query)
+                    .sort({ lastRunAt: -1 })
+                    .skip(skip)
+                    .limit(limit);
 
-                // Structure job details with clear separation
-                const jobDetails = {
-                    config: {
-                        ...jobConfig,
-                        enabled: jobConfig?.enabled ?? true,
-                        defaultConfig: {
-                            schedule: { value: latestJob?.repeatInterval },
-                            retry: {
-                                attempts: latestJob?.attempts,
-                                delay: latestJob?.backoff
-                            }
-                        },
-                        scheduleLabel: getScheduleLabel(jobConfig?.defaultConfig?.schedule?.value),
-                        scheduleOverride: jobConfig?.schedule,
-                        retryOverride: jobConfig?.retry
-                    },
-                    currentState: latestJob ? {
-                        status: getJobStatus(latestJob),
-                        nextRun: latestJob.nextRunAt,
-                        lastRun: latestJob.lastFinishedAt,
-                        lastRunStatus: getRunStatus(latestJob),
-                        failReason: latestJob.failReason,
-                        lastRunDuration: formatJobDuration(latestJob.lastRunAt, latestJob.lastFinishedAt),
-                        finishedCount: latestJob.finishedCount,
-                        totalRuns: latestJob.runCount
-                    } : null
-                };
+                const [normalDocs, total] = await Promise.all([
+                    normalDocsCursor.toArray(),
+                    jobsCollection.countDocuments(query)
+                ]);
+
+                // Fetch config override
+                const jobConfig = await jobConfigsCollection.findOne({ jobName });
+
+                // Build jobDetails
+                const jobDetails = buildJobDetails(scheduledDoc, jobConfig);
+
+                // Process normal docs
+                const processedRuns = normalDocs.map(doc => processRunDoc(doc));
 
                 common.returnOutput(ob.params, {
                     sEcho: ob.params.qstring.sEcho,
@@ -361,212 +323,85 @@ plugins.register('/o/jobs', async function(ob) {
                 });
                 return;
             }
+            else {
+                // Listing all unique job names
+                const search = sSearch ? { name: new RegExp(sSearch, 'i') } : {};
+
+                // We'll create a pipeline that groups by name. 
+                // We prefer the doc with type='single' if it exists, because we sort by type ascending first.
+                const pipeline = [
+                    { $match: search },
+                    {
+                        $sort: {
+                            type: -1, // ensures 'single' (0) is sorted before 'normal' (1)
+                            [sortColumn]: sortDir
+                        }
+                    },
+                    {
+                        $group: {
+                            _id: "$name",
+                            doc: { $first: "$$ROOT" }, // the first doc in each group
+                            lastRunAt: { $max: "$lastRunAt" },
+                            lastFinishedAt: { $max: "$lastFinishedAt" },
+                            failedAt: { $max: "$failedAt" }
+                        }
+                    },
+                    {
+                        $project: {
+                            _id: 0,
+                            name: "$_id",
+                            doc: 1,
+                            lastRunAt: 1,
+                            lastFinishedAt: 1,
+                            failedAt: 1
+                        }
+                    },
+                    { $skip: skip },
+                    { $limit: limit }
+                ];
+
+                // total distinct job names
+                const allNames = await jobsCollection.distinct('name', search);
+                const totalCount = allNames.length;
+
+                // Run pipeline
+                const rawJobs = await jobsCollection.aggregate(pipeline).toArray();
+
+                // Fetch jobConfigs in one pass
+                const jobConfigs = await jobConfigsCollection.find({}).toArray();
+                const configMap = {};
+                jobConfigs.forEach(cfg => {
+                    configMap[cfg.jobName] = cfg;
+                });
 
-            // Handle list view if no jobName provided
-            const skip = Number(iDisplayStart) || 0;
-            const limit = Number(iDisplayLength) || 50;
-            const search = sSearch || '';
-            const sort = {
-                [iSortCol_0 || 'nextRunAt']: sSortDir_0 === 'desc' ? -1 : 1
-            };
+                // Merge data
+                const processed = rawJobs.map(item => {
+                    const jobName = item.name;
+                    const mainDoc = item.doc; // the doc from $first
+                    const jobConfig = configMap[jobName] || null;
+
+                    return buildListViewJob(mainDoc, jobConfig, item);
+                });
+
+                // Each entry in final array should be { job: {...}, config: {...} }
+                const finalOutput = processed.map(data => ({
+                    job: data.job,
+                    config: data.config
+                }));
 
-            await handleListView(ob, jobsCollection, jobConfigsCollection, search, sort, skip, limit);
+                // Return
+                common.returnOutput(ob.params, {
+                    sEcho: ob.params.qstring.sEcho,
+                    iTotalRecords: totalCount,
+                    iTotalDisplayRecords: totalCount,
+                    aaData: finalOutput
+                });
+            }
         }
         catch (error) {
-            log.e('Error in jobs API:', { error: error.message, stack: error.stack });
+            log.e('Error in /o/jobs:', { error: error.message, stack: error.stack });
             common.returnMessage(ob.params, 500, 'Internal server error');
         }
     });
     return true;
 });
-
-/**
- * Handles detailed view for a specific job
- * @param {Object} ob - Request object
- * @param {string} jobName - Name of the job
- * @param {Collection} jobsCollection - MongoDB jobs collection
- * @param {Collection} jobConfigsCollection - MongoDB job configs collection
- * @param {number} skip - Number of records to skip
- * @param {number} limit - Number of records to return
- * @returns {Promise<void>}
- */
-async function handleDetailedView(ob, jobName, jobsCollection, jobConfigsCollection, skip, limit) {
-    const query = { name: jobName };
-
-    const [jobRuns, jobConfig, latestJob] = await Promise.all([
-        jobsCollection.find({ ...query, type: "single" })
-            .sort({ nextRunAt: -1 })
-            .skip(skip)
-            .limit(limit)
-            .toArray(),
-        jobConfigsCollection.findOne({ jobName }),
-        jobsCollection.findOne(
-            { name: jobName },
-            { sort: { lastFinishedAt: -1, lastRunAt: -1 } }
-        )
-    ]);
-
-    const total = await jobsCollection.countDocuments({ ...query, type: "single" });
-
-    const processedRuns = jobRuns.map(run => ({
-        name: run.name,
-        status: getJobStatus(run),
-        schedule: run.repeatInterval,
-        scheduleLabel: run.repeatInterval ? getScheduleLabel(run.repeatInterval) : 'Manual Run',
-        next: run.nextRunAt,
-        finished: run.lastFinishedAt,
-        lastRunStatus: getRunStatus(run),
-        failReason: run.failReason,
-        data: run.data,
-        dataAsString: JSON.stringify(run.data, null, 2),
-        duration: formatJobDuration(run.lastRunAt, run.lastFinishedAt),
-        durationInSeconds: formatJobDuration(run.lastRunAt, run.lastFinishedAt) + 's',
-        lastRunAt: run.lastRunAt,
-        nextRunDate: run.nextRunAt ? moment(run.nextRunAt).format('D MMM, YYYY') : '-',
-        nextRunTime: run.nextRunAt ? moment(run.nextRunAt).format('HH:mm:ss') : '-',
-        lastRun: run.lastFinishedAt ? moment(run.lastFinishedAt).fromNow() : '-'
-    }));
-
-    const jobDetails = {
-        config: {
-            ...jobConfig,
-            scheduleLabel: jobConfig?.defaultConfig?.schedule?.value ?
-                getScheduleLabel(jobConfig.defaultConfig.schedule.value) : '-',
-        },
-        currentState: latestJob ? {
-            status: getJobStatus(latestJob),
-            nextRun: latestJob.nextRunAt,
-            lastRun: latestJob.lastFinishedAt,
-            lastRunStatus: getRunStatus(latestJob),
-            failReason: latestJob.failReason,
-            lastRunDuration: formatJobDuration(latestJob.lastRunAt, latestJob.lastFinishedAt)
-        } : null
-    };
-
-    common.returnOutput(ob.params, {
-        sEcho: ob.params.qstring.sEcho,
-        iTotalRecords: total,
-        iTotalDisplayRecords: total,
-        aaData: processedRuns,
-        jobDetails: jobDetails
-    });
-}
-
-/**
- * Handles list view for all jobs
- * @param {Object} ob - Request object
- * @param {Collection} jobsCollection - MongoDB jobs collection
- * @param {Collection} jobConfigsCollection - MongoDB job configs collection
- * @param {string} search - Search string to filter jobs
- * @param {Object} sort - Sort configuration
- * @param {number} skip - Number of records to skip
- * @param {number} limit - Number of records to return
- * @returns {Promise<void>}
- */
-async function handleListView(ob, jobsCollection, jobConfigsCollection, search, sort, skip, limit) {
-    const query = search ? { name: new RegExp(search, 'i') } : {};
-
-    const pipeline = [
-        { $match: query },
-        {
-            $group: {
-                _id: "$name",
-                doc: { $first: "$$ROOT" },
-                lastFinishedAt: { $max: "$lastFinishedAt" },
-                lastRunAt: { $max: "$lastRunAt" },
-                lastFailedAt: { $max: "$failedAt" },
-                lastStatus: {
-                    $first: {
-                        $cond: [
-                            { $eq: ["$type", "single"] },
-                            {
-                                status: {
-                                    $switch: {
-                                        branches: [
-                                            { case: { $ne: ["$lockedAt", null] }, then: "RUNNING" },
-                                            { case: { $ne: ["$failedAt", null] }, then: "FAILED" },
-                                            { case: { $ne: ["$lastFinishedAt", null] }, then: "COMPLETED" }
-                                        ],
-                                        default: "SCHEDULED"
-                                    }
-                                },
-                                failReason: "$failReason",
-                                lastRunStatus: {
-                                    $switch: {
-                                        branches: [
-                                            { case: { $ne: ["$failedAt", null] }, then: "failed" },
-                                            { case: { $ne: ["$lastFinishedAt", null] }, then: "success" }
-                                        ],
-                                        default: "pending"
-                                    }
-                                }
-                            },
-                            "$$ROOT"
-                        ]
-                    }
-                }
-            }
-        },
-        {
-            $addFields: {
-                "doc.lastFinishedAt": "$lastFinishedAt",
-                "doc.lastRunAt": "$lastRunAt",
-                "doc.failedAt": "$lastFailedAt",
-                "doc.status": "$lastStatus.status",
-                "doc.failReason": "$lastStatus.failReason",
-                "doc.lastRunStatus": "$lastStatus.lastRunStatus"
-            }
-        },
-        { $replaceRoot: { newRoot: "$doc" }},
-        { $sort: sort },
-        { $skip: skip },
-        { $limit: limit }
-    ];
-
-    const [jobs, configs, total] = await Promise.all([
-        jobsCollection.aggregate(pipeline).toArray(),
-        jobConfigsCollection.find({}).toArray(),
-        jobsCollection.distinct('name', query).then(names => names.length)
-    ]);
-
-    const configMap = configs.reduce((map, config) => {
-        map[config.jobName] = config;
-        return map;
-    }, {});
-
-    const processedJobs = jobs.map(job => {
-        const config = configMap[job.name] || {
-            enabled: true,
-            defaultConfig: {
-                schedule: { value: job.repeatInterval },
-                retry: { attempts: job.attempts, delay: job.backoff }
-            }
-        };
-
-        return {
-            job: {
-                name: job.name,
-                status: getJobStatus(job),
-                schedule: config.defaultConfig.schedule.value,
-                scheduleLabel: getScheduleLabel(config.defaultConfig.schedule.value),
-                nextRunAt: job.nextRunAt,
-                lastFinishedAt: job.lastFinishedAt,
-                lastRunStatus: getRunStatus(job),
-                failReason: job.failReason,
-                total
-            },
-            config: {
-                enabled: config.enabled,
-                schedule: config.defaultConfig.schedule.value,
-                retry: config.defaultConfig.retry
-            }
-        };
-    });
-
-    common.returnOutput(ob.params, {
-        sEcho: ob.params.qstring.sEcho,
-        iTotalRecords: total,
-        iTotalDisplayRecords: total,
-        aaData: processedJobs
-    });
-}
\ No newline at end of file

From 3d70ec692604c15b3b86a67945983ea6ac0bea29 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Sat, 25 Jan 2025 12:07:55 +0530
Subject: [PATCH 47/90] lint fix

---
 jobServer/api.js | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/jobServer/api.js b/jobServer/api.js
index 6b97ae53e26..a012397d31b 100644
--- a/jobServer/api.js
+++ b/jobServer/api.js
@@ -7,7 +7,6 @@ const plugins = require("../plugins/pluginManager");
 const {validateGlobalAdmin} = require("../api/utils/rights");
 const common = require("../api/utils/common");
 const cronstrue = require('cronstrue');
-const moment = require('moment');
 
 // ----------------------------------
 // Helper Functions
@@ -33,7 +32,7 @@ function getJobStatus(job) {
 
 /**
  * Determine run status for a single job doc
- * @param {Object} job
+ * @param {Object} job doc from pulseJobs
  * @returns {String} "failed", "success", or "pending"
  */
 function getRunStatus(job) {
@@ -48,8 +47,8 @@ function getRunStatus(job) {
 
 /**
  * Returns job duration in seconds (string), or null if start/end not present
- * @param {Date} startDate
- * @param {Date} endDate
+ * @param {Date} startDate doc.lastRunAt
+ * @param {Date} endDate doc.lastFinishedAt
  * @returns {String|null} e.g. "2.34"
  */
 function formatJobDuration(startDate, endDate) {
@@ -62,8 +61,8 @@ function formatJobDuration(startDate, endDate) {
 
 /**
  * Returns a human-friendly label for a cron expression using cronstrue, or the raw expression if invalid
- * @param {String} schedule
- * @returns {String}
+ * @param {String} schedule doc.repeatInterval
+ * @returns {String} e.g. "Every 10 minutes"
  */
 function getScheduleLabel(schedule) {
     if (!schedule) {
@@ -81,7 +80,7 @@ function getScheduleLabel(schedule) {
  * Build the "jobDetails" object for a scheduled doc
  * @param {Object} scheduledDoc doc from pulseJobs, type='single'
  * @param {Object} jobConfig doc from jobConfigs
- * @returns {Object}
+ * @returns {Object} e.g. { config: {...}, currentState: {...} }
  */
 function buildJobDetails(scheduledDoc, jobConfig) {
     if (!scheduledDoc && !jobConfig) {
@@ -137,8 +136,8 @@ function buildJobDetails(scheduledDoc, jobConfig) {
 
 /**
  * Process a "normal" (type='normal') doc for job details table
- * @param {Object} doc
- * @returns {Object}
+ * @param {Object} doc doc from pulseJobs, type='normal'
+ * @returns {Object} e.g. { lastRunAt: Date, status: String, duration: String, result: Object, failReason: String, dataAsString: String }
  */
 function processRunDoc(doc) {
     return {
@@ -333,6 +332,7 @@ plugins.register('/o/jobs', async function(ob) {
                     { $match: search },
                     {
                         $sort: {
+
                             type: -1, // ensures 'single' (0) is sorted before 'normal' (1)
                             [sortColumn]: sortDir
                         }
@@ -376,9 +376,9 @@ plugins.register('/o/jobs', async function(ob) {
 
                 // Merge data
                 const processed = rawJobs.map(item => {
-                    const jobName = item.name;
+                    const name = item.name; // Changed from jobName to name
                     const mainDoc = item.doc; // the doc from $first
-                    const jobConfig = configMap[jobName] || null;
+                    const jobConfig = configMap[name] || null; // Use name here
 
                     return buildListViewJob(mainDoc, jobConfig, item);
                 });

From 880dc4669692766ac1ce568ca902a40c4dcedbd1 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Sat, 25 Jan 2025 12:08:09 +0530
Subject: [PATCH 48/90] wip: fixes for frontend

---
 .../core/jobs/javascripts/countly.views.js    | 30 +++++++++++--------
 .../core/jobs/templates/jobs-details.html     |  3 +-
 2 files changed, 20 insertions(+), 13 deletions(-)

diff --git a/frontend/express/public/core/jobs/javascripts/countly.views.js b/frontend/express/public/core/jobs/javascripts/countly.views.js
index cb1cdfe577d..11bb30ce912 100644
--- a/frontend/express/public/core/jobs/javascripts/countly.views.js
+++ b/frontend/express/public/core/jobs/javascripts/countly.views.js
@@ -3,6 +3,8 @@
 (function() {
     /**
      * Helper function to map the job status to a color tag
+     * @param {Object} row The job row object
+     * @returns {string} Color code for the status
      */
     var getColor = function(row) {
         // row is the merged job object
@@ -23,6 +25,8 @@
 
     /**
      * Helper to update row display fields (like nextRunDate, nextRunTime, lastRun)
+     * @param {Object} row The job row object to update
+     * @returns {void}
      */
     var updateScheduleRow = function(row) {
         row.nextRunDate = row.nextRunAt ? moment(row.nextRunAt).format('YYYY-MM-DD') : '';
@@ -114,6 +118,7 @@
         computed: {
             /**
              * Whether the current user can enable/disable jobs
+             * @returns {boolean} True if user has admin rights
              */
             canSuspendJob: function() {
                 return countlyGlobal.member.global_admin || countlyGlobal.admin_apps[countlyCommon.ACTIVE_APP_ID];
@@ -157,13 +162,18 @@
             },
             /**
              * Navigates to job details page
+             * @param {Object} row The job row to navigate to
+             * @returns {void}
              */
             goTo: function(row) {
                 app.navigate("#/manage/jobs/" + row.name, true);
             },
             getColor: getColor,
             /**
-             * Called from the row’s more options, e.g. "enable", "disable", "schedule", "runNow"
+             * Called from the row's more options, e.g. "enable", "disable", "schedule", "runNow"
+             * @param {string} command The command to execute
+             * @param {Object} row The job row
+             * @returns {void}
              */
             handleCommand: function(command, row) {
                 if (row.name) {
@@ -275,6 +285,7 @@
         computed: {
             /**
              * Check if there are any scheduleOverride or retryOverride in the config
+             * @returns {boolean} True if overrides exist
              */
             hasOverrides: function() {
                 return this.jobDetails &&
@@ -340,6 +351,8 @@
             },
             /**
              * Map jobDetails.currentState.status to a color
+             * @param {Object} jobDetails The job details object
+             * @returns {string} Color code for the status
              */
             getStatusColor: function(jobDetails) {
                 if (!jobDetails.config?.enabled) {
@@ -353,7 +366,9 @@
                 }
             },
             /**
-             * Map each run’s status to a color
+             * Map each run's status to a color
+             * @param {Object} run The job run object
+             * @returns {string} Color code for the status
              */
             getRunStatusColor: function(run) {
                 switch (run.status) {
@@ -372,6 +387,7 @@
 
     /**
      * Wrap the JobsView as a Countly Backbone view
+     * @returns {Object} Backbone wrapper view
      */
     var getMainView = function() {
         return new countlyVue.views.BackboneWrapper({
@@ -380,16 +396,6 @@
         });
     };
 
-    /**
-     * Wrap the JobDetailsView as a Countly Backbone view
-     */
-    var getDetailedView = function() {
-        return new countlyVue.views.BackboneWrapper({
-            component: JobDetailsView,
-            vuex: [] // empty array if none
-        });
-    };
-
     /**
      * Define routes for #/manage/jobs and #/manage/jobs/:jobName
      */
diff --git a/frontend/express/public/core/jobs/templates/jobs-details.html b/frontend/express/public/core/jobs/templates/jobs-details.html
index 9fdacb62120..46470044516 100644
--- a/frontend/express/public/core/jobs/templates/jobs-details.html
+++ b/frontend/express/public/core/jobs/templates/jobs-details.html
@@ -108,8 +108,9 @@ <h5 class="bu-is-size-5 bu-mb-2">
             <cly-section>
                 <h4 class="bu-is-size-4 bu-mb-4">
                     {{ i18n('jobs.job-history') }}
-                </h4>
 
+                </h4>
+                <h3>Single runs of jobs </h3>
                 <cly-datatable-n
                     :rows="jobRuns"
                     :columns="jobRunColumns"

From b317dcac004149a02a6526ce17d376ab1957a4c7 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Sat, 25 Jan 2025 12:08:33 +0530
Subject: [PATCH 49/90] simplify server connections

---
 jobServer/index.js | 119 ++++++++++++++-------------------------------
 1 file changed, 36 insertions(+), 83 deletions(-)

diff --git a/jobServer/index.js b/jobServer/index.js
index 6f443327fea..4c7b1e25863 100644
--- a/jobServer/index.js
+++ b/jobServer/index.js
@@ -55,26 +55,6 @@ if (require.main === module) {
 
     log.i('Initializing job server process...');
 
-
-    /**
-     * Initialize configuration
-     * @returns {Promise<Object>} A promise that resolves to an object containing configuration
-     */
-    const initializeConfig = async() => {
-        try {
-            const config = await pluginManager.getConfig();
-            log.d('Configuration initialized successfully');
-            return config;
-        }
-        catch (error) {
-            log.e('Failed to initialize configuration:', {
-                error: error.message,
-                stack: error.stack
-            });
-            throw new Error('Configuration initialization failed: ' + error.message);
-        }
-    };
-
     /**
      * Initialize countly request
      * @param {Object} config - Configuration object
@@ -96,31 +76,6 @@ if (require.main === module) {
         }
     };
 
-    /**
-     * Override common db objects with the provided connections
-     * @param {Object} dbConnections - Object containing database connections
-     */
-    const overrideCommonDb = async(dbConnections) => {
-        try {
-            if (!dbConnections || !dbConnections.countlyDb || !dbConnections.drillDb || !dbConnections.outDb) {
-                throw new Error('Invalid database connections provided');
-            }
-            const { countlyDb, drillDb, outDb } = dbConnections;
-            common.db = countlyDb;
-            common.drillDb = drillDb;
-            common.outDb = outDb;
-            log.d('Common db objects overridden successfully');
-        }
-        catch (error) {
-            log.e('Failed to override common db objects:', {
-                error: error.message,
-                stack: error.stack,
-                connections: Object.keys(dbConnections || {})
-            });
-            throw new Error('Common db override failed: ' + error.message);
-        }
-    };
-
     /**
      * Override common batcher with the provided connections
      * @param {Db} commonDb - Object containing database connections
@@ -138,37 +93,6 @@ if (require.main === module) {
         }
     };
 
-    /**
-     * Initialize database connections
-     * @returns {Promise<DbConnections>} A promise that resolves to an object containing database connections
-     */
-    const initializeDbConnections = async() => {
-        try {
-            log.d('Initializing database connections...');
-            const countlyDb = await pluginManager.dbConnection('countly');
-            log.d('Countly DB connection established');
-
-            const drillDb = await pluginManager.dbConnection('countly_drill');
-            log.d('Drill DB connection established');
-
-            const outDb = await pluginManager.dbConnection('countly_out');
-            log.d('Out DB connection established');
-
-            return {
-                countlyDb,
-                drillDb,
-                outDb,
-            };
-        }
-        catch (error) {
-            log.e('Failed to initialize database connections:', {
-                error: error.message,
-                stack: error.stack
-            });
-            throw new Error('Database connections initialization failed: ' + error.message);
-        }
-    };
-
     /**
      * Override common dispatch with a Countly server request
      * @param {Function} countlyRequest - Countly request function
@@ -229,17 +153,46 @@ if (require.main === module) {
         }
     };
 
-
-    initializeConfig()
-        .then(async config => {
-            log.i('Starting job server initialization sequence...');
+    /**
+     * Initialize configuration and database connections
+     * @returns {Promise<{config: Object, dbConnections: Array}>} e.g. { config: {...}, dbConnections: { countlyDb, outDb, fsDb, drillDb } }
+     */
+    const initializeSystem = async() => {
+        try {
+            const config = await pluginManager.getConfig();
+            log.d('Configuration initialized successfully');
 
             const countlyRequest = await initializeCountlyRequest(config);
             await overrideCommonDispatch(countlyRequest, config);
 
-            const dbConnections = await initializeDbConnections();
-            await overrideCommonDb(dbConnections);
-            await overrideCommonBatcher(dbConnections.countlyDb);
+            // Use connectToAllDatabases which handles config loading and db connections
+            const [countlyDb, outDb, fsDb, drillDb] = await pluginManager.connectToAllDatabases();
+
+            // Only need to override batcher since connectToAllDatabases handles other overrides
+            await overrideCommonBatcher(countlyDb);
+
+            return {
+                config,
+                dbConnections: {
+                    countlyDb,
+                    drillDb,
+                    outDb,
+                    fsDb
+                }
+            };
+        }
+        catch (error) {
+            log.e('Failed to initialize system:', {
+                error: error.message,
+                stack: error.stack
+            });
+            throw new Error('System initialization failed: ' + error.message);
+        }
+    };
+
+    initializeSystem()
+        .then(async({dbConnections}) => {
+            log.i('Starting job server initialization sequence...');
 
             jobServer = await JobServer.create(Logger, pluginManager, dbConnections);
 

From 290b4db944e4114bb0ab2fd9bd7a3c5dbeabe827 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 28 Jan 2025 18:09:08 +0530
Subject: [PATCH 50/90] remove hooks from multi process and job scheduling

---
 plugins/browser/api/api.js      |  2 +-
 plugins/crashes/api/api.js      |  4 ---
 plugins/density/api/api.js      |  2 +-
 plugins/locale/api/api.js       |  2 +-
 plugins/logger/api/api.js       |  7 -----
 plugins/plugins/api/api.js      | 45 ---------------------------------
 plugins/reports/api/api.js      |  7 -----
 plugins/server-stats/api/api.js |  7 -----
 plugins/sources/api/api.js      |  2 +-
 plugins/views/api/api.js        |  9 +------
 10 files changed, 5 insertions(+), 82 deletions(-)

diff --git a/plugins/browser/api/api.js b/plugins/browser/api/api.js
index 017516d046c..bf65bfa8d34 100644
--- a/plugins/browser/api/api.js
+++ b/plugins/browser/api/api.js
@@ -10,7 +10,7 @@ const FEATURE_NAME = 'browser';
     plugins.register("/permissions/features", function(ob) {
         ob.features.push(FEATURE_NAME);
     });
-    plugins.register("/worker", function() {
+    plugins.register("/master", function() {
         common.dbUserMap.browser = 'brw';
     });
     plugins.register("/o/method/total_users", function(ob) {
diff --git a/plugins/crashes/api/api.js b/plugins/crashes/api/api.js
index 3c0fef4bd23..08884fbf938 100644
--- a/plugins/crashes/api/api.js
+++ b/plugins/crashes/api/api.js
@@ -58,10 +58,6 @@ plugins.setConfigs("crashes", {
                 console.log(err);
             }
         });
-
-        // setTimeout(() => {
-        // require('../../../api/parts/jobs').job('crashes:cleanup_custom_field').replace().schedule('at 01:01 am ' + 'every 1 day');
-        // }, 10000);
     });
     var ranges = ["ram", "bat", "disk", "run", "session"];
     var segments = ["os_version", "os_name", "manufacture", "device", "resolution", "app_version", "cpu", "opengl", "orientation", "view", "browser"];
diff --git a/plugins/density/api/api.js b/plugins/density/api/api.js
index 77bc16c929a..c256ca9045d 100644
--- a/plugins/density/api/api.js
+++ b/plugins/density/api/api.js
@@ -10,7 +10,7 @@ const FEATURE_NAME = 'density';
     plugins.register("/permissions/features", function(ob) {
         ob.features.push(FEATURE_NAME);
     });
-    plugins.register("/worker", function() {
+    plugins.register("/master", function() {
         common.dbUserMap.density = 'dnst';
     });
     plugins.register("/o/method/total_users", function(ob) {
diff --git a/plugins/locale/api/api.js b/plugins/locale/api/api.js
index ddca14a6018..1baf6f37142 100644
--- a/plugins/locale/api/api.js
+++ b/plugins/locale/api/api.js
@@ -11,7 +11,7 @@ const FEATURE_NAME = 'locale';
     plugins.register("/permissions/features", function(ob) {
         ob.features.push(FEATURE_NAME);
     });
-    plugins.register("/worker", function() {
+    plugins.register("/master", function() {
         common.dbUserMap.locale = 'lo'; // full ISO locale from device
         common.dbUserMap.lang = 'la'; // language extracted from locale
     });
diff --git a/plugins/logger/api/api.js b/plugins/logger/api/api.js
index d1a6e5cadf5..246e398a66c 100644
--- a/plugins/logger/api/api.js
+++ b/plugins/logger/api/api.js
@@ -21,13 +21,6 @@ plugins.setConfigs("logger", {
 });
 
 (function() {
-    // plugins.register("/master", function() {
-    //     setTimeout(() => {
-    //         JOB.job('logger:clear', { max: MAX_NUMBER_OF_LOG_ENTRIES })
-    //             .replace()
-    //             .schedule("every 5 minutes");
-    //     }, 10000);
-    // });
 
     plugins.register("/permissions/features", function(ob) {
         ob.features.push(FEATURE_NAME);
diff --git a/plugins/plugins/api/api.js b/plugins/plugins/api/api.js
index 02165726e8a..2cfe276839b 100644
--- a/plugins/plugins/api/api.js
+++ b/plugins/plugins/api/api.js
@@ -29,51 +29,6 @@ var plugin = {},
                         common.returnMessage(params, 200, "started");
                     }
                 });
-                /*
-                try {
-                    params.qstring.plugin = JSON.parse(params.qstring.plugin);
-                }
-                catch (err) {
-                    console.log('Error parsing plugins');
-                }
-
-                if (params.qstring.plugin && typeof params.qstring.plugin === 'object') {
-                    // updatePluginState("start");
-
-                    var before = {};
-                    var fordb = {};
-                    var arr = plugins.getPlugins();
-                    for (var i in params.qstring.plugin) {
-                        fordb['plugins.' + i] = params.qstring.plugin[i];
-                        if (arr.indexOf(i) === -1) {
-                            before[i] = false;
-                        }
-                        else {
-                            before[i] = true;
-                        }
-                    }
-                    common.db.collection('plugins').updateOne({'_id': 'plugins'}, {'$set': fordb}, function(err1) {
-                        if (err1) {
-                            log.e(err1);
-                        }
-                        else {
-                            plugins.dispatch("/systemlogs", {params: params, action: "change_plugins", data: {before: before, update: params.qstring.plugin}});
-                            // process.send({ cmd: "startPlugins" });
-                            plugins.loadConfigs(common.db, function() {
-                                
-                                 plugins.syncPlugins(params.qstring.plugin, function(err) {
-									if (!err) {
-										process.send({ cmd: "endPlugins" });
-										updatePluginState("end");
-									}
-									else {
-										updatePluginState("failed");
-									}
-								}, common.db);
-                            });
-                        }
-                    });
-                }*/
             }
             else {
                 common.returnOutput(params, "Not enough parameters");
diff --git a/plugins/reports/api/api.js b/plugins/reports/api/api.js
index fdbbef28218..307c1ca1e2b 100644
--- a/plugins/reports/api/api.js
+++ b/plugins/reports/api/api.js
@@ -14,13 +14,6 @@ const FEATURE_NAME = 'reports';
         ob.features.push(FEATURE_NAME);
     });
 
-    // plugins.register("/master", function() {
-    //     // Allow configs to load & scanner to find all jobs classes
-    //     setTimeout(() => {
-    //         require('../../../api/parts/jobs').job('reports:send').replace().schedule("every 1 hour starting on the 0 min");
-    //     }, 10000);
-    // });
-
     /**
      * @api {get} /o/reports/all Get reports data 
      * @apiName  getData
diff --git a/plugins/server-stats/api/api.js b/plugins/server-stats/api/api.js
index a096f5c903b..c2ce52cc27e 100644
--- a/plugins/server-stats/api/api.js
+++ b/plugins/server-stats/api/api.js
@@ -18,13 +18,6 @@ const internalEventsSkipped = ["[CLY]_orientation"];
         ob.features.push(FEATURE_NAME);
     });
 
-    // plugins.register("/master", function() {
-    //     // Allow configs to load & scanner to find all jobs classes
-    //     setTimeout(() => {
-    //         require('../../../api/parts/jobs').job('server-stats:stats').replace().schedule('every 1 day');
-    //     }, 10000);
-    // });
-
     /**
      * @param {string} events - events to be mapped
      * @returns {object} Returns object with event key and count
diff --git a/plugins/sources/api/api.js b/plugins/sources/api/api.js
index eddd42c35e4..7a9efa61ffc 100644
--- a/plugins/sources/api/api.js
+++ b/plugins/sources/api/api.js
@@ -101,7 +101,7 @@ var utmTags = ["_ga", "_gac", "utm_source", "utm_medium", "utm_campaign", "utm_t
         return processedURL;
     };
 
-    plugins.register("/worker", function() {
+    plugins.register("/master", function() {
         common.dbUserMap.source = 'src';
     });
     plugins.register("/o/method/total_users", function(ob) {
diff --git a/plugins/views/api/api.js b/plugins/views/api/api.js
index 0dcb8764ae0..3ed7f1cf374 100644
--- a/plugins/views/api/api.js
+++ b/plugins/views/api/api.js
@@ -29,17 +29,10 @@ const escapedViewSegments = { "name": true, "segment": true, "height": true, "wi
     plugins.internalDrillEvents.push("[CLY]_view");
     plugins.internalDrillEvents.push("[CLY]_action");
 
-    plugins.register("/worker", function() {
+    plugins.register("/master", function() {
         common.dbUniqueMap.users.push("vc");
     });
 
-    // plugins.register("/master", function() {
-    //     // Allow configs to load & scanner to find all jobs classes
-    //     setTimeout(() => {
-    //         require('../../../api/parts/jobs').job('views:cleanupMeta')?.replace()?.schedule("every 1 day");
-    //     }, 3000);
-    // });
-
     plugins.register("/i/user_merge", function(ob) {
         var newAppUser = ob.newAppUser;
         var oldAppUser = ob.oldAppUser;

From 6be5c9b48d26f593391563ddd0c7967b1d75516a Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 28 Jan 2025 18:18:46 +0530
Subject: [PATCH 51/90] remove job handler from fetch.js

---
 api/parts/data/fetch.js | 125 ----------------------------------------
 1 file changed, 125 deletions(-)

diff --git a/api/parts/data/fetch.js b/api/parts/data/fetch.js
index 210c277571a..4eefc8d046d 100644
--- a/api/parts/data/fetch.js
+++ b/api/parts/data/fetch.js
@@ -20,7 +20,6 @@ var fetch = {},
     _ = require('underscore'),
     crypto = require('crypto'),
     usage = require('./usage.js'),
-    STATUS_MAP = require('../jobs/job').STATUS_MAP,
     plugins = require('../../../plugins/pluginManager.js');
 
 /**
@@ -2007,130 +2006,6 @@ function union(x, y) {
     return res;
 }
 
-/**
-* Get data for jobs listing for jobs api
-* @param {string} metric - name of the collection where to get data from
-* @param {params} params - params object with app_id and date
-*/
-fetch.fetchJobs = async function(metric, params) {
-    try {
-        if (params.qstring.name) {
-            await fetch.jobDetails(metric, params);
-        }
-        else {
-            await fetch.alljobs(metric, params);
-        }
-    }
-    catch (e) {
-        console.log(e);
-        common.returnOutput(params, 500, "Fetching jobs failed");
-    }
-};
-
-/**
-* Get all jobs grouped by job name for jobs api
-* @param {string} metric - name of the collection where to get data from
-* @param {params} params - params object with app_id and date
-*/
-fetch.alljobs = async function(metric, params) {
-    const columns = ["name", "schedule", "next", "finished", "status", "total"];
-    let sort = {};
-    let total = await common.db.collection('jobs').aggregate([
-        {
-            $group: { _id: "$name" }
-        },
-        {
-            $count: 'total'
-        }
-    ]).toArray();
-    total = total.length > 0 ? total[0].total : 0;
-    const pipeline = [
-        {
-            $addFields: {
-                sortKey: {
-                    $cond: {
-                        if: { $eq: ["$status", 0] },
-                        then: 0,
-                        else: {
-                            $cond: {
-                                if: { $eq: ["$status", 7] },
-                                then: 1,
-                                else: 2
-                            }
-                        }
-                    }
-                }
-            }
-        },
-        {
-            $sort: {
-                sortKey: 1,
-                finished: -1
-            }
-        },
-        {
-            $group: {
-                _id: "$name",
-                name: { $first: "$name" },
-                status: { $first: "$status" },
-                schedule: { $first: "$schedule" },
-                next: { $first: "$next" },
-                finished: { $first: "$finished" },
-                total: { $sum: 1 },
-                rowId: { $first: "$_id" }
-            }
-        }
-    ];
-    if (params.qstring.sSearch) {
-        var rr;
-        try {
-            rr = new RegExp(params.qstring.sSearch, "i");
-            pipeline.unshift({
-                $match: { name: { $regex: rr } }
-            });
-        }
-        catch (e) {
-            console.log('Could not use as regex:' + params.qstring.sSearch);
-        }
-    }
-    const cursor = common.db.collection('jobs').aggregate(pipeline, { allowDiskUse: true });
-    sort[columns[params.qstring.iSortCol_0 || 0]] = (params.qstring.sSortDir_0 === "asc") ? 1 : -1;
-    cursor.sort(sort);
-    cursor.skip(Number(params.qstring.iDisplayStart || 0));
-    cursor.limit(Number(params.qstring.iDisplayLength || 10));
-    let items = await cursor.toArray();
-    items = items.map((job) => {
-        job.status = STATUS_MAP[job.status];
-        return job;
-    });
-    cursor.close();
-    common.returnOutput(params, { sEcho: params.qstring.sEcho, iTotalRecords: total, iTotalDisplayRecords: total, aaData: items || [] });
-};
-
-/**
-* Get all documents for a given job name
-* @param {string} metric - name of the collection where to get data from
-* @param {params} params - params object with app_id and date
-*/
-fetch.jobDetails = async function(metric, params) {
-    const columns = ["schedule", "next", "finished", "status", "data", "duration"];
-    let sort = {};
-    const total = await common.db.collection('jobs').count({ name: params.qstring.name });
-    const cursor = common.db.collection('jobs').find({ name: params.qstring.name });
-    sort[columns[params.qstring.iSortCol_0 || 0]] = (params.qstring.sSortDir_0 === "asc") ? 1 : -1;
-    cursor.sort(sort);
-    cursor.skip(Number(params.qstring.iDisplayStart || 0));
-    cursor.limit(Number(params.qstring.iDisplayLength || 10));
-    let items = await cursor.toArray();
-    items = items.map((job) => {
-        job.status = STATUS_MAP[job.status];
-        return job;
-    });
-    cursor.close();
-    common.returnOutput(params, { sEcho: params.qstring.sEcho, iTotalRecords: total, iTotalDisplayRecords: total, aaData: items || [] });
-};
-
-
 /**
  * Fetch data for tops
  * @param {params} params - params object

From 9d13e0c080f2878202b87ac38cbf6a888f939814 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 28 Jan 2025 18:19:25 +0530
Subject: [PATCH 52/90] remove cluster from cache

---
 api/parts/data/cache.js | 619 +++++-----------------------------------
 1 file changed, 76 insertions(+), 543 deletions(-)

diff --git a/api/parts/data/cache.js b/api/parts/data/cache.js
index a2da942aaf1..cc4263b6e8f 100644
--- a/api/parts/data/cache.js
+++ b/api/parts/data/cache.js
@@ -1,13 +1,11 @@
 'use strict';
 
-const log = require('../../utils/log.js')('cache:' + process.pid),
-    // common = require('../../utils/common.js'),
-    { CentralWorker, CentralMaster } = require('../jobs/ipc.js'),
+const log = require('../../utils/log.js')('cache'),
     { Jsonable } = require('../../utils/models'),
     LRU = require('lru-cache'),
     config = require('../../config.js');
 
-const CENTRAL = 'cache', OP = {INIT: 'i', PURGE: 'p', READ: 'r', WRITE: 'w', UPDATE: 'u'};
+// const CENTRAL = 'cache', OP = {INIT: 'i', PURGE: 'p', READ: 'r', WRITE: 'w', UPDATE: 'u'};
 // new job: {o: 2, k: 'ObjectId', g: 'jobs', d: '{"name": "jobs:clean", "status": 0, ...}'}
 // job update: {o: 3, k: 'ObjectId', g: 'jobs', d: '{"status": 1}'}
 // job retreival: {o: 1, k: 'ObjectId', g: 'jobs'}
@@ -16,10 +14,12 @@ const CENTRAL = 'cache', OP = {INIT: 'i', PURGE: 'p', READ: 'r', WRITE: 'w', UPD
 
 /**
  * Get value in nested objects
- * @param  {object} obj         - object to checl
+ * @param  {object} obj         - object to check
  * @param  {string|array} is    - keys for nested value
  * @param  {any} value          - if provided acts as setter setting this value in nested object
  * @return {varies} returns value in provided key in nested object
+ * @note
+ * TODO: change to iterative if profiling finds memory issues
  */
 const dot = function(obj, is, value) {
     if (typeof is === 'string') {
@@ -54,7 +54,13 @@ class DataStore {
     constructor(size, age, dispose, Cls) {
         this.size = size;
         this.age = age;
-        this.lru = new LRU({max: size || 10000, ttl: age || Number.MAX_SAFE_INTEGER, dispose: dispose, noDisposeOnSet: true, updateAgeOnGet: true});
+        this.lru = new LRU({
+            max: size || 10000,
+            ttl: age || Number.MAX_SAFE_INTEGER,
+            dispose: dispose,
+            noDisposeOnSet: true,
+            updateAgeOnGet: true
+        });
         if (Cls) {
             this.Cls = Cls;
             this.Clas = require('../../../' + Cls[0])[Cls[1]];
@@ -151,415 +157,14 @@ class DataStore {
  * - notifies master about updates;
  * - loads data from master when local copy misses a particular key.
  */
-class CacheWorker {
+class Cache {
     /**
      * Constructor
-     *
      * @param  {Number} size max number of cache groups
      */
     constructor(size = 100) {
         this.data = new DataStore(size);
-        this.started = false;
-
-        this.ipc = new CentralWorker(CENTRAL, (m, reply) => {
-            let {o, k, g, d} = m || {};
-
-            if (!g) {
-                return;
-            }
-            log.d('handling %s: %j', reply ? 'reply' : 'broadcast', m);
-
-            let store = this.data.read(g);
-
-            if (o === OP.INIT) {
-                this.data.write(g, new DataStore(d.size, d.age, undefined, d.Cls));
-                return;
-            }
-            else if (!store) {
-                log.d('Group store is not initialized');
-                return;
-            }
-
-            if (o === OP.PURGE) {
-                if (k) {
-                    store.write(k, null);
-                }
-                else { // purgeAll
-                    store.iterate(id => store.write(id, null));
-                }
-            }
-            else if (o === OP.READ) {
-                store.write(k, d);
-            }
-            else if (o === OP.WRITE) {
-                store.write(k, d);
-            }
-            else if (o === OP.UPDATE) {
-                store.update(k, d);
-            }
-            else {
-                throw new Error(`Illegal cache operaion: ${o}, ${k}, ${g}, ${d}`);
-            }
-
-            // store.iterate((k, v) => {
-            //     log.d('have %s: %j', k, v);
-            // });
-        });
-    }
-
-    /**
-     * Start listening to IPC messages
-     */
-    async start() {
-        if (this.started === true) {
-            return;
-        }
-
-        if (this.started === false) {
-            log.d('starting worker');
-            this.started = new Promise((resolve, reject) => {
-                let timeout = setTimeout(() => {
-                    reject(new Error('Failed to start CacheWorker on timeout'));
-                }, 10000);
-                this.ipc.attach();
-                this.ipc.request({o: OP.INIT}).then(ret => {
-                    log.d('got init response: %j', ret);
-                    Object.keys(ret).forEach(g => {
-                        if (!this.data.read(g)) {
-                            this.data.write(g, new DataStore(ret[g].size, ret[g].age, undefined, ret[g].Cls));
-                            if (ret[g].data) {
-                                log.d('got %d data objects in init response', Object.keys(ret[g].data).length);
-                                for (let k in ret[g].data) {
-                                    this.data.read(g).write(k, ret[g].data[k]);
-                                }
-                            }
-                        }
-                    });
-                    this.started = true;
-                    clearTimeout(timeout);
-                    resolve();
-                });
-            });
-        }
-
-        await this.started;
-    }
-
-    /**
-     * Stop worker
-     */
-    async stop() {
-        this.ipc.detach();
-    }
-
-    /**
-     * Write data to cache:
-     * - send a write to the master;
-     * - wait for a response with write status;
-     * - write the data to local copy and return it in case of success, throw error otherwise.
-     *
-     * @param  {String} group group key
-     * @param  {String} id    data key
-     * @param  {Object} data  data to store
-     * @return {Object}       data if succeeded, null otherwise, throws in case of an error
-     */
-    async write(group, id, data) {
-        await this.start();
-
-        if (!group || !id || !data || typeof id !== 'string') {
-            throw new Error('Where are my args?');
-        }
-        else if (!this.data.read(group)) {
-            throw new Error('No such cache group');
-        }
-        log.d(`writing ${group}:${id}`);
-        let rsp = await this.ipc.request({o: OP.WRITE, g: group, k: id, d: data instanceof Jsonable ? data.json : data});
-        if (rsp) {
-            this.data.read(group).write(id, rsp);
-        }
-
-        return this.has(group, id);
-    }
-
-    /**
-     * Update data in the cache:
-     * - send an update to the master;
-     * - wait for a response with update status;
-     * - update the data in the local copy and return updated object in case of success, throw error otherwise.
-     *
-     * @param  {String} group  group key
-     * @param  {String} id     data key
-     * @param  {Object} update data to store
-     * @return {Object}        data if succeeded, null otherwise, throws in case of an error
-     */
-    async update(group, id, update) {
-        await this.start();
-
-        if (!group || !id || !update || typeof id !== 'string') {
-            throw new Error('Where are my args?!');
-        }
-        else if (!this.data.read(group)) {
-            throw new Error('No such cache group');
-        }
-        log.d(`updating ${group}:${id}`);
-        let rsp = await this.ipc.request({o: OP.UPDATE, g: group, k: id, d: update}),
-            store = this.data.read(group);
-        if (rsp) {
-            store.update(id, rsp);
-        }
-        else {
-            store.remove(id);
-
-        }
-        return this.has(group, id);
-    }
-
-    /**
-     * Remove a record from cache.
-     *
-     * @param  {String} group  group key
-     * @param  {String} id     data key
-     * @return {Boolean}       true if removed
-     */
-    async remove(group, id) {
-        await this.start();
-
-        if (!group || !id || typeof id !== 'string') {
-            throw new Error('Where are my args?!');
-        }
-        else if (!this.data.read(group)) {
-            throw new Error('No such cache group');
-        }
-        log.d(`removing ${group}:${id}`);
-        await this.ipc.request({o: OP.WRITE, g: group, k: id, d: null});
-        let store = this.data.read(group);
-        if (store) {
-            store.remove(id);
-        }
-        return this.has(group, id) === null;
-    }
-
-    /**
-     * Remove a record from cache.
-     *
-     * @param  {String} group  group key
-     * @param  {String} id     data key
-     * @return {Boolean}       true if removed
-     */
-    async purge(group, id) {
-        await this.start();
-
-        if (!group || !id || typeof id !== 'string') {
-            throw new Error('Where are my args?!');
-        }
-        else if (!this.data.read(group)) {
-            throw new Error('No such cache group');
-        }
-        log.d(`purging ${group}:${id}`);
-        await this.ipc.request({o: OP.PURGE, g: group, k: id});
-        let store = this.data.read(group);
-        if (store) {
-            store.remove(id);
-        }
-        return this.has(group, id) === null;
-    }
-
-    /**
-     * Remove from cache all records for a given group.
-     *
-     * @param  {String} group  group key
-     */
-    async purgeAll(group) {
-        await this.start();
-
-        if (!group) {
-            throw new Error('Where are my args?!');
-        }
-        else if (!this.data.read(group)) {
-            throw new Error('No such cache group');
-        }
-        log.d(`purging ${group}`);
-        await this.ipc.request({o: OP.PURGE, g: group});
-        let store = this.data.read(group);
-        store.iterate(id => store.write(id, null));
-    }
-
-    /**
-     * Read a record from cache:
-     * - from local copy if exists;
-     * - send a read request to master otherwise.
-     *
-     * @param  {String} group  group key
-     * @param  {String} id     data key
-     * @return {Object}        data if any, null otherwise
-     */
-    async read(group, id) {
-        await this.start();
-
-        if (!group || !id) {
-            throw new Error('Where are my args?!');
-        }
-        let data = this.has(group, id);
-        if (data) {
-            return data;
-        }
-        else {
-            let rsp = await this.ipc.request({o: OP.READ, g: group, k: id});
-            if (rsp) {
-                let store = this.data.read(group);
-                if (!store) {
-                    throw new Error(`No store for a group ${group}?!`);
-                    // store = this.data.write(group, new DataStore(this.size));
-                }
-                store.write(id, rsp);
-            }
-            return this.has(group, id);
-        }
-    }
-
-    /**
-     * Check if local copy has data under the key.
-     *
-     * @param  {String} group  group key
-     * @param  {String} id     data key
-     * @return {Object}        data if any, null otherwise
-     */
-    has(group, id) {
-        if (!group) {
-            throw new Error('Where are my args?!');
-        }
-        let store = this.data.read(group);
-        if (id) {
-            return store && store.read(id) || null;
-        }
-        else {
-            return store;
-        }
-    }
-
-    /**
-     * Just a handy method which returns an object with partials with given group.
-     *
-     * @param  {String} group group name
-     * @return {Object}       object with all the {@code CacheWorker} methods without group
-     */
-    cls(group) {
-        return {
-            read: this.read.bind(this, group),
-            write: this.write.bind(this, group),
-            update: this.update.bind(this, group),
-            remove: this.remove.bind(this, group),
-            purge: this.purge.bind(this, group),
-            purgeAll: this.purgeAll.bind(this, group),
-            has: this.has.bind(this, group),
-            iterate: f => {
-                let g = this.data.read(group);
-                if (g) {
-                    g.iterate(f);
-                }
-                else {
-                    log.e('no cache group %s to iterate on', group);
-                }
-            }
-        };
-    }
-}
-
-/**
- * Cache instance for master process:
- * - listen for requests from workers;
- * - call group operators to read/write/udpate
- */
-class CacheMaster {
-    /**
-     * Constructor
-     *
-     * @param  {Number} size max number of cache groups
-     */
-    constructor(size = 100) {
-        this.data = new DataStore(size, Number.MAX_SAFE_INTEGER);
         this.operators = {};
-        this.initialized = {};
-        this.delayed_messages = [];
-        this.ipc = new CentralMaster(CENTRAL, ({o, g, k, d}, reply, from) => {
-            log.d('handling %s: %j / %j / %j / %j', reply ? 'reply' : 'broadcast', o, g, k, d);
-
-            if (o === OP.INIT) {
-                this.initialized[from] = true;
-                let ret = {};
-                this.data.iterate((group, store) => {
-                    let data = {};
-                    store.iterate((key, obj) => {
-                        data[key] = obj instanceof Jsonable ? obj.json : obj;
-                    });
-                    ret[group] = {size: store.size, age: store.age, Cls: store.Cls, data};
-                });
-                setImmediate(() => {
-                    let remove = [];
-                    this.delayed_messages.filter(arr => arr[0] === from).forEach(arr => {
-                        remove.push(arr);
-                        this.ipc.send(arr[0], arr[1]);
-                    });
-                    if (remove.length) {
-                        log.d('sent %d delayed messages after %d worker\'s init', remove.length, from);
-                        remove.forEach(m => {
-                            const i = this.delayed_messages.indexOf(m);
-                            if (i !== -1) {
-                                this.delayed_messages.splice(i, 1);
-                            }
-                        });
-                    }
-                });
-                return ret;
-            }
-
-            let store = this.data.read(g);
-            if (!store) {
-                log.d(`No store for group ${g}`);
-                throw new Error('No such store ' + g);
-            }
-
-            if (o === OP.PURGE) {
-                if (k) {
-                    return this.purge(g, k, from);
-                }
-                else {
-                    return this.purgeAll(g, from);
-                }
-            }
-            else if (o === OP.READ) {
-                return this.read(g, k, from);
-            }
-            else if (o === OP.WRITE) {
-                return this.write(g, k, d, from);
-            }
-            else if (o === OP.UPDATE) {
-                return this.update(g, k, d, from);
-            }
-            else if (o === OP.REMOVE) {
-                return this.remove(g, k, from);
-            }
-            else {
-                throw new Error(`Illegal cache operaion: ${o}, ${k}, ${g}, ${d}`);
-            }
-        });
-    }
-
-    /**
-     * Attach to IPC
-     *
-     * @return {Promise<undefined>} void
-     */
-    async start() {
-        this.ipc.attach();
-        log.d('started master');
-    }
-
-    /**
-     * Detaches IPC instance
-     */
-    stop() {
-        this.ipc.detach();
     }
 
     /**
@@ -569,55 +174,44 @@ class CacheMaster {
      * @param  {Function} options.init   initializer - an "async () => [Object]" kind of function, preloads data to cache on startup
      * @param  {string[]} options.Cls    class - an optional array of ["require path", "export name"] which resolves to a Jsonable subclass to construct instances
      * @param  {Function} options.read   reader - an "async (key) => Object" kind of function, returns data to cache if any for the key supplied
-     * @param  {Function} options.write  writer - an "async (key, data) => Object" kind of function, persists the data cached if needed (must return the data persisted on success)
+     * @param  {Function} options.write  writer - an "async (key, data) => Object" kind of function, persists the data cached if needed
      * @param  {Function} options.update updater - an "async (key, update) => Object" kind of function, updates persisted data if needed
      * @param  {Function} options.remove remover - an "async (key) => Object" kind of function, removes persisted data if needed
      * @param  {int} age                 how long in ms to keep records in memory for the group
      * @param  {int} size                how much records to keep in memory for the group
      */
-    init(group, {init, Cls, read, write, update, remove}, size = null, age = null) {
+    async init(group, {init, Cls, read, write, update, remove}, size = null, age = null) {
         this.operators[group] = {init, Cls, read, write, update, remove};
 
         if (!size && size !== 0) {
-            size = config.api && config.api.cache && config.api.cache[group] && config.api.cache[group].size !== undefined ? config.api.cache[group].size : 10000;
+            size = config.api?.cache?.[group]?.size ?? 10000;
         }
 
         if (!age && age !== 0) {
-            age = config.api && config.api.cache && config.api.cache[group] && config.api.cache[group].age !== undefined ? config.api.cache[group].age : Number.MAX_SAFE_INTEGER;
+            age = config.api?.cache?.[group]?.age ?? Number.MAX_SAFE_INTEGER;
         }
 
-        this.data.write(group, new DataStore(size, age, k => {
-            this.ipc.send(0, {o: OP.PURGE, g: group, k});
-        }, Cls));
-
-        this.ipc.send(0, {o: OP.INIT, g: group, d: {size, age, Cls}});
+        this.data.write(group, new DataStore(size, age, null, Cls));
 
-        init().then(arr => {
+        try {
+            const arr = await init();
             (arr || []).forEach(([k, d]) => {
                 this.data.read(group).write(k, d);
-                const msg = {o: OP.READ, g: group, k, d: d && (d instanceof Jsonable) ? d.json : d};
-                for (const pid in this.ipc.workers) {
-                    if (this.initialized[pid]) {
-                        this.ipc.send(parseInt(pid), msg);
-                    }
-                    else {
-                        this.delayed_messages.push([parseInt(pid), msg]);
-                    }
-                }
             });
-        }, log.e.bind(log, 'Error during initialization of cache group %s', group));
+        }
+        catch (err) {
+            log.e('Error during initialization of cache group %s', group, err);
+        }
     }
 
     /**
      * Write data to the cache
-     *
      * @param  {String} group  group key
      * @param  {String} id     data key
      * @param  {Object} data   data to store
-     * @param  {int} from      originating pid if any
      * @return {Object}        data if succeeded, null otherwise, throws in case of an error
      */
-    async write(group, id, data, from = 0) {
+    async write(group, id, data) {
         if (!group || !id || (data === undefined) || typeof id !== 'string') {
             throw new Error('Where are my args?!');
         }
@@ -627,38 +221,29 @@ class CacheMaster {
         log.d(`writing ${group}:${id}: %j`, data);
 
         if (group in this.operators) {
-            return this.operators[group][data === null ? 'remove' : 'write'](id, data).then(rc => {
-                if (rc) {
-                    if (data === null) {
-                        rc = null;
-                    }
-                    if (rc instanceof Jsonable) {
-                        rc = rc.json;
-                    }
-                    this.data.read(group)[data === null ? 'remove' : 'write'](id, rc);
-                    this.ipc.send(-from, {o: OP.WRITE, g: group, k: id, d: rc});
-                    return data === null ? true : rc;
-                }
-                else {
-                    return null;
+            const rc = await this.operators[group][data === null ? 'remove' : 'write'](id, data);
+            if (rc) {
+                if (data === null) {
+                    this.data.read(group).remove(id);
+                    return true;
                 }
-            });
-        }
-        else {
+                const toStore = rc instanceof Jsonable ? rc.json : rc;
+                this.data.read(group).write(id, toStore);
+                return toStore;
+            }
             return null;
         }
+        return null;
     }
 
     /**
      * Update data in the cache
-     *
      * @param  {String} group  group key
      * @param  {String} id     data key
      * @param  {Object} update data to store
-     * @param  {int} from      originating pid if any
      * @return {Object}        data if succeeded, null otherwise, throws in case of an error
      */
-    async update(group, id, update, from = 0) {
+    async update(group, id, update) {
         if (!group || !id || !update || typeof id !== 'string') {
             throw new Error('Where are my args?!');
         }
@@ -668,15 +253,11 @@ class CacheMaster {
         log.d(`updating ${group}:${id} with %j`, update);
 
         if (group in this.operators) {
-            return this.operators[group].update(id, update).then(() => {
-                this.data.read(group).update(id, update);
-                this.ipc.send(-from, {o: OP.UPDATE, g: group, k: id, d: update});
-                return update;
-            });
-        }
-        else {
-            return null;
+            await this.operators[group].update(id, update);
+            this.data.read(group).update(id, update);
+            return update;
         }
+        return null;
     }
 
     /**
@@ -684,33 +265,18 @@ class CacheMaster {
      *
      * @param  {String} group  group key
      * @param  {String} id     data key
-     * @param  {int} from      originating pid if any
      * @return {Boolean}       true if removed
      */
-    async remove(group, id, from) {
+    async remove(group, id) {
         if (!group || !id || typeof id !== 'string') {
             throw new Error('Where are my args?!');
         }
         else if (!this.data.read(group)) {
             throw new Error('No such cache group');
         }
-
         log.d(`removing ${group}:${id}`);
-        if (group in this.operators) {
-            return this.operators[group].remove(id).then(rc => {
-                if (rc) {
-                    this.data.read(group).remove(id);
-                    this.ipc.send(-from, {o: OP.WRITE, g: group, k: id, d: null});
-                    return true;
-                }
-                else {
-                    return null;
-                }
-            });
-        }
-        else {
-            return null;
-        }
+        this.data.read(group).remove(id);
+        return true;
     }
 
     /**
@@ -718,10 +284,9 @@ class CacheMaster {
      *
      * @param  {String} group  group key
      * @param  {String} id     data key
-     * @param  {int} from      originating pid if any
      * @return {Boolean}       true if removed
      */
-    async purge(group, id, from = 0) {
+    async purge(group, id) {
         if (!group || !id || typeof id !== 'string') {
             throw new Error('Where are my args?!');
         }
@@ -729,42 +294,33 @@ class CacheMaster {
             throw new Error('No such cache group');
         }
         log.d(`purging ${group}:${id}`);
-
         this.data.read(group).write(id, null);
-        this.ipc.send(-from, {o: OP.PURGE, g: group, k: id});
         return true;
     }
 
     /**
-     * Remove from cache all record for given group.
+     * Remove from cache all records for a given group.
      *
      * @param  {String} group  group key
-     * @param  {int} from      originating pid if any
-     * @return {Boolean}       true if removed
      */
-    async purgeAll(group, from = 0) {
+    async purgeAll(group) {
         if (!group) {
             throw new Error('Where are my args?!');
         }
+        else if (!this.data.read(group)) {
+            throw new Error('No such cache group');
+        }
         log.d(`purging ${group}`);
-
-        let grp = this.data.read(group);
-        grp.iterate(k => grp.write(k, null));
-        this.ipc.send(-from, {o: OP.PURGE, g: group});
-        return true;
+        this.data.read(group).iterate(id => this.data.read(group).write(id, null));
     }
 
     /**
-     * Read a record from cache:
-     * - from local copy if exists;
-     * - send a read request to master otherwise.
-     *
+     * Read data from the cache
      * @param  {String} group  group key
      * @param  {String} id     data key
-     * @param  {int} from      originating pid if any
-     * @return {Object}        data if any, null otherwise
+     * @return {Object}        data if succeeded, null otherwise, throws in case of an error
      */
-    async read(group, id, from = 0) {
+    async read(group, id) {
         if (!group || !id || typeof id !== 'string') {
             throw new Error('Where are my args?!');
         }
@@ -772,75 +328,52 @@ class CacheMaster {
             throw new Error('No such cache group');
         }
 
-        let store = this.data.read(group),
-            rc = store.read(id);
-        if (rc) {
-            return rc;
-        }
-        else if (group in this.operators) {
-            return this.operators[group].read(id).then(x => {
-                if (x) {
-                    this.ipc.send(-from, {o: OP.READ, g: group, k: id, d: x instanceof Jsonable ? x.json : x});
-                    store.write(id, x);
-                    return x;
-                }
-                else {
-                    return null;
-                }
-            });
+        let data = this.has(group, id);
+        if (data) {
+            return data;
         }
-        else {
-            return null;
+
+        if (group in this.operators) {
+            const rc = await this.operators[group].read(id);
+            if (rc) {
+                const toStore = rc instanceof Jsonable ? rc.json : rc;
+                this.data.read(group).write(id, toStore);
+                return toStore;
+            }
         }
+        return null;
     }
 
     /**
-     * Check if local copy has data under the key.
-     *
+     * Check if data exists in cache
      * @param  {String} group  group key
      * @param  {String} id     data key
-     * @return {Object}        data if any, undefined otherwise
+     * @return {Object|null}   data if exists, null otherwise
      */
     has(group, id) {
-        if (!group) {
+        if (!group || !id || typeof id !== 'string') {
             throw new Error('Where are my args?!');
         }
-        let store = this.data.read(group);
-        if (id) {
-            return store && store.read(id) || null;
-        }
-        else {
-            return store;
+        else if (!this.data.read(group)) {
+            throw new Error('No such cache group');
         }
+
+        return this.data.read(group).read(id);
     }
 
     /**
-     * Just a handy method which returns an object with partials with given group.
-     *
+     * Get class interface for a group
      * @param  {String} group group name
-     * @return {Object}       object with all the {@code CacheWorker} methods without group
+     * @return {Object}       object with read/write/update methods bound to the group
      */
     cls(group) {
         return {
             read: this.read.bind(this, group),
             write: this.write.bind(this, group),
             update: this.update.bind(this, group),
-            remove: this.remove.bind(this, group),
-            purge: this.purge.bind(this, group),
-            purgeAll: this.purgeAll.bind(this, group),
-            has: this.has.bind(this, group),
-            iterate: f => {
-                let g = this.data.read(group);
-                if (g) {
-                    g.iterate(f);
-                }
-                else {
-                    log.e('no cache group %s to iterate on', group);
-                }
-            }
+            remove: this.remove.bind(this, group)
         };
     }
-
 }
 
 /**
@@ -856,4 +389,4 @@ class TestDataClass extends Jsonable {
 }
 
 
-module.exports = {CacheMaster, CacheWorker, TestDataClass};
+module.exports = {Cache, TestDataClass};

From 6454c863ce32fbe6c3ee254ab6a8852af611a92d Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 28 Jan 2025 18:19:52 +0530
Subject: [PATCH 53/90] remove cluster from batcher

---
 api/parts/data/batcher.js | 193 ++++++++++++++------------------------
 1 file changed, 71 insertions(+), 122 deletions(-)

diff --git a/api/parts/data/batcher.js b/api/parts/data/batcher.js
index ccb879b51dc..4260f991975 100644
--- a/api/parts/data/batcher.js
+++ b/api/parts/data/batcher.js
@@ -1,5 +1,3 @@
-const crypto = require('crypto');
-const cluster = require('cluster');
 const plugins = require('../../../plugins/pluginManager.js');
 const log = require('../../utils/log.js')("batcher");
 const common = require('../../utils/common.js');
@@ -59,7 +57,6 @@ class InsertBatcher {
         let config = plugins.getConfig("api");
         this.period = config.batch_period * 1000;
         this.process = config.batch_processing;
-        this.shared = false;
     }
 
     /**
@@ -152,26 +149,21 @@ class InsertBatcher {
      *  @param {string} db - name of the database for which to write data
      */
     insert(collection, doc, db = "countly") {
-        if (!this.shared || cluster.isMaster) {
-            if (!this.data[db][collection]) {
-                this.data[db][collection] = [];
-            }
-            if (Array.isArray(doc)) {
-                for (let i = 0; i < doc.length; i++) {
-                    this.data[db][collection].push(doc[i]);
-                }
-                batcherStats.insert_queued += doc.length;
-            }
-            else {
-                batcherStats.insert_queued++;
-                this.data[db][collection].push(doc);
-            }
-            if (!this.process) {
-                this.flush(db, collection);
+        if (!this.data[db][collection]) {
+            this.data[db][collection] = [];
+        }
+        if (Array.isArray(doc)) {
+            for (let i = 0; i < doc.length; i++) {
+                this.data[db][collection].push(doc[i]);
             }
+            batcherStats.insert_queued += doc.length;
         }
         else {
-            process.send({ cmd: "batch_insert", data: {collection, doc, db} });
+            batcherStats.insert_queued++;
+            this.data[db][collection].push(doc);
+        }
+        if (!this.process) {
+            this.flush(db, collection);
         }
     }
 }
@@ -213,7 +205,6 @@ class WriteBatcher {
         let config = plugins.getConfig("api");
         this.period = config.batch_period * 1000;
         this.process = config.batch_processing;
-        this.shared = false;
     }
 
     /**
@@ -343,29 +334,24 @@ class WriteBatcher {
      *  @param {object=} options - options for operation ((upsert: false) - if you don't want to upsert document)
      */
     add(collection, id, operation, db = "countly", options) {
-        if (!this.shared || cluster.isMaster) {
-            if (!this.data[db][collection]) {
-                this.data[db][collection] = {};
-            }
-            if (!this.data[db][collection][id]) {
-                this.data[db][collection][id] = {id: id, value: operation, upsert: true};
-                if (options && options.upsert === false) {
-                    this.data[db][collection][id].upsert = false;
-                }
-                batcherStats.update_queued++;
-            }
-            else {
-                this.data[db][collection][id].value = common.mergeQuery(this.data[db][collection][id].value, operation);
-                if (options && options.upsert === false) {
-                    this.data[db][collection][id].upsert = this.data[db][collection][id].upsert || false;
-                }
-            }
-            if (!this.process) {
-                this.flush(db, collection);
+        if (!this.data[db][collection]) {
+            this.data[db][collection] = {};
+        }
+        if (!this.data[db][collection][id]) {
+            this.data[db][collection][id] = {id: id, value: operation, upsert: true};
+            if (options && options.upsert === false) {
+                this.data[db][collection][id].upsert = false;
             }
+            batcherStats.update_queued++;
         }
         else {
-            process.send({ cmd: "batch_write", data: {collection, id, operation, db} });
+            this.data[db][collection][id].value = common.mergeQuery(this.data[db][collection][id].value, operation);
+            if (options && options.upsert === false) {
+                this.data[db][collection][id].upsert = this.data[db][collection][id].upsert || false;
+            }
+        }
+        if (!this.process) {
+            this.flush(db, collection);
         }
     }
 }
@@ -384,25 +370,10 @@ class ReadBatcher {
     constructor(db) {
         this.db = db;
         this.data = {};
-        this.promises = {};
         plugins.loadConfigs(db, () => {
             this.loadConfig();
             this.schedule();
         });
-
-        if (!cluster.isMaster) {
-            process.on("message", (msg) => {
-                if (msg.cmd === "batch_read" && msg.data && msg.data.msgId && this.promises[msg.data.msgId]) {
-                    if (msg.data.data) {
-                        this.promises[msg.data.msgId].resolve(msg.data.data);
-                    }
-                    else {
-                        this.promises[msg.data.msgId].reject(msg.data.err);
-                    }
-                    delete this.promises[msg.data.msgId];
-                }
-            });
-        }
     }
 
     /**
@@ -413,7 +384,6 @@ class ReadBatcher {
         this.period = config.batch_read_period * 1000;
         this.ttl = config.batch_read_ttl * 1000;
         this.process = config.batch_read_processing;
-        this.onMaster = false;
     }
 
     /**
@@ -520,17 +490,12 @@ class ReadBatcher {
      *  @param {bool} multi - true if multiple documents
      */
     invalidate(collection, query, projection, multi) {
-        if (!this.onMaster || cluster.isMaster) {
-            var id = JSON.stringify(query) + "_" + multi;
-            if (!this.data[collection]) {
-                this.data[collection] = {};
-            }
-            if (this.data[collection][id] && !this.data[collection][id].promise) {
-                delete this.data[collection][id];
-            }
+        var id = JSON.stringify(query) + "_" + multi;
+        if (!this.data[collection]) {
+            this.data[collection] = {};
         }
-        else {
-            process.send({ cmd: "batch_invalidate", data: {collection, query, projection, multi} });
+        if (this.data[collection][id] && !this.data[collection][id].promise) {
+            delete this.data[collection][id];
         }
     }
 
@@ -543,65 +508,56 @@ class ReadBatcher {
      *  @returns {Promise} promise
      */
     get(collection, query, projection, multi) {
-        if (!this.onMaster || cluster.isMaster) {
-            var id = JSON.stringify(query) + "_" + multi;
-            if (!this.data[collection]) {
-                this.data[collection] = {};
-            }
-            var good_projection = true;
-            var keysSaved = this.keysFromProjectionObject(this.data[collection][id] && this.data[collection][id].projection);
-            var keysNew = this.keysFromProjectionObject(projection);
-
-            if (this.data[collection][id] && (keysSaved.have_projection || keysNew.have_projection)) {
-                if (keysSaved.have_projection) {
-                    for (let p = 0; p < keysNew.keys.length; p++) {
-                        if (keysSaved.keys.indexOf(keysNew.keys[p]) === -1) {
-                            good_projection = false;
-                            keysSaved.keys.push(keysNew.keys[p]);
-                        }
-                    }
-                }
-                if (!good_projection) {
-                    projection = {};
-                    for (var p = 0; p < keysSaved.keys.length; p++) {
-                        projection[keysSaved.keys[p]] = 1;
+        var id = JSON.stringify(query) + "_" + multi;
+        if (!this.data[collection]) {
+            this.data[collection] = {};
+        }
+        var good_projection = true;
+        var keysSaved = this.keysFromProjectionObject(this.data[collection][id] && this.data[collection][id].projection);
+        var keysNew = this.keysFromProjectionObject(projection);
+
+        if (this.data[collection][id] && (keysSaved.have_projection || keysNew.have_projection)) {
+            if (keysSaved.have_projection) {
+                for (let p = 0; p < keysNew.keys.length; p++) {
+                    if (keysSaved.keys.indexOf(keysNew.keys[p]) === -1) {
+                        good_projection = false;
+                        keysSaved.keys.push(keysNew.keys[p]);
                     }
                 }
             }
-
-            if (!this.process || !good_projection || !this.data[collection][id] || this.data[collection][id].last_updated < Date.now() - this.period) {
-                if (this.process) {
-                    this.data[collection][id] = {
-                        query: query,
-                        promise: this.getData(collection, id, query, projection, multi),
-                        projection: projection,
-                        last_used: Date.now(),
-                        last_updated: Date.now(),
-                        multi: multi
-                    };
-                    return this.data[collection][id].promise;
-                }
-                else {
-                    return this.getData(collection, id, query, projection, multi);
+            if (!good_projection) {
+                projection = {};
+                for (var p = 0; p < keysSaved.keys.length; p++) {
+                    projection[keysSaved.keys[p]] = 1;
                 }
             }
-            //we already have a read for this
-            else if (this.data[collection][id] && this.data[collection][id].promise) {
+        }
+
+        if (!this.process || !good_projection || !this.data[collection][id] || this.data[collection][id].last_updated < Date.now() - this.period) {
+            if (this.process) {
+                this.data[collection][id] = {
+                    query: query,
+                    promise: this.getData(collection, id, query, projection, multi),
+                    projection: projection,
+                    last_used: Date.now(),
+                    last_updated: Date.now(),
+                    multi: multi
+                };
                 return this.data[collection][id].promise;
             }
             else {
-                this.data[collection][id].last_used = Date.now();
-
-                return new Promise((resolve) => {
-                    resolve(this.data[collection][id].data);
-                });
+                return this.getData(collection, id, query, projection, multi);
             }
         }
+        //we already have a read for this
+        else if (this.data[collection][id] && this.data[collection][id].promise) {
+            return this.data[collection][id].promise;
+        }
         else {
-            return new Promise((resolve, reject) => {
-                let msgId = getId();
-                this.promises[msgId] = {resolve, reject};
-                process.send({ cmd: "batch_read", data: {collection, query, projection, multi, msgId} });
+            this.data[collection][id].last_used = Date.now();
+
+            return new Promise((resolve) => {
+                resolve(this.data[collection][id].data);
             });
         }
     }
@@ -685,12 +641,5 @@ function promiseOrCallback(promise, callback) {
     return promise;
 }
 
-/**
- *  Generate random id
- *  @returns {string} randomly generated id
- */
-function getId() {
-    return crypto.randomBytes(16).toString("hex");
-}
 
 module.exports = {WriteBatcher, ReadBatcher, InsertBatcher};

From 4a2d44550b5ba68744469bf96b6444c97f88891d Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 28 Jan 2025 18:20:38 +0530
Subject: [PATCH 54/90] remove ipchandler from logger

---
 api/utils/log.js | 81 ------------------------------------------------
 1 file changed, 81 deletions(-)

diff --git a/api/utils/log.js b/api/utils/log.js
index 40bdc3e6a3d..2fbc1a4f736 100644
--- a/api/utils/log.js
+++ b/api/utils/log.js
@@ -193,86 +193,6 @@ var getEnabledWithLevel = function(acceptable, module) {
         return acceptable.indexOf(levels[module] || deflt) !== -1;
     };
 };
-/**
-* Handle messages from ipc
-* @static
-* @param {string} msg - message received from other processes
-**/
-var ipcHandler = function(msg) {
-    var m, l, modules, i;
-
-    if (!msg || msg.cmd !== 'log' || !msg.config) {
-        return;
-    }
-
-    // console.log('%d: Setting logging config to %j (was %j)', process.pid, msg.config, levels);
-
-    if (msg.config.default) {
-        deflt = msg.config.default;
-    }
-
-    for (m in levels) {
-        var found = null;
-        for (l in msg.config) {
-            modules = msg.config[l].split(',').map(function(v) {
-                return v.trim();
-            });
-
-            for (i = 0; i < modules.length; i++) {
-                if (modules[i] === m) {
-                    found = l;
-                }
-            }
-        }
-
-        if (found === null) {
-            for (l in msg.config) {
-                modules = msg.config[l].split(',').map(function(v) {
-                    return v.trim();
-                });
-
-                for (i = 0; i < modules.length; i++) {
-                    if (modules[i].indexOf('*') === -1 && modules[i] === m.split(':')[0]) {
-                        found = l;
-                    }
-                    else if (modules[i].indexOf('*') !== -1 && modules[i].split(':')[1] === '*' && modules[i].split(':')[0] === m.split(':')[0]) {
-                        found = l;
-                    }
-                }
-            }
-        }
-
-        if (found !== null) {
-            levels[m] = found;
-        }
-        else {
-            levels[m] = deflt;
-        }
-    }
-
-    for (l in msg.config) {
-        if (msg.config[l] && l !== 'default') {
-            modules = msg.config[l].split(',').map(function(v) {
-                return v.trim();
-            });
-            prefs[l] = modules;
-
-            for (i in modules) {
-                m = modules[i];
-                if (!(m in levels)) {
-                    levels[m] = l;
-                }
-            }
-        }
-        else {
-            prefs[l] = [];
-        }
-    }
-
-    prefs.default = msg.config.default;
-
-    // console.log('%d: Set logging config to %j (now %j)', process.pid, msg.config, levels);
-};
 
 /**
  * @typedef {Object} Logger
@@ -551,4 +471,3 @@ module.exports = function(name) {
 module.exports.setLevel = setLevel;
 module.exports.setDefault = setDefault;
 module.exports.getLevel = getLevel;
-module.exports.ipcHandler = ipcHandler;
\ No newline at end of file

From 3f27384e78651acf042aa3ab8021b86d30ae842a Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 28 Jan 2025 18:21:00 +0530
Subject: [PATCH 55/90] remove job handler from requestprocessor

---
 api/utils/requestProcessor.js | 99 -----------------------------------
 1 file changed, 99 deletions(-)

diff --git a/api/utils/requestProcessor.js b/api/utils/requestProcessor.js
index 17f71afe129..0d6f6318e0f 100644
--- a/api/utils/requestProcessor.js
+++ b/api/utils/requestProcessor.js
@@ -2645,105 +2645,6 @@ const processRequest = (params) => {
                 }
 
                 switch (params.qstring.method) {
-                case 'jobs':
-                    /**
-                     * @api {get} /o?method=jobs Get Jobs Table Information
-                     * @apiName GetJobsTableInfo
-                     * @apiGroup Jobs
-                     * 
-                     * @apiDescription Get jobs information in the jobs table
-                     * @apiQuery {String} method which kind jobs requested, it should be 'jobs'
-                     * 
-                     * @apiSuccess {Number} iTotalRecords Total number of jobs
-                     * @apiSuccess {Number} iTotalDisplayRecords Total number of jobs by filtering
-                     * @apiSuccess {Objects[]} aaData Job details
-                     * @apiSuccess {Number} sEcho DataTable's internal counter
-                     * 
-                     * @apiSuccessExample {json} Success-Response:
-                     * HTTP/1.1 200 OK
-                     * {
-                     *   "sEcho": "0",
-                     *   "iTotalRecords": 14,
-                     *   "iTotalDisplayRecords": 14,
-                     *   "aaData": [{
-                     *     "_id": "server-stats:stats",
-                     *     "name": "server-stats:stats",
-                     *     "status": "SCHEDULED",
-                     *     "schedule": "every 1 day",
-                     *     "next": 1650326400000,
-                     *     "finished": 1650240007917,
-                     *     "total": 1
-                     *   }]
-                     * }
-                     */
-
-                    /**
-                    * @api {get} /o?method=jobs/name Get Job Details Table Information
-                    * @apiName GetJobDetailsTableInfo
-                    * @apiGroup Jobs
-                    * 
-                    * @apiDescription Get the information of the filtered job in the table
-                    * @apiQuery {String} method Which kind jobs requested, it should be 'jobs'
-                    * @apiQuery {String} name The job name is required to redirect to the selected job
-                    * 
-                    * @apiSuccess {Number} iTotalRecords Total number of jobs
-                    * @apiSuccess {Number} iTotalDisplayRecords Total number of jobs by filtering
-                    * @apiSuccess {Objects[]} aaData Job details
-                    * @apiSuccess {Number} sEcho DataTable's internal counter
-                    * 
-                    * @apiSuccessExample {json} Success-Response:
-                    * HTTP/1.1 200 OK
-                    * {
-                    *   "sEcho": "0",
-                    *   "iTotalRecords": 1,
-                    *   "iTotalDisplayRecords": 1,
-                    *   "aaData": [{
-                    *     "_id": "62596cd41307dc89c269b5a8",
-                    *     "name": "api:ping",
-                    *     "created": 1650027732240,
-                    *     "status": "SCHEDULED",
-                    *     "started": 1650240000865,
-                    *     "finished": 1650240000891,
-                    *     "duration": 30,
-                    *     "data": {},
-                    *     "schedule": "every 1 day",
-                    *     "next": 1650326400000,
-                    *     "modified": 1650240000895,
-                    *     "error": null
-                    *   }]
-                    * }
-                    */
-
-                    validateUserForGlobalAdmin(params, countlyApi.data.fetch.fetchJobs, 'jobs');
-                    break;
-                case 'suspend_job': {
-                    /**
-                     * @api {get} /o?method=suspend_job Suspend Job
-                     * @apiName SuspendJob
-                     * @apiGroup Jobs
-                     *  
-                     * @apiDescription Suspend the selected job
-                     * * 
-                     * @apiSuccessExample {json} Success-Response:
-                     * HTTP/1.1 200 OK
-                     * {
-                     *  "result": true,
-                     *  "message": "Job suspended successfully"
-                     * }
-                     * 
-                     * @apiErrorExample {json} Error-Response:
-                     * HTTP/1.1 400 Bad Request
-                     * {
-                     *  "result": "Updating job status failed" 
-                     * }
-                     * 
-                    */
-                    validateUserForGlobalAdmin(params, async() => {
-                        // await Handle.suspendJob(params);
-                        // TODO: UPDATE TO NEW JOB PROCESSOR
-                    });
-                    break;
-                }
                 case 'total_users':
                     validateUserForDataReadAPI(params, 'core', countlyApi.data.fetch.fetchTotalUsersObj, params.qstring.metric || 'users');
                     break;

From c0ad9e920e6b327495fb5d05d4ad38964d3abd3f Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 28 Jan 2025 18:21:20 +0530
Subject: [PATCH 56/90] remove cluster from tracker

---
 api/parts/mgmt/tracker.js | 26 ++++++++++++--------------
 1 file changed, 12 insertions(+), 14 deletions(-)

diff --git a/api/parts/mgmt/tracker.js b/api/parts/mgmt/tracker.js
index d878ab7188f..835bc951c35 100644
--- a/api/parts/mgmt/tracker.js
+++ b/api/parts/mgmt/tracker.js
@@ -13,7 +13,6 @@ var tracker = {},
     countlyConfig = require("../../../frontend/express/config.js"),
     versionInfo = require('../../../frontend/express/version.info'),
     ip = require('./ip.js'),
-    cluster = require('cluster'),
     os = require('os'),
     fs = require('fs'),
     asyncjs = require('async'),
@@ -107,20 +106,19 @@ tracker.enable = function() {
     if (countlyConfig.web.track !== "none" && countlyConfig.web.server_track !== "none") {
         Countly.track_errors();
     }
-    if (cluster.isMaster) {
-        setTimeout(function() {
-            if (countlyConfig.web.track !== "none" && countlyConfig.web.server_track !== "none") {
-                Countly.begin_session(true);
-                setTimeout(function() {
-                    collectServerStats();
-                    collectServerData();
-                }, 20000);
-            }
-        }, 1000);
-        //report app start trace
-        if (Countly.report_app_start) {
-            Countly.report_app_start();
+
+    setTimeout(function() {
+        if (countlyConfig.web.track !== "none" && countlyConfig.web.server_track !== "none") {
+            Countly.begin_session(true);
+            setTimeout(function() {
+                collectServerStats();
+                collectServerData();
+            }, 20000);
         }
+    }, 1000);
+    //report app start trace
+    if (Countly.report_app_start) {
+        Countly.report_app_start();
     }
 };
 

From bb7c836661af5ef3dddb633cc32f27ca72f8c939 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 28 Jan 2025 18:21:43 +0530
Subject: [PATCH 57/90] cleaner import in jobserver

---
 jobServer/index.js | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/jobServer/index.js b/jobServer/index.js
index 4c7b1e25863..4971cc812bd 100644
--- a/jobServer/index.js
+++ b/jobServer/index.js
@@ -42,12 +42,11 @@ const Logger = require('../api/utils/log.js');
 const log = new Logger('jobServer:index');
 const CountlyRequest = require("countly-request");
 const {ReadBatcher, WriteBatcher} = require('../api/parts/data/batcher');
+const common = require('../api/utils/common.js');
+const pluginManager = require('../plugins/pluginManager.js');
 
 // Start the process if this file is run directly
 if (require.main === module) {
-    const common = require('../api/utils/common.js');
-    const pluginManager = require('../plugins/pluginManager.js');
-
     /**
      * @type {JobServer|null}
      */

From ee53851dc05a26e5fe0a156452a63401acd946f9 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 28 Jan 2025 18:22:47 +0530
Subject: [PATCH 58/90] much slimmer api.js

---
 api/api.js | 346 +++++++++++++++--------------------------------------
 1 file changed, 98 insertions(+), 248 deletions(-)

diff --git a/api/api.js b/api/api.js
index 31f3404d393..f9649d8e6af 100644
--- a/api/api.js
+++ b/api/api.js
@@ -1,15 +1,11 @@
 const http = require('http');
-const cluster = require('cluster');
 const formidable = require('formidable');
-const os = require('os');
 const countlyConfig = require('./config', 'dont-enclose');
 const plugins = require('../plugins/pluginManager.js');
-// const jobs = require('./parts/jobs');
 const log = require('./utils/log.js')('core:api');
 const common = require('./utils/common.js');
 const {processRequest} = require('./utils/requestProcessor');
 const frontendConfig = require('../frontend/express/config.js');
-const {CacheMaster, CacheWorker} = require('./parts/data/cache.js');
 const {WriteBatcher, ReadBatcher, InsertBatcher} = require('./parts/data/batcher.js');
 const pack = require('../package.json');
 const versionInfo = require('../frontend/express/version.info.js');
@@ -18,18 +14,9 @@ const moment = require("moment");
 var t = ["countly:", "api"];
 common.processRequest = processRequest;
 
-if (cluster.isMaster) {
-    console.log("Starting Countly", "version", versionInfo.version, "package", pack.version);
-    if (!common.checkDatabaseConfigMatch(countlyConfig.mongodb, frontendConfig.mongodb)) {
-        log.w('API AND FRONTEND DATABASE CONFIGS ARE DIFFERENT');
-    }
-    t.push("master");
-    t.push("node");
-    t.push(process.argv[1]);
-}
-else {
-    t.push("worker");
-    t.push("node");
+console.log("Starting Countly", "version", versionInfo.version, "package", pack.version);
+if (!common.checkDatabaseConfigMatch(countlyConfig.mongodb, frontendConfig.mongodb)) {
+    log.w('API AND FRONTEND DATABASE CONFIGS ARE DIFFERENT');
 }
 
 // Finaly set the visible title
@@ -43,8 +30,6 @@ plugins.connectToAllDatabases().then(function() {
         common.drillReadBatcher = new ReadBatcher(common.drillDb);
     }
 
-    let workers = [];
-
     /**
     * Set Max Sockets
     */
@@ -130,17 +115,6 @@ plugins.connectToAllDatabases().then(function() {
             warn: (countlyConfig.logging && countlyConfig.logging.warn) ? countlyConfig.logging.warn.join(', ') : '',
             error: (countlyConfig.logging && countlyConfig.logging.error) ? countlyConfig.logging.error.join(', ') : '',
             default: (countlyConfig.logging && countlyConfig.logging.default) ? countlyConfig.logging.default : 'warn',
-        },
-        undefined,
-        () => {
-            const cfg = plugins.getConfig('logs'), msg = {
-                cmd: 'log',
-                config: cfg
-            };
-            if (process.send) {
-                process.send(msg);
-            }
-            require('./utils/log.js').ipcHandler(msg);
         }
     );
 
@@ -208,241 +182,117 @@ plugins.connectToAllDatabases().then(function() {
         console.trace();
     });
 
-    /**
-    * Pass To Master
-    * @param {cluster.Worker} worker - worker thatw as spawned by master
-    */
-    const passToMaster = (worker) => {
-        worker.on('message', (msg) => {
-            if (msg.cmd === 'log') {
-                workers.forEach((w) => {
-                    if (w !== worker) {
-                        w.send({
-                            cmd: 'log',
-                            config: msg.config
-                        });
-                    }
-                });
-                require('./utils/log.js').ipcHandler(msg);
-            }
-            else if (msg.cmd === "checkPlugins") {
-                plugins.checkPluginsMaster();
-            }
-            else if (msg.cmd === "startPlugins") {
-                plugins.startSyncing();
-            }
-            else if (msg.cmd === "endPlugins") {
-                plugins.stopSyncing();
-            }
-            else if (msg.cmd === "batch_insert") {
-                const {collection, doc, db} = msg.data;
-                common.insertBatcher.insert(collection, doc, db);
-            }
-            else if (msg.cmd === "batch_write") {
-                const {collection, id, operation, db} = msg.data;
-                common.writeBatcher.add(collection, id, operation, db);
-            }
-            else if (msg.cmd === "batch_read") {
-                const {collection, query, projection, multi, msgId} = msg.data;
-                common.readBatcher.get(collection, query, projection, multi).then((data) => {
-                    worker.send({ cmd: "batch_read", data: {msgId, data} });
-                })
-                    .catch((err) => {
-                        worker.send({ cmd: "batch_read", data: {msgId, err} });
-                    });
-            }
-            else if (msg.cmd === "batch_invalidate") {
-                const {collection, query, projection, multi} = msg.data;
-                common.readBatcher.invalidate(collection, query, projection, multi);
-            }
-            else if (msg.cmd === "dispatchMaster" && msg.event) {
-                plugins.dispatch(msg.event, msg.data);
-            }
-            else if (msg.cmd === "dispatch" && msg.event) {
-                workers.forEach((w) => {
-                    w.send(msg);
-                });
-            }
-        });
-    };
-
-    if (cluster.isMaster) {
-        plugins.installMissingPlugins(common.db);
-        common.runners = require('./parts/jobs/runner');
-        common.cache = new CacheMaster();
-        common.cache.start().then(() => {
-            setImmediate(() => {
-                plugins.dispatch('/cache/init', {});
-            });
-        }, e => {
-            console.log(e);
-            process.exit(1);
-        });
-
-        const workerCount = (countlyConfig.api.workers)
-            ? countlyConfig.api.workers
-            : os.cpus().length;
-
-        for (let i = 0; i < workerCount; i++) {
-            // there's no way to define inspector port of a worker in the code. So if we don't
-            // pick a unique port for each worker, they conflict with each other.
-            let nodeOptions = {};
-            if (countlyConfig?.symlinked !== true) { // countlyConfig.symlinked is passed when running in a symlinked setup
-                const inspectorPort = i + 1 + (common?.config?.masterInspectorPort || 9229);
-                nodeOptions = { NODE_OPTIONS: "--inspect-port=" + inspectorPort };
-            }
-            const worker = cluster.fork(nodeOptions);
-            workers.push(worker);
+    // jobs.job('api:topEvents').replace().schedule('at 00:01 am ' + 'every 1 day'); // PORTED
+    // jobs.job('api:ping').replace().schedule('every 1 day'); // PORTED
+    // jobs.job('api:clear').replace().schedule('every 1 day'); // REMOVED
+    // jobs.job('api:clearTokens').replace().schedule('every 1 day'); // PORTED
+    // jobs.job('api:clearAutoTasks').replace().schedule('every 1 day'); // PORTED
+    // jobs.job('api:task').replace().schedule('every 5 minutes'); // PORTED
+    // jobs.job('api:userMerge').replace().schedule('every 10 minutes'); // PORTED
+    // jobs.job("api:ttlCleanup").replace().schedule("every 1 minute"); // PORTED
+    //jobs.job('api:appExpire').replace().schedule('every 1 day'); // Deprecated
+
+
+    var utcMoment = moment.utc();
+    var incObj = {};
+    incObj.r = 1;
+    incObj[`d.${utcMoment.format("D")}.${utcMoment.format("H")}.r`] = 1;
+    common.db.collection("diagnostic").updateOne({"_id": "no-segment_" + utcMoment.format("YYYY:M")}, {"$set": {"m": utcMoment.format("YYYY:M")}, "$inc": incObj}, {"upsert": true}, function(err) {
+        if (err) {
+            log.e(err);
         }
+    });
 
-        workers.forEach(passToMaster);
-
-        cluster.on('exit', (worker) => {
-            workers = workers.filter((w) => {
-                return w !== worker;
-            });
-            const newWorker = cluster.fork();
-            workers.push(newWorker);
-            passToMaster(newWorker);
-        });
-
-        plugins.dispatch("/master", {});
 
-        // Allow configs to load & scanner to find all jobs classes
-        // setTimeout(() => {
-        // jobs.job('api:topEvents').replace().schedule('at 00:01 am ' + 'every 1 day'); // PORTED
-        // jobs.job('api:ping').replace().schedule('every 1 day'); // PORTED
-        // jobs.job('api:clear').replace().schedule('every 1 day'); // REMOVED
-        // jobs.job('api:clearTokens').replace().schedule('every 1 day'); // PORTED
-        // jobs.job('api:clearAutoTasks').replace().schedule('every 1 day'); // PORTED
-        // jobs.job('api:task').replace().schedule('every 5 minutes'); // PORTED
-        // jobs.job('api:userMerge').replace().schedule('every 10 minutes'); // PORTED
-        // jobs.job("api:ttlCleanup").replace().schedule("every 1 minute"); // PORTED
-        //jobs.job('api:appExpire').replace().schedule('every 1 day'); // Deprecated
-        // }, 10000);
+    plugins.installMissingPlugins(common.db);
+    const taskManager = require('./utils/taskmanager.js');
+    //since process restarted mark running tasks as errored
+    taskManager.errorResults({db: common.db});
 
-        //Record as restarted
+    // plugins.dispatch("/worker", {common: common});
+    plugins.dispatch("/master", {}); // init hook
 
-        var utcMoment = moment.utc();
+    http.Server((req, res) => {
+        const params = {
+            qstring: {},
+            res: res,
+            req: req
+        };
 
-        var incObj = {};
-        incObj.r = 1;
-        incObj[`d.${utcMoment.format("D")}.${utcMoment.format("H")}.r`] = 1;
-        common.db.collection("diagnostic").updateOne({"_id": "no-segment_" + utcMoment.format("YYYY:M")}, {"$set": {"m": utcMoment.format("YYYY:M")}, "$inc": incObj}, {"upsert": true}, function(err) {
-            if (err) {
-                log.e(err);
+        if (req.method.toLowerCase() === 'post') {
+            const formidableOptions = {};
+            if (countlyConfig.api.maxUploadFileSize) {
+                formidableOptions.maxFileSize = countlyConfig.api.maxUploadFileSize;
             }
-        });
-    }
-    else {
-        console.log("Starting worker", process.pid, "parent:", process.ppid);
-        const taskManager = require('./utils/taskmanager.js');
-
-        common.cache = new CacheWorker();
-        common.cache.start();
 
-        //since process restarted mark running tasks as errored
-        taskManager.errorResults({db: common.db});
-
-        process.on('message', common.log.ipcHandler);
-
-        process.on('message', (msg) => {
-            if (msg.cmd === 'log') {
-                common.log.ipcHandler(msg);
+            const form = new formidable.IncomingForm(formidableOptions);
+            if (/crash_symbols\/(add_symbol|upload_symbol)/.test(req.url)) {
+                req.body = [];
+                req.on('data', (data) => {
+                    req.body.push(data);
+                });
             }
-            else if (msg.cmd === "dispatch" && msg.event) {
-                plugins.dispatch(msg.event, msg.data || {});
+            else {
+                req.body = '';
+                req.on('data', (data) => {
+                    req.body += data;
+                });
             }
-        });
-
-        process.on('exit', () => {
-            console.log('Exiting due to master exited');
-        });
-
-        plugins.dispatch("/worker", {common: common});
 
-        http.Server((req, res) => {
-            const params = {
-                qstring: {},
-                res: res,
-                req: req
-            };
+            let multiFormData = false;
+            // Check if we have 'multipart/form-data'
+            if (req.headers['content-type']?.startsWith('multipart/form-data')) {
+                multiFormData = true;
+            }
 
-            if (req.method.toLowerCase() === 'post') {
-                const formidableOptions = {};
-                if (countlyConfig.api.maxUploadFileSize) {
-                    formidableOptions.maxFileSize = countlyConfig.api.maxUploadFileSize;
+            form.parse(req, (err, fields, files) => {
+                //handle bakcwards compatability with formiddble v1
+                for (let i in files) {
+                    if (files[i].filepath) {
+                        files[i].path = files[i].filepath;
+                    }
+                    if (files[i].mimetype) {
+                        files[i].type = files[i].mimetype;
+                    }
+                    if (files[i].originalFilename) {
+                        files[i].name = files[i].originalFilename;
+                    }
                 }
-
-                const form = new formidable.IncomingForm(formidableOptions);
-                if (/crash_symbols\/(add_symbol|upload_symbol)/.test(req.url)) {
-                    req.body = [];
-                    req.on('data', (data) => {
-                        req.body.push(data);
-                    });
+                params.files = files;
+                if (multiFormData) {
+                    let formDataUrl = [];
+                    for (const i in fields) {
+                        params.qstring[i] = fields[i];
+                        formDataUrl.push(`${i}=${fields[i]}`);
+                    }
+                    params.formDataUrl = formDataUrl.join('&');
                 }
                 else {
-                    req.body = '';
-                    req.on('data', (data) => {
-                        req.body += data;
-                    });
+                    for (const i in fields) {
+                        params.qstring[i] = fields[i];
+                    }
                 }
-
-                let multiFormData = false;
-                // Check if we have 'multipart/form-data'
-                if (req.headers['content-type']?.startsWith('multipart/form-data')) {
-                    multiFormData = true;
+                if (!params.apiPath) {
+                    processRequest(params);
                 }
+            });
+        }
+        else if (req.method.toLowerCase() === 'options') {
+            const headers = {};
+            headers["Access-Control-Allow-Origin"] = "*";
+            headers["Access-Control-Allow-Methods"] = "POST, GET, OPTIONS";
+            headers["Access-Control-Allow-Headers"] = "countly-token, Content-Type";
+            res.writeHead(200, headers);
+            res.end();
+        }
+        //attempt process GET request
+        else if (req.method.toLowerCase() === 'get') {
+            processRequest(params);
+        }
+        else {
+            common.returnMessage(params, 405, "Method not allowed");
+        }
+    }).listen(common.config.api.port, common.config.api.host || '').timeout = common.config.api.timeout || 120000;
 
-                form.parse(req, (err, fields, files) => {
-                    //handle bakcwards compatability with formiddble v1
-                    for (let i in files) {
-                        if (files[i].filepath) {
-                            files[i].path = files[i].filepath;
-                        }
-                        if (files[i].mimetype) {
-                            files[i].type = files[i].mimetype;
-                        }
-                        if (files[i].originalFilename) {
-                            files[i].name = files[i].originalFilename;
-                        }
-                    }
-                    params.files = files;
-                    if (multiFormData) {
-                        let formDataUrl = [];
-                        for (const i in fields) {
-                            params.qstring[i] = fields[i];
-                            formDataUrl.push(`${i}=${fields[i]}`);
-                        }
-                        params.formDataUrl = formDataUrl.join('&');
-                    }
-                    else {
-                        for (const i in fields) {
-                            params.qstring[i] = fields[i];
-                        }
-                    }
-                    if (!params.apiPath) {
-                        processRequest(params);
-                    }
-                });
-            }
-            else if (req.method.toLowerCase() === 'options') {
-                const headers = {};
-                headers["Access-Control-Allow-Origin"] = "*";
-                headers["Access-Control-Allow-Methods"] = "POST, GET, OPTIONS";
-                headers["Access-Control-Allow-Headers"] = "countly-token, Content-Type";
-                res.writeHead(200, headers);
-                res.end();
-            }
-            //attempt process GET request
-            else if (req.method.toLowerCase() === 'get') {
-                processRequest(params);
-            }
-            else {
-                common.returnMessage(params, 405, "Method not allowed");
-            }
-        }).listen(common.config.api.port, common.config.api.host || '').timeout = common.config.api.timeout || 120000;
-
-        plugins.loadConfigs(common.db);
-    }
+    plugins.loadConfigs(common.db);
 });

From 1c9abc0a916f4302a227a738936b10e8f1f463c7 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 28 Jan 2025 18:26:10 +0530
Subject: [PATCH 59/90] remove cluster from pluginmanager

---
 plugins/pluginManager.js | 77 ++++++++++++++++------------------------
 1 file changed, 31 insertions(+), 46 deletions(-)

diff --git a/plugins/pluginManager.js b/plugins/pluginManager.js
index 11b3b4dbea2..398efbec306 100644
--- a/plugins/pluginManager.js
+++ b/plugins/pluginManager.js
@@ -6,7 +6,6 @@ var pluginDependencies = require('./pluginDependencies.js'),
     }),
     pluginsApis = {},
     mongodb = require('mongodb'),
-    cluster = require('cluster'),
     countlyConfig = require('../frontend/express/config', 'dont-enclose'),
     apiCountlyConfig = require('../api/config', 'dont-enclose'),
     utils = require('../api/utils/utils.js'),
@@ -159,7 +158,6 @@ var pluginManager = function pluginManager() {
                 }
                 else {
                     self.dispatch("/systemlogs", {params: params, action: "change_plugins", data: {before: before, update: params.qstring.plugin}});
-                    // process.send({ cmd: "startPlugins" });
                     self.loadConfigs(db, function() {
                         callback();
                     });
@@ -1228,54 +1226,46 @@ var pluginManager = function pluginManager() {
     };
 
     /**
-    * We check plugins and sync between processes/servers
+    * We check plugins and sync configuration
     * @param {object} db - connection to countly database
     * @param {function} callback - when finished checking and syncing
     **/
     this.checkPlugins = function(db, callback) {
-        var plugConf;
-        if (cluster.isMaster) {
-            plugConf = this.getConfig("plugins");
-            if (Object.keys(plugConf).length === 0) {
-                //no plugins inserted yet, upgrading by inserting current plugins
-                var list = this.getPlugins();
-                for (let i = 0; i < list.length; i++) {
-                    plugConf[list[i]] = true;
-                }
-                this.updateConfigs(db, "plugins", plugConf, callback);
-            }
-            else {
-                this.syncPlugins(plugConf, callback);
+        var plugConf = this.getConfig("plugins");
+
+        if (Object.keys(plugConf).length === 0) {
+            // No plugins inserted yet, initialize by inserting current plugins
+            var list = this.getPlugins();
+            for (let i = 0; i < list.length; i++) {
+                plugConf[list[i]] = true;
             }
+            this.updateConfigs(db, "plugins", plugConf, callback);
         }
         else {
-            //check if we need to sync plugins
+            // Check if we need to sync plugins
             var pluginList = this.getPlugins();
-            plugConf = this.getConfig("plugins") || {};
-            //let master know we need to include initial plugins
-            if (Object.keys(plugConf).length === 0) {
-                process.send({ cmd: "checkPlugins" });
-            }
-            else {
-                //check if we need to sync plugins
-                var changes = 0;
-                for (let plugin in plugConf) {
-                    var state = plugConf[plugin],
-                        index = pluginList.indexOf(plugin);
-                    if (index !== -1 && !state) {
-                        changes++;
-                        plugins.splice(plugins.indexOf(plugin), 1);
-                    }
-                    else if (index === -1 && state) {
-                        changes++;
-                        plugins.push(plugin);
-                    }
+            var changes = 0;
+
+            for (let plugin in plugConf) {
+                var state = plugConf[plugin],
+                    index = pluginList.indexOf(plugin);
+                if (index !== -1 && !state) {
+                    changes++;
+                    plugins.splice(plugins.indexOf(plugin), 1);
                 }
-                if (changes > 0) {
-                    //let master process know we need to sync plugins
-                    process.send({ cmd: "checkPlugins" });
+                else if (index === -1 && state) {
+                    changes++;
+                    plugins.push(plugin);
                 }
             }
+
+            if (changes > 0) {
+                // Sync plugins directly since we're in a single process
+                this.syncPlugins(plugConf, callback);
+            }
+            else if (callback) {
+                callback();
+            }
         }
     };
 
@@ -1833,17 +1823,12 @@ var pluginManager = function pluginManager() {
     * @returns {object} db connection params
     **/
     this.dbConnection = async function(config) {
-        var db, maxPoolSize = 10;
+        var db, maxPoolSize = 100;
         var mngr = this;
         var dbList = [];
 
-        if (!cluster.isMaster) {
-            //we are in worker
-            maxPoolSize = 100;
-        }
-
         var useConfig = JSON.parse(JSON.stringify(countlyConfig));
-        if (process.argv[1] && process.argv[1].endsWith('api/api.js') && !cluster.isMaster) {
+        if (process.argv[1] && process.argv[1].endsWith('api/api.js')) {
             useConfig = JSON.parse(JSON.stringify(apiCountlyConfig));
         }
         if (typeof config === "string") {

From 5e542862b9906915e82a0185d6e33b07515663e7 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Wed, 29 Jan 2025 13:02:59 +0530
Subject: [PATCH 60/90] missing dependency for jobs ui

---
 package.json | 1 +
 1 file changed, 1 insertion(+)

diff --git a/package.json b/package.json
index 645b256306a..91a007c41b5 100644
--- a/package.json
+++ b/package.json
@@ -51,6 +51,7 @@
     "countly-sdk-nodejs": "*",
     "countly-sdk-web": "*",
     "cron-validator": "^1.3.1",
+    "cronstrue": "^2.53.0",
     "csurf": "^1.11.0",
     "csvtojson": "2.0.10",
     "ejs": "3.1.10",

From 79886ad3805cc99062d7b3870f9355377753c904 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Wed, 29 Jan 2025 18:26:18 +0530
Subject: [PATCH 61/90] remove push from default json for temporarily

---
 plugins/plugins.default.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/plugins.default.json b/plugins/plugins.default.json
index b191ff043fa..03274cde8a3 100644
--- a/plugins/plugins.default.json
+++ b/plugins/plugins.default.json
@@ -1 +1 @@
-["mobile","web","desktop","plugins","density","locale","browser","sources","views","logger","systemlogs","errorlogs","populator","reports","crashes","push","star-rating","slipping-away-users","compare","server-stats","dbviewer","times-of-day","compliance-hub","alerts","onboarding","consolidate","remote-config","hooks","dashboards","sdk","data-manager","guides"]
+["mobile","web","desktop","plugins","density","locale","browser","sources","views","logger","systemlogs","errorlogs","populator","reports","crashes","star-rating","slipping-away-users","compare","server-stats","dbviewer","times-of-day","compliance-hub","alerts","onboarding","consolidate","remote-config","hooks","dashboards","sdk","data-manager","guides"]

From 9015238b3bd50618d8de50963474386cf93a9a9a Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Wed, 29 Jan 2025 18:33:32 +0530
Subject: [PATCH 62/90] disable old jobs tests

---
 test/2.api/17.jobs.js | 98 +++++++++++++++++++++----------------------
 1 file changed, 49 insertions(+), 49 deletions(-)

diff --git a/test/2.api/17.jobs.js b/test/2.api/17.jobs.js
index cfb48cdc434..f4daf72b6a3 100644
--- a/test/2.api/17.jobs.js
+++ b/test/2.api/17.jobs.js
@@ -7,55 +7,55 @@ var API_KEY_ADMIN = "";
 var APP_ID = "";
 
 describe("Testing Jobs", function() {
-    it("Fetching jobs table data", function(done) {
-        API_KEY_ADMIN = testUtils.get("API_KEY_ADMIN");
-        APP_ID = testUtils.get("APP_ID");
-        request
-            .get("/o?app_id=" + APP_ID + "&api_key=" + API_KEY_ADMIN + "&method=jobs")
-            .expect(200)
-            .end(function(err, res) {
-                if (err) {
-                    return done(err);
-                }
-                var ob = JSON.parse(res.text);
-                var iTotalRecords = ob.iTotalRecords;
-                var iTotalDisplayRecords = ob.iTotalDisplayRecords;
-                if (iTotalDisplayRecords > 0 && iTotalRecords > 0) {
-                    ob.should.have.property("aaData");
-                    done();
-                }
-                else {
-                    done("There are no records to show");
-                }
-            });
-    });
-
-    it("Fetching jobs table data filtered by api:task", function(done) {
-        request
-            .get("/o?app_id=" + APP_ID + "&api_key=" + API_KEY_ADMIN + "&method=jobs&name=api:task")
-            .expect(200)
-            .end(function(err, res) {
-                if (err) {
-                    return done(err);
-                }
-                var ob = JSON.parse(res.text);
-                var iTotalRecords = ob.iTotalRecords;
-                var iTotalDisplayRecords = ob.iTotalDisplayRecords;
-                if (iTotalDisplayRecords > 0 && iTotalRecords > 0) {
-                    ob.should.have.property("aaData");
-                    for (var z = 0; z < ob.aaData.length; z++) {
-                        if (ob.aaData[z].name !== "api:task") {
-                            done("Invalid task name. All should be api:task");
-                            return;
-                        }
-                    }
-                    done();
-                }
-                else {
-                    done("There are no records to show");
-                }
-            });
-    });
+    // it("Fetching jobs table data", function(done) {
+    //     API_KEY_ADMIN = testUtils.get("API_KEY_ADMIN");
+    //     APP_ID = testUtils.get("APP_ID");
+    //     request
+    //         .get("/o?app_id=" + APP_ID + "&api_key=" + API_KEY_ADMIN + "&method=jobs")
+    //         .expect(200)
+    //         .end(function(err, res) {
+    //             if (err) {
+    //                 return done(err);
+    //             }
+    //             var ob = JSON.parse(res.text);
+    //             var iTotalRecords = ob.iTotalRecords;
+    //             var iTotalDisplayRecords = ob.iTotalDisplayRecords;
+    //             if (iTotalDisplayRecords > 0 && iTotalRecords > 0) {
+    //                 ob.should.have.property("aaData");
+    //                 done();
+    //             }
+    //             else {
+    //                 done("There are no records to show");
+    //             }
+    //         });
+    // });
+    //
+    // it("Fetching jobs table data filtered by api:task", function(done) {
+    //     request
+    //         .get("/o?app_id=" + APP_ID + "&api_key=" + API_KEY_ADMIN + "&method=jobs&name=api:task")
+    //         .expect(200)
+    //         .end(function(err, res) {
+    //             if (err) {
+    //                 return done(err);
+    //             }
+    //             var ob = JSON.parse(res.text);
+    //             var iTotalRecords = ob.iTotalRecords;
+    //             var iTotalDisplayRecords = ob.iTotalDisplayRecords;
+    //             if (iTotalDisplayRecords > 0 && iTotalRecords > 0) {
+    //                 ob.should.have.property("aaData");
+    //                 for (var z = 0; z < ob.aaData.length; z++) {
+    //                     if (ob.aaData[z].name !== "api:task") {
+    //                         done("Invalid task name. All should be api:task");
+    //                         return;
+    //                     }
+    //                 }
+    //                 done();
+    //             }
+    //             else {
+    //                 done("There are no records to show");
+    //             }
+    //         });
+    // });
 });
 
 // 'use strict';

From 2a882ac233c243317bcaa38691ee1a8045cee69f Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Thu, 30 Jan 2025 19:20:28 +0530
Subject: [PATCH 63/90] update alerts to new job server

---
 plugins/alerts/api/api.js                 | 279 ++++------------------
 plugins/alerts/api/jobs/AlertProcessor.js | 139 +++++++++++
 plugins/alerts/api/jobs/monitor.js        |  70 ------
 3 files changed, 185 insertions(+), 303 deletions(-)
 create mode 100644 plugins/alerts/api/jobs/AlertProcessor.js
 delete mode 100644 plugins/alerts/api/jobs/monitor.js

diff --git a/plugins/alerts/api/api.js b/plugins/alerts/api/api.js
index 900dc343c0d..741e17dce0c 100644
--- a/plugins/alerts/api/api.js
+++ b/plugins/alerts/api/api.js
@@ -2,7 +2,6 @@ const common = require('../../../api/utils/common.js');
 const plugins = require('../../pluginManager.js');
 const log = require('../../../api/utils/log.js')('alert:api');
 var Promise = require("bluebird");
-const JOB = require('../../../api/parts/jobs');
 const utils = require('./parts/utils');
 const _ = require('lodash');
 const { validateCreate, validateRead, validateUpdate } = require('../../../api/utils/rights.js');
@@ -23,67 +22,14 @@ const TRIGGER_BY_EVENT = Object.keys(commonLib.TRIGGERED_BY_EVENT).map(name => (
     name
 }));
 
-// FIX THIS: workaround for the job.schedule
-const _date = new Date("2024-03-25T23:59:00.000Z");
-const _timeDelta = _date.getTimezoneOffset() / 60;
-const _hours = String((23 + _timeDelta) % 24).padStart(2, "0");
-
-const PERIOD_TO_TEXT_EXPRESSION_MAPPER = {
-    "hourly": "every 1 hour on the 59th min",
-    "daily": "at " + _hours + ":59",
-    "monthly": "on the last day of the month at " + _hours + ":59",
-};
-
 (function() {
     /**
-	 * delete alert job
-	 * @param {string} alertID  - alert record id from db
-	 * @param {function} callback - callback after deleting
-	 */
-    function deleteJob(alertID, callback) {
-        if (typeof alertID === 'string') {
-            alertID = common.db.ObjectID(alertID);
-        }
-        common.db.collection("jobs").remove({ 'data.alertID': alertID }, function(err) {
-            if (err) {
-                log.e('delete job failed, alertID:', alertID, err);
-                return;
-            }
-            log.d('delete job, alertID:', alertID);
-            if (callback) {
-                callback();
-            }
-        });
-    }
-    /**
-	 * update alert job
-	 * @param {object} alert  - alert record data
-	 */
-    function updateJobForAlert(alert) {
-        if (alert.enabled) {
-            const textExpression = PERIOD_TO_TEXT_EXPRESSION_MAPPER[alert.period];
-            if (textExpression) {
-                JOB.job('alerts:monitor', { alertID: alert._id }).replace().schedule(textExpression);
-                // JOB.job('alerts:monitor', { alertID: alert._id }).replace().schedule("every seconds");
-            }
-        }
-        else {
-            deleteJob(alert._id);
-        }
-    }
-    /**
-	 * load job list
-	 */
-    function loadJobs() {
-        common.readBatcher.getMany("alerts", {}, function(err, alertsList) {
-            log.d(alertsList, "get alert configs");
-            alertsList && alertsList.forEach(t => {
-                //period type
-                if (t.period) {
-                    updateJobForAlert(t);
-                }
-            });
-        });
+     * Notify the alert processor about changes in alerts configuration
+     * This will be picked up by the alert processor on its next run
+     */
+    function notifyAlertProcessor() {
+        // Invalidate the alerts cache so processor picks up changes
+        common.readBatcher.invalidate("alerts", {}, {}, true);
     }
 
     plugins.register("/i", async function(ob) {
@@ -123,70 +69,15 @@ const PERIOD_TO_TEXT_EXPRESSION_MAPPER = {
         ob.features.push(FEATURE_NAME);
     });
 
-    plugins.register("/master", function() {
-        loadJobs();
-    });
-
-    plugins.register("/updateAlert", function(ob) {
-        setTimeout(() => {
-            if (ob && (ob.method === "alertTrigger")) {
-                if (ob.alert) {
-                    deleteJob(ob.alert, function() {
-                        updateJobForAlert(ob.alert);
-                    });
-                }
-                else {
-                    loadJobs();
-                }
-            }
-        }, 2000);
-    });
-
-
     plugins.register("/alerts/addAlertCount", function(ob) {
         log.d("/alerts/addAlertCount", ob);
         utils.addAlertCount(ob);
     });
 
-    setTimeout(function() {
-        plugins.dispatch("/updateAlert", { method: "alertTrigger" });
-    }, 10000);
-
-
-
     /**
-     * @api {get} /i/alert/save save new create or updated alert data. 
-     * @apiName  saveAlert 
-     * @apiGroup alerts 
-     *
-     * @apiDescription  create or update alert. 
-     * @apiQuery {string} alert_config alert Configuration JSON object string. 
-     *  if contains "_id" will update related alert in DB.
-     * @apiQuery {String} app_id target app id of the alert.  
-     *
-     * @apiSuccessExample {json} Success-Response:
-     * HTTP/1.1 200 OK
-     *
-     * {
-          "_id": "626270afbf7392a8bfd8c1f3",
-          "alertName": "test",
-          "alertDataType": "metric",
-          "alertDataSubType": "Total users",
-          "alertDataSubType2": null,
-          "compareType": "increased by at least",
-          "compareValue": "2",
-          "selectedApps": [
-            "60a94dce686d3eea363ac325"
-          ],
-          "period": "every 1 hour on the 59th min",
-          "alertBy": "email",
-          "enabled": true,
-          "compareDescribe": "Total users increased by at least 2%",
-          "alertValues": [
-            "a@abc.com"
-          ],
-          "createdBy": "60afbaa84723f369db477fee"
-        }
+     * @api {get} /i/alert/save save new create or updated alert data.
+     * @apiName  saveAlert
+     * @apiGroup alerts
      */
     plugins.register("/i/alert/save", function(ob) {
         let params = ob.params;
@@ -196,17 +87,16 @@ const PERIOD_TO_TEXT_EXPRESSION_MAPPER = {
             try {
                 alertConfig = JSON.parse(alertConfig);
                 var checkProps = {
-                    'alertName': { 'required': alertConfig._id ? false : true, 'type': 'String', 'min-length': 1 },
-                    'alertDataType': { 'required': alertConfig._id ? false : true, 'type': 'String', 'min-length': 1 },
-                    'alertDataSubType': { 'required': alertConfig._id ? false : true, 'type': 'String', 'min-length': 1 },
-                    // 'period': { 'required': alertConfig._id ? false : true, 'type': 'String', 'min-length': 1 },
-                    'selectedApps': { 'required': alertConfig._id ? false : true, 'type': 'Array', 'min-length': 1 }
-
+                    'alertName': { 'required': !alertConfig._id, 'type': 'String', 'min-length': 1 },
+                    'alertDataType': { 'required': !alertConfig._id, 'type': 'String', 'min-length': 1 },
+                    'alertDataSubType': { 'required': !alertConfig._id, 'type': 'String', 'min-length': 1 },
+                    'selectedApps': { 'required': !alertConfig._id, 'type': 'Array', 'min-length': 1 }
                 };
                 if (!(common.validateArgs(alertConfig, checkProps))) {
                     common.returnMessage(params, 200, 'Not enough args');
                     return true;
                 }
+
                 if (alertConfig._id) {
                     const id = alertConfig._id;
                     delete alertConfig._id;
@@ -217,11 +107,7 @@ const PERIOD_TO_TEXT_EXPRESSION_MAPPER = {
                         {$set: alertConfig},
                         function(err, result) {
                             if (!err) {
-                                if (result && result.value) {
-                                    plugins.dispatch("/updateAlert", { method: "alertTrigger", alert: result.value });
-                                }
-                                plugins.dispatch("/updateAlert", { method: "alertTrigger" });
-
+                                notifyAlertProcessor();
                                 common.returnOutput(params, result && result.value);
                             }
                             else {
@@ -229,16 +115,15 @@ const PERIOD_TO_TEXT_EXPRESSION_MAPPER = {
                             }
                         });
                 }
-                if (!alertConfig._id) {
-                    alertConfig.createdAt = new Date().getTime();
-                }
+
+                alertConfig.createdAt = new Date().getTime();
                 alertConfig.createdBy = params.member._id;
                 return common.db.collection("alerts").insert(
                     alertConfig,
                     function(err, result) {
                         log.d("insert new alert:", err, result);
                         if (!err && result && result.insertedIds && result.insertedIds[0]) {
-                            plugins.dispatch("/updateAlert", { method: "alertTrigger", alert: result.ops[0] });
+                            notifyAlertProcessor();
                             common.returnOutput(params, result.insertedIds[0]);
                         }
                         else {
@@ -255,24 +140,11 @@ const PERIOD_TO_TEXT_EXPRESSION_MAPPER = {
         return true;
     });
 
-
-
-
     /**
-     * @api {get} /i/alert/delete delete alert by alert ID 
-     * @apiName deleteAlert 
-     * @apiGroup alerts 
-     *
-     * @apiDescription delete alert by id.
-     * @apiQuery {string} alertID  target alert id from db.
-     * @apiQuery {String} app_id target app id of the alert.  
-     *
-     * @apiSuccessExample {json} Success-Response:
-     * HTTP/1.1 200 OK
-     *
-     * {"result":"Deleted an alert"}
-     *
-    */
+     * @api {get} /i/alert/delete delete alert by alert ID
+     * @apiName deleteAlert
+     * @apiGroup alerts
+     */
     plugins.register("/i/alert/delete", function(ob) {
         let params = ob.params;
 
@@ -284,7 +156,7 @@ const PERIOD_TO_TEXT_EXPRESSION_MAPPER = {
                     function(err, result) {
                         log.d(err, result, "delete an alert");
                         if (!err) {
-                            deleteJob(alertID);
+                            notifyAlertProcessor();
                             common.returnMessage(params, 200, "Deleted an alert");
                         }
                     }
@@ -300,40 +172,28 @@ const PERIOD_TO_TEXT_EXPRESSION_MAPPER = {
 
     /**
      * @api {post} /i/alert/status change alert status
-     * @apiName changeAlertStatus 
-     * @apiGroup alerts 
-     *
-     * @apiDescription change alerts status by boolean flag.
-     * @apiQuery {string} JSON string of status object for alerts record want to update.
-     *  for example: {"626270afbf7392a8bfd8c1f3":false, "42dafbf7392a8bfd8c1e1": true}
-     * @apiQuery {String} app_id target app id of the alert.  
-     *
-     * @apiSuccessExample {text} Success-Response:
-     * HTTP/1.1 200 OK
-     *
-     * true
-     *
-    */
+     * @apiName changeAlertStatus
+     * @apiGroup alerts
+     */
     plugins.register("/i/alert/status", function(ob) {
         let params = ob.params;
 
         validateUpdate(params, FEATURE_NAME, function() {
             const statusList = JSON.parse(params.qstring.status);
             const batch = [];
-            for (const appID in statusList) {
+            for (const alertID in statusList) {
                 batch.push(
                     common.db.collection("alerts").findAndModify(
-                        { _id: common.db.ObjectID(appID) },
+                        { _id: common.db.ObjectID(alertID) },
                         {},
-                        { $set: { enabled: statusList[appID] } },
+                        { $set: { enabled: statusList[alertID] } },
                         { new: false, upsert: false }
                     )
                 );
             }
             Promise.all(batch).then(function() {
-                log.d("alert all updated.");
-                common.readBatcher.invalidate("alerts", {}, {}, true);
-                plugins.dispatch("/updateAlert", { method: "alertTrigger" });
+                log.d("alert statuses updated");
+                notifyAlertProcessor();
                 common.returnOutput(params, true);
             });
         });
@@ -341,52 +201,10 @@ const PERIOD_TO_TEXT_EXPRESSION_MAPPER = {
     });
 
     /**
-     * @api {post} /i/alert/list get alert list 
-     * @apiName getAlertList 
-     * @apiGroup alerts 
-     *
-     * @apiDescription get Alert List user can view. 
-     *
-     * @apiQuery {String} app_id target app id of the alert.  
-     *
-     * @apiSuccessExample {json} Success-Response:
-     * HTTP/1.1 200 OK
-     *
-     * {
-          "alertsList": [
-            {
-              "_id": "626270afbf7392a8bfd8c1f3",
-              "alertName": "test",
-              "alertDataType": "metric",
-              "alertDataSubType": "Total users",
-              "alertDataSubType2": null,
-              "compareType": "increased by at least",
-              "compareValue": "2",
-              "selectedApps": [
-                "60a94dce686d3eea363ac325"
-              ],
-              "period": "every 1 hour on the 59th min",
-              "alertBy": "email",
-              "enabled": false,
-              "compareDescribe": "Total users increased by at least 2%",
-              "alertValues": [
-                "a@abc.com"
-              ],
-              "createdBy": "60afbaa84723f369db477fee",
-              "appNameList": "Mobile Test",
-              "app_id": "60a94dce686d3eea363ac325",
-              "condtionText": "Total users increased by at least 2%",
-              "createdByUser": "abc",
-              "type": "Total users"
-            }
-          ],
-          "count": {
-            "r": 0
-          }
-        }
-     *
-     *
-    */
+     * @api {post} /o/alert/list get alert list
+     * @apiName getAlertList
+     * @apiGroup alerts
+     */
     plugins.register("/o/alert/list", function(ob) {
         const params = ob.params;
         validateRead(params, FEATURE_NAME, function() {
@@ -426,33 +244,28 @@ const PERIOD_TO_TEXT_EXPRESSION_MAPPER = {
     });
 
     /**
-	 * remove app related alerts record and  alert job records;
-	 * @param {string} appId  - app id
-	 */
+     * remove app related alerts
+     * @param {string} appId - app id
+     */
     function removeAlertsForApp(appId) {
         common.db.collection('alerts').find({selectedApps: {$all: [appId]}}).toArray(function(err, result) {
-            if (!err) {
-                const ids = result.map((record)=>{
-                    return record._id;
-                }) || [];
-                common.db.collection('alerts').remove({selectedApps: {$all: [appId]}}, function() {});
-                common.db.collection('jobs').remove({'data.alertID': {$in: ids}}, function() {});
+            if (!err && result.length > 0) {
+                common.db.collection('alerts').remove({selectedApps: {$all: [appId]}}, function() {
+                    notifyAlertProcessor();
+                });
             }
         });
     }
 
     plugins.register("/i/apps/delete", function(ob) {
-        var appId = ob.appId;
-        removeAlertsForApp(appId);
+        removeAlertsForApp(ob.appId);
     });
 
     plugins.register("/i/apps/clear_all", function(ob) {
-        var appId = ob.appId;
-        removeAlertsForApp(appId);
+        removeAlertsForApp(ob.appId);
     });
 
     plugins.register("/i/apps/reset", function(ob) {
-        var appId = ob.appId;
-        removeAlertsForApp(appId);
+        removeAlertsForApp(ob.appId);
     });
-}());
+}());
\ No newline at end of file
diff --git a/plugins/alerts/api/jobs/AlertProcessor.js b/plugins/alerts/api/jobs/AlertProcessor.js
new file mode 100644
index 00000000000..79d0f53af6c
--- /dev/null
+++ b/plugins/alerts/api/jobs/AlertProcessor.js
@@ -0,0 +1,139 @@
+const Job = require('../../../../jobServer/Job');
+const common = require('../../../../api/utils/common.js');
+const { TRIGGERED_BY_EVENT } = require('../parts/common-lib.js');
+const ALERT_MODULES = {
+    "views": require("../alertModules/views.js"),
+    "users": require("../alertModules/users.js"),
+    "sessions": require("../alertModules/sessions.js"),
+    "survey": require("../alertModules/survey.js"),
+    "nps": require("../alertModules/nps.js"),
+    "revenue": require("../alertModules/revenue.js"),
+    "events": require("../alertModules/events.js"),
+    "rating": require("../alertModules/rating.js"),
+    "cohorts": require("../alertModules/cohorts.js"),
+    "dataPoints": require("../alertModules/dataPoints.js"),
+    "crashes": require("../alertModules/crashes.js"),
+};
+
+/**
+ * Alert Processor Job
+ */
+class AlertProcessor extends Job {
+
+    /**
+     * Run the alert processor job
+     * @returns {GetScheduleConfig} Schedule configuration object
+     */
+    getSchedule() {
+        return {
+            type: 'schedule',
+            period: "0 * * * *" // Every hour
+        };
+    }
+
+    /**
+     * @param {Db} db The database connection
+     * @param {Function} done Callback function to be called when the job is complete
+     *                       Call with error as first parameter if job fails
+     *                       Call with null and optional result as second parameter if job succeeds
+     */
+    async run(db, done) {
+        try {
+            const currentTime = new Date();
+            this.log.d('Starting alerts processing at:', currentTime);
+
+            // Determine which type of alerts to process based on current time
+            const minute = currentTime.getMinutes();
+            const hour = currentTime.getHours();
+            const isLastDayOfMonth = currentTime.getDate() === new Date(currentTime.getFullYear(), currentTime.getMonth() + 1, 0).getDate();
+
+            // Build query conditions
+            const timeConditions = [];
+
+            // Check for hourly alerts (run at minute 59)
+            if (minute === 59) {
+                timeConditions.push({ period: 'hourly' });
+            }
+
+            // Check for daily alerts (run at 23:59)
+            if (hour === 23 && minute === 59) {
+                timeConditions.push({ period: 'daily' });
+            }
+
+            // Check for monthly alerts (run at 23:59 on last day of month)
+            if (isLastDayOfMonth && hour === 23 && minute === 59) {
+                timeConditions.push({ period: 'monthly' });
+            }
+
+            // If no alerts should run at this time, exit early
+            if (timeConditions.length === 0) {
+                this.log.d('No alerts scheduled for current time');
+                return done(null, 'No alerts scheduled for current time');
+            }
+
+            // Get all enabled alerts that match the time conditions
+            const alerts = await common.db.collection("alerts").find({
+                $and: [
+                    { enabled: true },
+                    { alertDataSubType: { $nin: Object.values(TRIGGERED_BY_EVENT) } },
+                    { $or: timeConditions }
+                ]
+            }).toArray();
+
+            this.log.d('Found alerts to process:', alerts.length);
+
+            // Process each alert
+            const results = await Promise.allSettled(alerts.map(async(alert) => {
+                try {
+                    // Handle profile_groups to cohorts mapping
+                    if (alert.alertDataType === 'profile_groups') {
+                        alert.alertDataType = 'cohorts';
+                    }
+
+                    const module = ALERT_MODULES[alert.alertDataType];
+                    if (!module) {
+                        throw new Error(`Alert module not found: ${alert.alertDataType}`);
+                    }
+
+                    // Process the alert
+                    await module.check({
+                        alertConfigs: alert,
+                        scheduledTo: currentTime,
+                    });
+
+                    this.log.d('Successfully processed alert:', alert._id);
+                    return { alertId: alert._id, success: true };
+                }
+                catch (err) {
+                    this.log.e('Error processing alert:', alert._id, err);
+                    return { alertId: alert._id, success: false, error: err.message };
+                }
+            }));
+
+            // Log summary of processing results
+            const successful = results.filter(r => r.status === 'fulfilled');
+            const failed = results.filter(r => r.status === 'rejected');
+
+            this.log.d('Alerts processing completed:', {
+                total: alerts.length,
+                successful: successful.length,
+                failed: failed.length
+            });
+
+            // If any alerts failed, log details
+            if (failed.length > 0) {
+                this.log.e('Failed alerts:', failed.map(f => f.reason));
+                const errors = failed.map(f => f.reason).join('\n');
+                return done(new Error(`Failed to process alerts:\n${errors}`));
+            }
+            return done();
+        }
+        catch (err) {
+            this.log.e('Fatal error in alerts processing:', err);
+            done(err);
+        }
+
+    }
+}
+
+module.exports = AlertProcessor;
\ No newline at end of file
diff --git a/plugins/alerts/api/jobs/monitor.js b/plugins/alerts/api/jobs/monitor.js
deleted file mode 100644
index debe90c19c8..00000000000
--- a/plugins/alerts/api/jobs/monitor.js
+++ /dev/null
@@ -1,70 +0,0 @@
-'use strict';
-
-const { TRIGGERED_BY_EVENT } = require('../parts/common-lib.js');
-
-const { Job } = require('../../../../api/parts/jobs/job.js'),
-    log = require('../../../../api/utils/log.js')('alert:monitor'),
-    common = require('../../../../api/utils/common.js');
-
-const ALERT_MODULES = {
-    "views": require("../alertModules/views.js"),
-    "users": require("../alertModules/users.js"),
-    "sessions": require("../alertModules/sessions.js"),
-    "survey": require("../alertModules/survey.js"),
-    "nps": require("../alertModules/nps.js"),
-    "revenue": require("../alertModules/revenue.js"),
-    "events": require("../alertModules/events.js"),
-    "rating": require("../alertModules/rating.js"),
-    "cohorts": require("../alertModules/cohorts.js"),
-    "dataPoints": require("../alertModules/dataPoints.js"),
-    "crashes": require("../alertModules/crashes.js"),
-};
-/**
- * @class
- * @classdesc Class MonitorJob is Alert Monitor Job extend from Countly Job
- * @extends Job
- */
-class MonitorJob extends Job {
-    /**
-     * run task
-     * @param {object} _db - db object
-     * @param {function} done - callback function
-     */
-    run(_db, done) {
-        const alertID = this._json.data.alertID;
-        const scheduledTo = this._json.next;
-        const self = this;
-        common.db.collection("alerts").findOne({
-            _id: common.db.ObjectID(alertID),
-            // these are being triggered by the event listener in api.js
-            alertDataSubType: { $nin: Object.values(TRIGGERED_BY_EVENT) }
-        }, async function(err, alertConfigs) {
-            if (err) {
-                log.e(err);
-                return;
-            }
-            if (!alertConfigs) {
-                return;
-            }
-            log.d('Runing alerts Monitor Job ....');
-            log.d("job info:", self._json, alertConfigs);
-            if (alertConfigs.alertDataType === 'profile_groups') {
-                alertConfigs.alertDataType = 'cohorts';
-            }
-            const module = ALERT_MODULES[alertConfigs.alertDataType];
-            if (module) {
-                try {
-                    await module.check({ alertConfigs, done, scheduledTo });
-                }
-                catch (error) {
-                    log.e("Error while running " + alertConfigs.alertDataType + " alert check", error);
-                }
-            }
-            else {
-                log.e("Alert module " + alertConfigs.alertDataType + " not found");
-            }
-        });
-    }
-}
-
-module.exports = MonitorJob;

From 4b83b7506d73c19fb036ae8bba70d475ab6c9742 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Thu, 30 Jan 2025 19:39:37 +0530
Subject: [PATCH 64/90] clean

---
 plugins/hooks/api/parts/triggers/scheduled.js | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/plugins/hooks/api/parts/triggers/scheduled.js b/plugins/hooks/api/parts/triggers/scheduled.js
index b40e2f948f9..7d1230740ef 100644
--- a/plugins/hooks/api/parts/triggers/scheduled.js
+++ b/plugins/hooks/api/parts/triggers/scheduled.js
@@ -97,12 +97,4 @@ class ScheduledTrigger {
     }
 }
 
-
-// plugins.register("/master", function() {
-//     setTimeout(() => {
-//         JOB.job('hooks:schedule', {type: 'ScheduledTrigger'}).replace().schedule("every 1 hour on the 1st min");
-//     }, 10000);
-// });
-
-
 module.exports = ScheduledTrigger;

From a66d8510043ee65041e4405cbbd59cb5ca8874ff Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Fri, 31 Jan 2025 12:32:30 +0530
Subject: [PATCH 65/90] add dev deps

---
 package.json | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/package.json b/package.json
index 91a007c41b5..0ffeee5e6c0 100644
--- a/package.json
+++ b/package.json
@@ -16,6 +16,15 @@
   "engines": {
     "node": "^20.0.0 || ^22.0.0"
   },
+  "scripts": {
+    "test": "grunt --verbose",
+    "prepare": "husky"
+  },
+  "lint-staged": {
+    "*.{js,vue}": [
+      "eslint --fix"
+    ]
+  },
   "devDependencies": {
     "@stylistic/eslint-plugin": "^2.11.0",
     "docdash": "^2.0.1",
@@ -24,15 +33,14 @@
     "grunt-eslint": "24.3.0",
     "grunt-mocha-nyc": "1.0.3",
     "grunt-mocha-test": "0.13.3",
+    "husky": "^9.1.7",
     "jsdoc": "^4.0.0",
+    "lint-staged": "^15.4.3",
     "mocha": "10.8.2",
     "nyc": "17.1.0",
     "should": "13.2.3",
     "supertest": "7.0.0"
   },
-  "scripts": {
-    "test": "grunt --verbose"
-  },
   "dependencies": {
     "@breejs/later": "^4.2.0",
     "@pulsecron/pulse": "1.6.7",

From 8f38f9ca95a7df43b96fe310b51190d2c04dcdbf Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Fri, 31 Jan 2025 13:08:36 +0530
Subject: [PATCH 66/90] fix alertProcessor job schedule

---
 plugins/alerts/api/jobs/AlertProcessor.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/alerts/api/jobs/AlertProcessor.js b/plugins/alerts/api/jobs/AlertProcessor.js
index 79d0f53af6c..a413db067a9 100644
--- a/plugins/alerts/api/jobs/AlertProcessor.js
+++ b/plugins/alerts/api/jobs/AlertProcessor.js
@@ -27,7 +27,7 @@ class AlertProcessor extends Job {
     getSchedule() {
         return {
             type: 'schedule',
-            period: "0 * * * *" // Every hour
+            value: "0 * * * *" // Every hour
         };
     }
 

From 9038da436522676cb3ad1c0092f1fc1c65b1be99 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Fri, 31 Jan 2025 23:41:39 +0530
Subject: [PATCH 67/90] remove tests for old jobs

---
 api/jobs/ipcTest.js | 128 -------------------------------------------
 api/jobs/test.js    | 129 --------------------------------------------
 2 files changed, 257 deletions(-)
 delete mode 100644 api/jobs/ipcTest.js
 delete mode 100644 api/jobs/test.js

diff --git a/api/jobs/ipcTest.js b/api/jobs/ipcTest.js
deleted file mode 100644
index e7e63d39bd6..00000000000
--- a/api/jobs/ipcTest.js
+++ /dev/null
@@ -1,128 +0,0 @@
-'use strict';
-
-const job = require('../parts/jobs/job.js'),
-    res = require('../parts/jobs/resource.js'),
-    log = require('../utils/log.js')('job:ipcTest');
-
-/** Class for testing resource handling for jobs **/
-class TestResource extends res.Resource {
-    /** 
-    * Open resource 
-    * @returns {Promise} promise
-    **/
-    open() {
-        this.a = 0;
-        return new Promise((resolve) => {
-            log.d('open');
-            setTimeout(() => {
-                this.opened();
-                resolve();
-            }, 2000);
-        });
-    }
-
-    /** 
-    * Close resource 
-    * @returns {Promise} promise
-    **/
-    close() {
-        return new Promise((resolve) => {
-            log.d('close');
-            setTimeout(() => {
-                this.closed();
-                resolve();
-            }, 2000);
-        });
-    }
-
-    /** 
-    * Check if resource is used 
-    * @returns {Promise} promise
-    **/
-    checkActive() {
-        return new Promise((resolve) => {
-            log.d('checkActive');
-            setTimeout(() => {
-                resolve(this.a++ > 0 ? false : true);
-            }, 2000);
-        });
-    }
-
-}
-/** Class for testing ipc jobs **/
-class Test extends job.IPCJob {
-    /** 
-    * Create resource 
-    * @returns {Resource} resourse
-    **/
-    createResource() {
-        return new TestResource();
-    }
-
-    /** 
-    * Create between processes 
-    * @returns {Promise} promise
-    **/
-    divide() {
-        return new Promise((resolve) => {
-            log.d('dividing ', this._json);
-
-            setTimeout(() => {
-                resolve([{
-                    smth: 1,
-                    size: 40
-                }], [{
-                    smth: 1,
-                    size: 50
-                }]);
-            }, 500);
-        });
-    }
-
-    /**
-     * Run the job
-     * @param {Db} db connection
-     * @param {done} done callback
-     * @param {function} progress to report progress of the job
-     */
-    run(db, done, progress) {
-        log.d('running ', this._json);
-        log.d('resource is ', typeof this.resource);
-        log.d('resource is open? ', this.resource.isOpen);
-        log.d('resource is active? ', this.resource.isAssigned);
-
-        if (this.done < 10) {
-            setTimeout(() => {
-                progress(this._json.size, 10, 'ten');
-            }, 1000);
-        }
-
-        if (this.done < 20) {
-            setTimeout(() => {
-                progress(this._json.size, 20, 'twenty');
-                // a = b;
-            }, 5000);
-        }
-
-        if (this.done < 30) {
-            setTimeout(() => {
-                progress(this._json.size, 30, 'thirty');
-            }, 6000);
-        }
-
-        setTimeout(() => {
-            progress(100, 100, 'sixty');
-            done();
-        }, 60000);
-
-        setTimeout(() => {
-            log.d('after done resource is ', typeof this.resource);
-            done('Big fat error');
-            db.collection('jobs').findOne({_id: this._json._id}, (err, obj) => {
-                log.d('after done job findOne ', err, obj);
-            });
-        }, 120000);
-    }
-}
-
-module.exports = Test;
\ No newline at end of file
diff --git a/api/jobs/test.js b/api/jobs/test.js
deleted file mode 100644
index b25d3825e99..00000000000
--- a/api/jobs/test.js
+++ /dev/null
@@ -1,129 +0,0 @@
-'use strict';
-
-/* jshint ignore:start */
-
-const J = require('../parts/jobs/job.js'),
-    R = require('../parts/jobs/resource.js'),
-    RET = require('../parts/jobs/retry.js');
-
-/** Class for testing resource handling for jobs **/
-class TestResource extends R.Resource {
-    /** 
-    * Open resource 
-    * @returns {Promise} promise
-    **/
-    open() {
-        console.log('resource: opening in %d', process.pid);
-        this.opened();
-        this.openedTime = Date.now();
-        return Promise.resolve();
-    }
-
-    /** 
-    * Close resource 
-    * @returns {Promise} promise
-    **/
-    close() {
-        console.log('resource: closing in %d', process.pid);
-        this.closed();
-        return Promise.resolve();
-    }
-
-    /** 
-    * Kill resource 
-    * @returns {Promise} promise
-    **/
-    kill() {
-        console.log('resource: killed in %d', process.pid);
-        return Promise.resolve();
-    }
-
-    /** 
-    * Check if resource is used 
-    * @returns {Promise} promise
-    **/
-    checkActive() {
-        console.log('resource: checkActive in %d', process.pid);
-        return Promise.resolve(Date.now() - this.openedTime < 20000);
-    }
-
-    /** Start using resource **/
-    start() {
-        this.openedTime = Date.now();
-        super.start.apply(this, arguments);
-    }
-}
-/** Class for testing ipc jobs **/
-class IPCTestJob extends J.IPCJob {
-    /**
-     * Prepare the job
-     * @param {object} manager - resource manager
-     * @param {Db} db - db connection
-     */
-    async prepare(manager, db) {
-        console.log('preparing in %d', process.pid);
-        await new Promise((res, rej) => db.collection('jobs').updateOne({_id: this._id}, {$set: {'data.prepared': 1}}, err => err ? rej(err) : res()));
-    }
-
-    /** 
-    * Get resource name 
-    * @returns {string} resource name
-    **/
-    resourceName() {
-        return 'resource:test';
-    }
-
-    /**
-     * Create resource
-     * @param {string} _id - resource _id
-     * @param {string} name - resource name
-     * @param {Db} db - db connection
-     * @returns {Resource} resource
-     */
-    createResource(_id, name, db) {
-        return new TestResource(_id, name, db);
-    }
-
-    /** 
-    * Get retry policy 
-    * @returns {RetryPolicy} retry policy
-    **/
-    retryPolicy() {
-        return new RET.NoRetryPolicy();
-    }
-
-    /** 
-    * Get concurrency 
-    * @returns {number} concurency
-    **/
-    getConcurrency() {
-        return this.data && this.data.concurrency || 0;
-    }
-
-    /**
-     * Run the job
-     * @param {Db} db connection
-     */
-    async run(db) {
-        console.log('running in %d', process.pid);
-        if (!this.resource) {
-            throw new Error('Resource should exist');
-        }
-        if (!(this.resource instanceof TestResource)) {
-            throw new Error('Resource should be TestResource');
-        }
-        await new Promise((res, rej) => db.collection('jobs').updateOne({_id: this._id}, {$set: {'data.run': 1}}, err => err ? rej(err) : res()));
-
-        if (this.data && this.data.fail) {
-            throw new Error(this.data.fail);
-        }
-
-        if (this.data && this.data.concurrency) {
-            await new Promise(res => setTimeout(res, 3000));
-        }
-
-        console.log('done running in %d', process.pid);
-    }
-}
-
-module.exports = IPCTestJob;
\ No newline at end of file

From a291aaefb0bd059128ad23bb6510b9a43a22d8b9 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Fri, 31 Jan 2025 23:43:06 +0530
Subject: [PATCH 68/90] remove cluster from system-utility

---
 plugins/system-utility/api/api.js | 63 ++++++-------------------------
 1 file changed, 12 insertions(+), 51 deletions(-)

diff --git a/plugins/system-utility/api/api.js b/plugins/system-utility/api/api.js
index 6711909080f..8a6cf6a2b29 100644
--- a/plugins/system-utility/api/api.js
+++ b/plugins/system-utility/api/api.js
@@ -3,12 +3,10 @@ var plugin = {},
     tracker = require('../../../api/parts/mgmt/tracker.js'),
     plugins = require('../../pluginManager.js'),
     systemUtility = require('./system.utility'),
-    log = common.log('system-utility:api'),
-    cluster = require("cluster");
+    log = common.log('system-utility:api');
 
-const processName = (cluster.isMaster ? "master" : "worker") + "-" + process.pid;
+const processName = "master-" + process.pid;
 const profilerCmds = ["startProfiler", "stopProfiler", "startInspector", "stopInspector"];
-let numberOfWorkers;
 
 /**
  * Checks if the message is sent from profiler/inspector endpoints
@@ -20,14 +18,13 @@ function isInspectorMessage(msg) {
 }
 
 /**
- * Handles IPC messages sent by profiler and inspector endpoints.
- * @param {object} msg should contain at least "cmd" and "msgId" key
+ * Handles messages sent by profiler and inspector endpoints.
+ * @param {object} msg should contain at least "cmd" key
  */
 function handleMessage(msg) {
     if (isInspectorMessage(msg)) {
         let args = msg.args || [];
-        // each process will have their own processName. So we can't pass
-        // that from the main process:
+        // Set processName for profiler commands
         if (msg.cmd === "stopProfiler" || msg.cmd === "startProfiler") {
             args = [processName];
         }
@@ -37,67 +34,32 @@ function handleMessage(msg) {
             console.error(err);
         });
     }
-    else if (typeof msg === "object" && msg.cmd === "setNumberOfWorkers") {
-        numberOfWorkers = msg.params.numberOfWorkers;
-    }
 }
 
-// Handle messages broadcasted from master to worker.
-process.on("message", msg => handleMessage(msg));
-
-// Handle messages sent from worker to master.
-plugins.register("/master", () => {
-    const workers = Object.values(cluster.workers);
-
-    workers.forEach(worker => {
-        // set the numberOfWorkers variable on each worker
-        worker.on("listening", () => {
-            worker.send({
-                cmd: "setNumberOfWorkers",
-                params: { numberOfWorkers: workers.length }
-            });
-        });
-
-        // listen workers for inspector/profiler messages
-        worker.on("message", msg => {
-            if (isInspectorMessage(msg)) {
-                // handle on master
-                handleMessage(msg);
-
-                // broadcast to all workers except for "startInspector".
-                // running startInspector on master also starts worker's inspectors.
-                if (!["startInspector"].includes(msg.cmd)) {
-                    workers.forEach(_worker => _worker.send(msg));
-                }
-            }
-        });
-    });
-});
-
 // helper functions to start/stop with a timeout.
 let timeouts = { Profiler: null, Inspector: null };
 /**
- * Sends a "startInspector" or "startProfiler" message to main process.
- * Sets a timeout callback to stop the same operation.
+ * Starts Inspector or Profiler with a timeout
  * @param {string} type Inspector|Profiler
  */
 function startWithTimeout(type) {
     if (timeouts[type]) {
         throw new Error("Already started");
     }
-    process.send({ cmd: "start" + type });
+    handleMessage({ cmd: "start" + type });
     timeouts[type] = setTimeout(() => stopWithTimeout(type, true), 2 * 60 * 60 * 1000);
 }
+
 /**
- * Sends a "stopInspector" or "stopProfiler" message to main process.
+ * Stops Inspector or Profiler
  * @param {string} type Inspector|Profiler
- * @param {boolean} fromTimeout true if its being stoped because of the timeout
+ * @param {boolean} fromTimeout true if its being stopped because of the timeout
  */
 function stopWithTimeout(type, fromTimeout = false) {
     if (!timeouts[type]) {
         throw new Error(type + " needs to be started");
     }
-    process.send({ cmd: "stop" + type });
+    handleMessage({ cmd: "stop" + type });
     if (!fromTimeout) {
         clearTimeout(timeouts[type]);
     }
@@ -117,8 +79,7 @@ function stopWithTimeout(type, fromTimeout = false) {
                 try {
                     startWithTimeout("Inspector");
                     common.returnMessage(params, 200, {
-                        workers: numberOfWorkers,
-                        ports: [masterPort, masterPort + numberOfWorkers]
+                        ports: [masterPort]
                     });
                 }
                 catch (err) {

From a3ef55d8ff000b3e2ec0b5e2dbf4ecdbd8321df1 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Fri, 31 Jan 2025 23:58:38 +0530
Subject: [PATCH 69/90] small updates system-utility

---
 plugins/system-utility/api/system.utility.js | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/plugins/system-utility/api/system.utility.js b/plugins/system-utility/api/system.utility.js
index da6902503f4..d497210de7a 100644
--- a/plugins/system-utility/api/system.utility.js
+++ b/plugins/system-utility/api/system.utility.js
@@ -465,7 +465,7 @@ async function startProfiler() {
  *  - process-name.cpuprofile
  *  - process-name.heapprofile
  *  - process-name.coverage
- * @param {string} processName process or worker and process id
+ * @param {string} processName process id
  */
 async function stopProfiler(processName) {
     const errors = [];
@@ -590,9 +590,7 @@ async function profilerFilesTarStream() {
 }
 
 /**
- * Opens inspector. Running inspector.open on master process triggers workers to
- * open their inspector also. But this is not the case for closing the inspectors.
- * Each worker needs to be closed manually.
+ * Opens inspector. Running inspector.open on master process only.
  * @returns {void}
  */
 function startInspector() {
@@ -607,8 +605,7 @@ function startInspector() {
 }
 
 /**
- * Closes inspector. Running inspector.close on master doesn't trigger workers to
- * close their inspector. Each worker needs to be closed manually.
+ * Closes inspector for the main process.
  * @returns {void}
  */
 function stopInspector() {

From 6910c61e62f2c4f45386d747b044e6a215f2f1e1 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Sat, 1 Feb 2025 11:34:56 +0530
Subject: [PATCH 70/90] update Alerts to check hourly,daily,monthly queries
 more robustly

---
 plugins/alerts/api/jobs/AlertProcessor.js | 119 +++++++++++-----------
 1 file changed, 61 insertions(+), 58 deletions(-)

diff --git a/plugins/alerts/api/jobs/AlertProcessor.js b/plugins/alerts/api/jobs/AlertProcessor.js
index a413db067a9..2b31afff587 100644
--- a/plugins/alerts/api/jobs/AlertProcessor.js
+++ b/plugins/alerts/api/jobs/AlertProcessor.js
@@ -42,50 +42,53 @@ class AlertProcessor extends Job {
             const currentTime = new Date();
             this.log.d('Starting alerts processing at:', currentTime);
 
-            // Determine which type of alerts to process based on current time
-            const minute = currentTime.getMinutes();
-            const hour = currentTime.getHours();
-            const isLastDayOfMonth = currentTime.getDate() === new Date(currentTime.getFullYear(), currentTime.getMonth() + 1, 0).getDate();
-
-            // Build query conditions
-            const timeConditions = [];
-
-            // Check for hourly alerts (run at minute 59)
-            if (minute === 59) {
-                timeConditions.push({ period: 'hourly' });
-            }
-
-            // Check for daily alerts (run at 23:59)
-            if (hour === 23 && minute === 59) {
-                timeConditions.push({ period: 'daily' });
-            }
-
-            // Check for monthly alerts (run at 23:59 on last day of month)
-            if (isLastDayOfMonth && hour === 23 && minute === 59) {
-                timeConditions.push({ period: 'monthly' });
-            }
-
-            // If no alerts should run at this time, exit early
-            if (timeConditions.length === 0) {
-                this.log.d('No alerts scheduled for current time');
-                return done(null, 'No alerts scheduled for current time');
-            }
-
-            // Get all enabled alerts that match the time conditions
-            const alerts = await common.db.collection("alerts").find({
-                $and: [
-                    { enabled: true },
-                    { alertDataSubType: { $nin: Object.values(TRIGGERED_BY_EVENT) } },
-                    { $or: timeConditions }
+            // Calculate thresholds in JavaScript just before issuing the query
+            const hourThreshold = new Date(currentTime.getTime() - 60 * 60 * 1000);
+            const dayThreshold = new Date(currentTime.getTime() - 24 * 60 * 60 * 1000);
+            const monthThreshold = new Date(currentTime);
+            monthThreshold.setMonth(monthThreshold.getMonth() - 1);
+
+            // Build query conditions with pre-calculated date thresholds
+            const query = {
+                enabled: true,
+                alertDataSubType: { $nin: Object.values(TRIGGERED_BY_EVENT) },
+                $or: [
+                    {
+                        period: 'hourly',
+                        $or: [
+                            { lastRunTime: { $exists: false } },
+                            { lastRunTime: { $lte: hourThreshold } }
+                        ]
+                    },
+                    {
+                        period: 'daily',
+                        $or: [
+                            { lastRunTime: { $exists: false } },
+                            { lastRunTime: { $lte: dayThreshold } }
+                        ]
+                    },
+                    {
+                        period: 'monthly',
+                        $or: [
+                            { lastRunTime: { $exists: false } },
+                            { lastRunTime: { $lte: monthThreshold } }
+                        ]
+                    }
                 ]
-            }).toArray();
-
-            this.log.d('Found alerts to process:', alerts.length);
-
-            // Process each alert
-            const results = await Promise.allSettled(alerts.map(async(alert) => {
+            };
+
+            // Use a cursor to iterate through alerts one by one.
+            const cursor = common.db.collection("alerts").find(query);
+            let totalAlerts = 0;
+            let successfulCount = 0;
+            let failedCount = 0;
+            let failures = [];
+
+            while (await cursor.hasNext()) {
+                totalAlerts++;
+                const alert = await cursor.next();
                 try {
-                    // Handle profile_groups to cohorts mapping
+                    // Handle profile_groups mapping to cohorts
                     if (alert.alertDataType === 'profile_groups') {
                         alert.alertDataType = 'cohorts';
                     }
@@ -101,30 +104,31 @@ class AlertProcessor extends Job {
                         scheduledTo: currentTime,
                     });
 
+                    // Update lastRunTime in the alert document after successful processing
+                    await common.db.collection("alerts").updateOne(
+                        { _id: alert._id },
+                        { $set: { lastRunTime: currentTime } }
+                    );
+
                     this.log.d('Successfully processed alert:', alert._id);
-                    return { alertId: alert._id, success: true };
+                    successfulCount++;
                 }
                 catch (err) {
                     this.log.e('Error processing alert:', alert._id, err);
-                    return { alertId: alert._id, success: false, error: err.message };
+                    failedCount++;
+                    failures.push({ alertId: alert._id, error: err.message });
                 }
-            }));
-
-            // Log summary of processing results
-            const successful = results.filter(r => r.status === 'fulfilled');
-            const failed = results.filter(r => r.status === 'rejected');
+            }
 
             this.log.d('Alerts processing completed:', {
-                total: alerts.length,
-                successful: successful.length,
-                failed: failed.length
+                total: totalAlerts,
+                successful: successfulCount,
+                failed: failedCount
             });
 
-            // If any alerts failed, log details
-            if (failed.length > 0) {
-                this.log.e('Failed alerts:', failed.map(f => f.reason));
-                const errors = failed.map(f => f.reason).join('\n');
-                return done(new Error(`Failed to process alerts:\n${errors}`));
+            if (failedCount > 0) {
+                const errors = failures.map(f => `Alert ${f.alertId}: ${f.error}`).join('\n');
+                return done(new Error(`Failed to process some alerts:\n${errors}`));
             }
             return done();
         }
@@ -132,7 +136,6 @@ class AlertProcessor extends Job {
             this.log.e('Fatal error in alerts processing:', err);
             done(err);
         }
-
     }
 }
 

From 0db8ab6fe6d420a1747c502797a30cf0b2aa65b4 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Sat, 1 Feb 2025 23:03:19 +0530
Subject: [PATCH 71/90] allow default disabled state in jobs and Manual only
 jobs

---
 jobServer/Job.js                              | 99 +++++++++++--------
 jobServer/JobManager.js                       | 13 +--
 .../jobRunner/impl/pulse/PulseJobExecutor.js  |  9 +-
 .../jobRunner/impl/pulse/PulseJobScheduler.js | 15 ++-
 4 files changed, 84 insertions(+), 52 deletions(-)

diff --git a/jobServer/Job.js b/jobServer/Job.js
index 76f2271e617..bc5bafba8f0 100644
--- a/jobServer/Job.js
+++ b/jobServer/Job.js
@@ -20,7 +20,7 @@ const { JOB_PRIORITIES } = require('./constants/JobPriorities');
 
 /**
  * @typedef {Object} GetScheduleConfig
- * @property {('once'|'schedule'|'now')} type - Type of schedule
+ * @property {('once'|'schedule'|'now'|'manual')} type - Type of schedule
  * @property {(string|Date)} [value] - Schedule value (cron expression or Date)
  */
 
@@ -89,7 +89,7 @@ class Job {
     /** @type {Function|null} Progress method from job runner */
     #progressMethod = null;
 
-    /** @type {Object.<string, string>} Available job priorities */
+    /** @type {Object.<string, PriorityLevel>} Available job priorities */
     priorities = JOB_PRIORITIES;
 
     /**
@@ -157,6 +157,7 @@ class Job {
     /**
      * Get the schedule type and timing for the job.
      * This method must be implemented by the child class.
+     * @public
      * 
      * @returns {GetScheduleConfig} Schedule configuration object
      * @property {('once'|'schedule'|'now')} type - Type of schedule
@@ -198,8 +199,62 @@ class Job {
         throw new Error('getSchedule must be overridden');
     }
 
+    /**
+     * Get job retry configuration
+     * @public
+     * @typedef {Object} RetryConfig
+     * @property {boolean} enabled - Whether retries are enabled
+     * @property {number} attempts - Number of retry attempts
+     * @property {number} delay - Delay between retry attempts in milliseconds, delay is by default exponentially increasing after each attempt
+     * @returns {RetryConfig|null} Retry configuration or null for default
+     */
+    getRetryConfig() {
+        return {
+            enabled: false,
+            attempts: 0,
+            delay: 5 * 60 * 1000 // 5 minutes
+        };
+    }
+
+    /**
+     * Get job priority
+     * @public
+     * @returns {PriorityLevel} Priority level from JOB_PRIORITIES
+     */
+    getPriority() {
+        return this.priorities.NORMAL;
+    }
+
+    /**
+     * Get job concurrency
+     * @public
+     * @returns {number|null} Maximum concurrent instances or null for default
+     */
+    getConcurrency() {
+        return 1;
+    }
+
+    /**
+     * Get job lock lifetime in milliseconds
+     * @public
+     * @returns {number|null} Lock lifetime or null for default
+     */
+    getLockLifetime() {
+        return 55 * 60 * 1000; // 55 minutes
+    }
+
+    /**
+     * Determines if the job should be enabled when created
+     * @public
+     * @returns {boolean} True if job should be enabled by default, false otherwise
+     */
+    getEnabled() {
+        return true;
+    }
+
     /**
      * Internal method to run the job and handle both Promise and callback patterns.
+     * @private
      * @param {MongoDb} db The MongoDB database connection
      * @param {Object} job The job instance
      * @param {Function} done Callback to be called when job completes
@@ -304,46 +359,6 @@ class Job {
 
         this.log?.d(`[Job:${this.jobName}] Progress update`, progressData);
     }
-
-    /**
-     * Get job retry configuration
-     * @typedef {Object} RetryConfig
-     * @property {boolean} enabled - Whether retries are enabled
-     * @property {number} attempts - Number of retry attempts
-     * @property {number} delay - Delay between retry attempts in milliseconds, delay is by default exponentially increasing after each attempt
-     * @returns {RetryConfig|null} Retry configuration or null for default
-     */
-    getRetryConfig() {
-        return {
-            enabled: false,
-            attempts: 0,
-            delay: 5 * 60 * 1000 // 5 minutes
-        };
-    }
-
-    /**
-     * Get job priority
-     * @returns {string} Priority level from JOB_PRIORITIES
-     */
-    getPriority() {
-        return this.priorities.NORMAL;
-    }
-
-    /**
-     * Get job concurrency
-     * @returns {number|null} Maximum concurrent instances or null for default
-     */
-    getConcurrency() {
-        return 1;
-    }
-
-    /**
-     * Get job lock lifetime in milliseconds
-     * @returns {number|null} Lock lifetime or null for default
-     */
-    getLockLifetime() {
-        return 55 * 60 * 1000; // 55 minutes
-    }
 }
 
 module.exports = Job;
\ No newline at end of file
diff --git a/jobServer/JobManager.js b/jobServer/JobManager.js
index 6d5fc536b28..2350596dbe4 100644
--- a/jobServer/JobManager.js
+++ b/jobServer/JobManager.js
@@ -191,7 +191,7 @@ class JobManager {
      */
     async #initializeJobConfigs(jobClasses, jobDefinitionChecksums) {
         await Promise.all(
-            Object.entries(jobClasses).map(async([jobName, JobClass ]) => {
+            Object.entries(jobClasses).map(async([jobName, JobClass]) => {
                 this.#log.d(`Initializing job config for ${jobName}`);
                 const currentChecksum = jobDefinitionChecksums[jobName];
                 const existingConfigOverride = await this.#jobConfigsCollection.findOne({ jobName });
@@ -201,8 +201,8 @@ class JobManager {
                     await this.#createDefaultJobConfig(jobName, currentChecksum, JobClass);
                 }
                 else if (existingConfigOverride.checksum !== currentChecksum) {
-                    // Reset configuration if job implementation has changed
-                    await this.#resetJobConfig(jobName, currentChecksum);
+                    const instance = new JobClass(jobName);
+                    await this.#resetJobConfig(jobName, currentChecksum, instance.getEnabled());
                 }
             })
         );
@@ -220,7 +220,7 @@ class JobManager {
 
         await this.#jobConfigsCollection.insertOne({
             jobName,
-            enabled: true,
+            enabled: instance.getEnabled(),
             checksum,
             createdAt: new Date(),
             defaultConfig: {
@@ -239,14 +239,15 @@ class JobManager {
      * @private
      * @param {string} jobName - Name of the job
      * @param {string} newChecksum - New checksum of the job definition
+     * @param {boolean} defaultEnabled - Default enabled state of the job
      */
-    async #resetJobConfig(jobName, newChecksum) {
+    async #resetJobConfig(jobName, newChecksum, defaultEnabled) {
         this.#log.w(`Job ${jobName} implementation changed, resetting configuration`);
         await this.#jobConfigsCollection.updateOne(
             { jobName },
             {
                 $set: {
-                    enabled: true,
+                    enabled: defaultEnabled,
                     checksum: newChecksum,
                     updatedAt: new Date()
                 },
diff --git a/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js b/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js
index 2be672c1279..e385fedd909 100644
--- a/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js
+++ b/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js
@@ -45,6 +45,7 @@ class PulseJobExecutor extends IJobExecutor {
             const priority = this.#mapPriority(instance.getPriority());
             const concurrency = instance.getConcurrency();
             const lockLifetime = instance.getLockLifetime();
+            const isEnabled = instance.getEnabled();
 
             this.pulseRunner.define(
                 jobName,
@@ -68,9 +69,15 @@ class PulseJobExecutor extends IJobExecutor {
                 }
             );
 
+            // If job should be disabled by default, use disableJob method
+            if (!isEnabled) {
+                await this.disableJob(jobName);
+                this.log.d(`Job ${jobName} disabled after creation`);
+            }
+
             const scheduleConfig = instance.getSchedule();
             this.pendingSchedules.set(jobName, scheduleConfig);
-            this.log.d(`Job ${jobName} defined successfully`);
+            this.log.d(`Job ${jobName} defined successfully with enabled state: ${isEnabled}`);
 
             this.log.d(`Configuring job ${jobName} with priority: ${priority}, concurrency: ${concurrency}, lockLifetime: ${lockLifetime}`);
 
diff --git a/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js b/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js
index 951494bd4cd..1782f15ce75 100644
--- a/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js
+++ b/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js
@@ -7,7 +7,7 @@ const {isValidCron} = require('cron-validator');
  * @typedef {import('@logger/interface')} Logger
  * 
  * @typedef {Object} ScheduleConfig
- * @property {'schedule'|'once'|'now'} type - Type of schedule
+ * @property {'schedule'|'once'|'now'|'manual'} type - Type of schedule
  * @property {string|Date} [value] - Cron expression for 'schedule' type or Date for 'once' type
  */
 
@@ -43,12 +43,12 @@ class PulseJobScheduler extends IJobScheduler {
             throw new Error('Schedule type is required');
         }
 
-        const validTypes = ['schedule', 'once', 'now'];
+        const validTypes = ['schedule', 'once', 'now', 'manual'];
         if (!validTypes.includes(config.type)) {
             throw new Error(`Invalid schedule type: ${config.type}`);
         }
 
-        if (config.type !== 'now' && !config.value) {
+        if (config.type !== 'now' && config.type !== 'manual' && !config.value) {
             throw new Error('Schedule value is required');
         }
     }
@@ -87,6 +87,11 @@ class PulseJobScheduler extends IJobScheduler {
             case 'now':
                 await this.pulseRunner.now(name, data);
                 break;
+            case 'manual':
+                // For manual jobs, do not automatically schedule them.
+                // They should only run when explicitly triggered.
+                this.log.d(`Job '${name}' is set to manual; not scheduling automatically.`);
+                break;
             }
 
             this.log.i(`Successfully scheduled job '${name}'`, {
@@ -140,6 +145,10 @@ class PulseJobScheduler extends IJobScheduler {
             case 'now':
                 await this.pulseRunner.now(jobName);
                 break;
+            case 'manual':
+                // For manual jobs, we remove any existing schedule. They should be triggered manually.
+                this.log.d(`Job '${jobName}' updated to manual schedule; not rescheduling automatically.`);
+                break;
             }
 
             this.log.i(`Successfully updated schedule for job '${jobName}'`, {

From 32bdb3bd5d5232967d36a3b85d673d2251fa808d Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Sun, 2 Feb 2025 00:10:18 +0530
Subject: [PATCH 72/90] migrate AppExpire but disabled by default

---
 api/jobs/appExpire.js | 37 ++++++++++++++++++++++++++++---------
 1 file changed, 28 insertions(+), 9 deletions(-)

diff --git a/api/jobs/appExpire.js b/api/jobs/appExpire.js
index 7fb968c19f7..e33b9bae3df 100644
--- a/api/jobs/appExpire.js
+++ b/api/jobs/appExpire.js
@@ -1,15 +1,34 @@
-'use strict';
-
-const job = require('../parts/jobs/job.js'),
-    async = require('async'),
-    plugins = require('../../plugins/pluginManager.js'),
-    log = require('../utils/log.js')('job:appExpire'),
-    common = require('../utils/common.js'),
-    crypto = require('crypto');
+const async = require('async');
+const plugins = require('../../plugins/pluginManager.js');
+const log = require('../utils/log.js')('job:appExpire');
+const common = require('../utils/common.js');
+const crypto = require('crypto');
+const {Job} = require("../../jobServer");
 
 
 /** Class for the user mergind job **/
-class AppExpireJob extends job.Job {
+class AppExpireJob extends Job {
+
+    /**
+     * Determines if the job should be enabled when created
+     * @public
+     * @returns {boolean} True if job should be enabled by default, false otherwise
+     */
+    getEnabled() {
+        return false;
+    }
+
+    /**
+     * Get schedule for the job
+     * @returns {GetScheduleConfig} Schedule configuration object
+     */
+    getSchedule() {
+        return {
+            type: 'schedule',
+            value: '15 4 * * *' // every day at 4:15 AM
+        };
+    }
+
     /**
      * Run the job
      * @param {Db} database connection

From 65b35ba1ed8c88a1f6844e13e0d515c09fe1ef8d Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Sun, 2 Feb 2025 00:10:33 +0530
Subject: [PATCH 73/90] remove clear job job

---
 api/jobs/clear.js | 40 ----------------------------------------
 1 file changed, 40 deletions(-)
 delete mode 100644 api/jobs/clear.js

diff --git a/api/jobs/clear.js b/api/jobs/clear.js
deleted file mode 100644
index 1150e4c757a..00000000000
--- a/api/jobs/clear.js
+++ /dev/null
@@ -1,40 +0,0 @@
-'use strict';
-
-const job = require('../parts/jobs/job.js'),
-    log = require('../utils/log.js')('job:clear');
-
-/** Class for job of clearing old jobs **/
-class ClearJob extends job.Job {
-    /**
-     * Run the job
-     * @param {Db} db connection
-     * @param {done} done callback
-     */
-    run(db, done) {
-        log.d('Clearing jobs ...');
-        var query = {
-            $and: [
-                {status: {$in: [job.STATUS.DONE, job.STATUS.CANCELLED]}},
-                {
-                    $or: [
-                        {finished: {$exists: false}},
-                        {finished: {$lt: Date.now() - 60 * 60 * 24 * 1000}},
-                        {finished: null}
-                    ]
-                }
-            ]
-        };
-
-        db.collection('jobs').deleteMany(query, (err, result) => {
-            if (err) {
-                log.e('Error while clearing jobs: ', err);
-            }
-            else {
-                log.d('Done clearing old jobs done before %j:', query.$and[1].$or[1].finished.$lt, result.deletedCount);
-            }
-            done(err);
-        });
-    }
-}
-
-module.exports = ClearJob;
\ No newline at end of file

From 774273fee23764ccf66f1b53244c56b4e06661b8 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Sun, 2 Feb 2025 00:10:47 +0530
Subject: [PATCH 74/90] remove clues to the old jobs

---
 api/api.js | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/api/api.js b/api/api.js
index f9649d8e6af..4a31b872183 100644
--- a/api/api.js
+++ b/api/api.js
@@ -182,17 +182,6 @@ plugins.connectToAllDatabases().then(function() {
         console.trace();
     });
 
-    // jobs.job('api:topEvents').replace().schedule('at 00:01 am ' + 'every 1 day'); // PORTED
-    // jobs.job('api:ping').replace().schedule('every 1 day'); // PORTED
-    // jobs.job('api:clear').replace().schedule('every 1 day'); // REMOVED
-    // jobs.job('api:clearTokens').replace().schedule('every 1 day'); // PORTED
-    // jobs.job('api:clearAutoTasks').replace().schedule('every 1 day'); // PORTED
-    // jobs.job('api:task').replace().schedule('every 5 minutes'); // PORTED
-    // jobs.job('api:userMerge').replace().schedule('every 10 minutes'); // PORTED
-    // jobs.job("api:ttlCleanup").replace().schedule("every 1 minute"); // PORTED
-    //jobs.job('api:appExpire').replace().schedule('every 1 day'); // Deprecated
-
-
     var utcMoment = moment.utc();
     var incObj = {};
     incObj.r = 1;

From d08297c62aec25cac75dea90b0ee3fd63a7bc916 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Sun, 2 Feb 2025 00:11:16 +0530
Subject: [PATCH 75/90] literally 1 jsdoc added

---
 jobServer/Job.js | 1 +
 1 file changed, 1 insertion(+)

diff --git a/jobServer/Job.js b/jobServer/Job.js
index bc5bafba8f0..7b36ca283e4 100644
--- a/jobServer/Job.js
+++ b/jobServer/Job.js
@@ -247,6 +247,7 @@ class Job {
      * Determines if the job should be enabled when created
      * @public
      * @returns {boolean} True if job should be enabled by default, false otherwise
+     * @default true
      */
     getEnabled() {
         return true;

From 771d3cf8210e90ee3fd5ad7b9c4ac07c7efd67a2 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Sun, 2 Feb 2025 12:47:53 +0530
Subject: [PATCH 76/90] some cleaning up

---
 jobServer/JobManager.js | 8 ++++----
 jobServer/JobServer.js  | 6 ------
 2 files changed, 4 insertions(+), 10 deletions(-)

diff --git a/jobServer/JobManager.js b/jobServer/JobManager.js
index 2350596dbe4..a7f822f7d52 100644
--- a/jobServer/JobManager.js
+++ b/jobServer/JobManager.js
@@ -96,7 +96,7 @@ class JobManager {
                     _id: change.documentKey._id
                 });
                 if (jobConfig) {
-                    await this.#applyConfig({...change?.updateDescription?.updatedFields, jobName: jobConfig.jobName});
+                    await this.#applyConfig({...change.updateDescription?.updatedFields, jobName: jobConfig.jobName});
                 }
             }
         });
@@ -141,8 +141,8 @@ class JobManager {
                 await this.#jobRunner.configureRetry(jobName, jobConfig.retry);
             }
 
-            if (typeof jobConfig?.enabled === 'boolean') {
-                if (jobConfig?.enabled) {
+            if (typeof jobConfig.enabled === 'boolean') {
+                if (jobConfig.enabled) {
                     await this.#jobRunner.enableJob(jobName);
                     this.#log.i(`Job ${jobName} enabled via config`);
                 }
@@ -154,7 +154,7 @@ class JobManager {
         }
         catch (error) {
             this.#log.e('Failed to apply job configuration:', {
-                jobName: jobConfig?.jobName || "unknown",
+                jobName: jobConfig.jobName || "unknown",
                 error: error.message,
                 stack: error.stack
             });
diff --git a/jobServer/JobServer.js b/jobServer/JobServer.js
index 7d62ec807d4..a379edb6084 100644
--- a/jobServer/JobServer.js
+++ b/jobServer/JobServer.js
@@ -51,12 +51,6 @@ class JobServer {
      */
     #isRunning = false;
 
-    /**
-     * The database connection
-     * @type {MongoDb | null}
-     */
-    #db = null;
-
     /**
      * Collection for job configurations
      * @private

From a6d2134de1c1b9f16368048de54e38913bbfe78b Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Sun, 2 Feb 2025 15:16:40 +0530
Subject: [PATCH 77/90] fixing cross countly comm

---
 jobServer/index.js | 41 +++++++++++++++++++++++++++++------------
 1 file changed, 29 insertions(+), 12 deletions(-)

diff --git a/jobServer/index.js b/jobServer/index.js
index 4971cc812bd..ac8ad9e8ee6 100644
--- a/jobServer/index.js
+++ b/jobServer/index.js
@@ -116,26 +116,39 @@ if (require.main === module) {
              * @param {string} event - The event name
              * @param {Object} params - The event parameters
              * @param {Function} callback - Callback function
+             * @returns {void} Void
              */
-            common.dispatch = function(event, params, callback) {
-                // Construct the request to your Countly server
-                countlyRequest.post({
-                    url: baseUrl + "/i/plugins/dispatch",
-                    json: { event: event, params: params }
+            pluginManager.dispatch = function(event, params, callback) {
+                // Ignore dispatch if event starts with /db
+                if (event.startsWith('/db')) {
+                    if (typeof callback === 'function') {
+                        return callback(null, null, null);
+                    }
+                    return;
+                }
+                const requestBody = {
+                    ...(params || {}),
+                    source: 'job_server',
+                    api_key: countlyConfig.api_key || "b2cc5212e091193e13299e2ece6222a6"
+                };
+
+                countlyRequest({
+                    url: baseUrl + event,
+                    method: 'POST',
+                    form: requestBody
                 }, function(err, res, body) {
                     if (err) {
                         log.e("Error dispatching event to Countly server:", {
                             error: err,
                             event: event,
-                            params: params,
                             baseUrl: baseUrl,
                         });
                     }
                     if (typeof callback === 'function') {
-                        return callback(err, res, body);
+                        callback(err, res, body);
                     }
                     else {
-                        log.e("Callback is not a function");
+                        log.w("No callback provided for dispatch");
                     }
                 });
             };
@@ -161,14 +174,18 @@ if (require.main === module) {
             const config = await pluginManager.getConfig();
             log.d('Configuration initialized successfully');
 
-            const countlyRequest = await initializeCountlyRequest(config);
-            await overrideCommonDispatch(countlyRequest, config);
-
             // Use connectToAllDatabases which handles config loading and db connections
             const [countlyDb, outDb, fsDb, drillDb] = await pluginManager.connectToAllDatabases();
 
             // Only need to override batcher since connectToAllDatabases handles other overrides
-            await overrideCommonBatcher(countlyDb);
+            await new Promise(resolve => {
+                setTimeout(async() => {
+                    const countlyRequest = await initializeCountlyRequest(config);
+                    await overrideCommonDispatch(countlyRequest, config);
+                    await overrideCommonBatcher(countlyDb);
+                    resolve();
+                }, 3000);
+            });
 
             return {
                 config,

From 908609cc3f8a98cdf4db528e8e8fc84438821a2c Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Sun, 2 Feb 2025 15:39:53 +0530
Subject: [PATCH 78/90] remove obsolete migration jobs

---
 plugins/data_migration/api/jobs/migrate.js   | 82 --------------------
 plugins/data_migration/api/parts/resource.js | 82 --------------------
 plugins/data_migration/api/resource.js       | 71 -----------------
 3 files changed, 235 deletions(-)
 delete mode 100644 plugins/data_migration/api/jobs/migrate.js
 delete mode 100644 plugins/data_migration/api/parts/resource.js
 delete mode 100644 plugins/data_migration/api/resource.js

diff --git a/plugins/data_migration/api/jobs/migrate.js b/plugins/data_migration/api/jobs/migrate.js
deleted file mode 100644
index 78d9b24bfb4..00000000000
--- a/plugins/data_migration/api/jobs/migrate.js
+++ /dev/null
@@ -1,82 +0,0 @@
-'use strict';
-
-const job = require('../../../../api/parts/jobs/job.js'),
-    retry = require('../../../../api/parts/jobs/retry.js'),
-    log = require('../../../../api/utils/log.js')('job:data_migration:migrate'),
-    MigrationResource = require('../parts/resource.js');
-
-/** class - ValidateJob */
-class MigrateJob extends job.TransientJob {
-    /** 
-     * constructor
-     * @param {string} name - job name
-     * @param {object} data  - job data
-     */
-    constructor(name, data) {
-        super(name, data);
-    }
-
-    /** 
-     * Prepare job
-     * 
-     * @returns {Promise} promise, resolved(always)
-     */
-    prepare(/*manager, db*/) {
-        return Promise.resolve();
-    }
-
-    /** 
-     * Resource name
-     * 
-     * @returns {string} 'migration'
-     */
-    resourceName() {
-        return 'migration';
-    }
-
-    /** 
-     * Create resource
-     * 
-     * @param {string} _id - id
-     * @param {string} name  - name
-     * @returns {object} MigrationResource
-     */
-    createResource(_id, name) {
-        // TODO: custom data
-        log.d('creating resource in migrate.js');
-        return new MigrationResource(_id, name, {exportid: this.data.exportid}, this.db());
-    }
-
-    /** 
-     * Release Resource
-     * @param {object} resource to call on
-     * @returns {Promise} result on close
-     */
-    releaseResource(resource) {
-        return resource.close();
-    }
-
-    /** 
-     * Retry policy for this job - probably no retries needed
-     * @returns {object} retry policy
-     */
-    retryPolicy() {
-        return new retry.NoRetryPolicy();
-    }
-
-    /** 
-     * Main job function
-     * @return {Any} not used
-     */
-    async run() {
-        log.i('[%d] running migration %j', process.pid, this.data);
-
-        let result = await this.resource.migrate(['a', 'b', 'c']);
-
-        log.i('[%d] migration done with %j', process.pid, result);
-
-        return result;
-    }
-}
-
-module.exports = MigrateJob;
\ No newline at end of file
diff --git a/plugins/data_migration/api/parts/resource.js b/plugins/data_migration/api/parts/resource.js
deleted file mode 100644
index 50ec638f958..00000000000
--- a/plugins/data_migration/api/parts/resource.js
+++ /dev/null
@@ -1,82 +0,0 @@
-const res = require('../../../../api/parts/jobs/resource.js'),
-    log = require('../../../../api/utils/log.js')('job:data_migration:resource:' + process.pid);
-var Migrator = require("./../data_migration_helper.js");
-
-/** Migration resource class */
-class MigrationResource extends res.Resource {
-    /** 
-     * Constroctor
-     * @param {string} _id - id
-     * @param {string} name -name
-     * @param {object} args - args
-     * @param {object} db - db connection
-     */
-    constructor(_id, name, args, db) {
-        super(_id, name);
-        this.db = db;
-        // TODO: custom data
-        this.exportid = args.exportid;
-        log.d('[%d]: Initializing data_migration resource with %j / %j / %j', process.pid, _id, name, args);
-    }
-
-    /** 
-     * Open resource: prepare some data, open connections, etc.
-     * @returns {Promise} promise
-     */
-    open() {
-        log.d('[%s]: Opening %j', this._id, this.exportid);
-        this.opened(); //to give call that i am open (mandatory)
-        return Promise.resolve();
-    }
-
-    /** 
-     * Close resource: release data, close connections, etc.
-     * @returns {Promise} promise
-     */
-    close() {
-        log.d('[%s]: Closing %j', this._id, this.exportid);
-        return Promise.resolve();
-    }
-
-    /** 
-     * Some custom method to call from job
-     * @param {object} stuff - message
-     * @returns {Promise} promise
-     */
-    migrate(stuff) {
-        log.d('migrating: %j', stuff);
-        console.log("running export");
-        return new Promise((resolve/*, reject*/) => {
-            var migration_helper = new Migrator(this.db);
-            migration_helper.runExportFunction(this.exportid, this.db, function(/*err1, res1*/) {
-                log.d("migration exited");
-
-                resolve();
-                migration_helper.closeAssets();
-            });
-        });
-    }
-
-    /** 
-     * Keep-alive / cheking resource is still online
-     * @returns {Promise} - promise
-     */
-    checkActive() {
-        return new Promise((resolve) => {
-            log.d('checkActive');
-            setTimeout(() => {
-                resolve(true);
-            }, 2000);
-        });
-    }
-
-    /** 
-     * Don't terminate resource process on master exit
-     * @returns {boolean} always false
-     */
-    canBeTerminated() {
-        return false;
-    }
-}
-
-module.exports = MigrationResource;
\ No newline at end of file
diff --git a/plugins/data_migration/api/resource.js b/plugins/data_migration/api/resource.js
deleted file mode 100644
index 6f002a388c0..00000000000
--- a/plugins/data_migration/api/resource.js
+++ /dev/null
@@ -1,71 +0,0 @@
-const res = require('../../../../api/parts/jobs/resource.js'),
-    log = require('../../../../api/utils/log.js')('job:data_migration:resource:' + process.pid);
-
-/** resource class */
-class MigrationResource extends res.Resource {
-    /** 
-     * Constroctor
-     * @param {string} _id - id
-     * @param {string} name -name
-     * @param {object} args - args
-     * @param {object} db - db connection
-     */
-    constructor(_id, name, args, db) {
-        super(_id, name);
-        this.db = db;
-        // TODO: custom data
-        this.x = this.data.x;
-        log.d('[%d]: Initializing data_migration resource with %j / %j / %j', process.pid, _id, name, args);
-    }
-
-    /** 
-     * Open resource: prepare some data, open connections, etc.
-     * @returns {Promise} promise
-     */
-    open() {
-        log.d('[%s]: Opening %j', this._id, this.x);
-        return Promise.resolve();
-    }
-
-    /** 
-     * Close resource: release data, close connections, etc.
-     * @returns {Promise} promise
-     */
-    close() {
-        return Promise.resolve();
-    }
-
-    /** 
-     * Some custom method to call from job
-     * @param {object} stuff - message
-     * @returns {Promise} promise
-     */
-    migrate(stuff) {
-        log.d('migrating: %j', stuff);
-        console.log("running export");
-        return Promise.resolve();
-    }
-
-    /** 
-     * Keep-alive / cheking resource is still online
-     * @returns {Promise} - promise
-     */
-    checkActive() {
-        return new Promise((resolve) => {
-            log.d('checkActive');
-            setTimeout(() => {
-                resolve(true);
-            }, 2000);
-        });
-    }
-
-    /** 
-     * Don't terminate resource process on master exit
-     * @returns {boolean} false
-     */
-    canBeTerminated() {
-        return false;
-    }
-}
-
-module.exports = MigrationResource;
\ No newline at end of file

From 2e2b37b0e9edebe0313eeab4728e3627c9bf9600 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Sun, 2 Feb 2025 16:38:11 +0530
Subject: [PATCH 79/90] fix per job logger

---
 jobServer/Job.js                                   | 8 ++++++--
 jobServer/jobRunner/impl/pulse/PulseJobExecutor.js | 1 -
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/jobServer/Job.js b/jobServer/Job.js
index 7b36ca283e4..9bfa3b316f3 100644
--- a/jobServer/Job.js
+++ b/jobServer/Job.js
@@ -94,9 +94,13 @@ class Job {
 
     /**
      * Creates an instance of Job.
+     * @param {string} jobName The name of the job
      */
-    constructor() {
-        this.log = Logger('jobs:job');
+    constructor(jobName) {
+        if (jobName) {
+            this.jobName = jobName;
+        }
+        this.log = Logger('jobs:' + this.jobName);
         this.log.d(`Job instance "${this.jobName}" created`);
     }
 
diff --git a/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js b/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js
index e385fedd909..9a4538dfd54 100644
--- a/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js
+++ b/jobServer/jobRunner/impl/pulse/PulseJobExecutor.js
@@ -39,7 +39,6 @@ class PulseJobExecutor extends IJobExecutor {
         this.log.d(`Attempting to create job: ${jobName}`);
         try {
             const instance = new JobClass(jobName);
-            instance.setJobName(jobName);
 
             const retryConfig = instance.getRetryConfig();
             const priority = this.#mapPriority(instance.getPriority());

From 9763887359f80c75bf0d9ed3e69324ef21c45a20 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Sun, 2 Feb 2025 16:38:28 +0530
Subject: [PATCH 80/90] improvement to single run job

---
 jobServer/jobRunner/impl/pulse/PulseJobScheduler.js | 12 ++++++++----
 jobServer/jobRunner/interfaces/IJobScheduler.js     |  2 +-
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js b/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js
index 1782f15ce75..9b5344313cb 100644
--- a/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js
+++ b/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js
@@ -84,9 +84,11 @@ class PulseJobScheduler extends IJobScheduler {
                 await this.pulseRunner.schedule(scheduleConfig.value, name, data);
                 break;
 
-            case 'now':
-                await this.pulseRunner.now(name, data);
+            case 'now': {
+                const now = new Date();
+                await this.pulseRunner.schedule(now, name, data);
                 break;
+            }
             case 'manual':
                 // For manual jobs, do not automatically schedule them.
                 // They should only run when explicitly triggered.
@@ -142,9 +144,11 @@ class PulseJobScheduler extends IJobScheduler {
                 await this.pulseRunner.schedule(schedule.value, jobName);
                 break;
 
-            case 'now':
-                await this.pulseRunner.now(jobName);
+            case 'now': {
+                const now = new Date();
+                await this.pulseRunner.schedule(now, jobName);
                 break;
+            }
             case 'manual':
                 // For manual jobs, we remove any existing schedule. They should be triggered manually.
                 this.log.d(`Job '${jobName}' updated to manual schedule; not rescheduling automatically.`);
diff --git a/jobServer/jobRunner/interfaces/IJobScheduler.js b/jobServer/jobRunner/interfaces/IJobScheduler.js
index 9d0eaf0999f..57bc90a3898 100644
--- a/jobServer/jobRunner/interfaces/IJobScheduler.js
+++ b/jobServer/jobRunner/interfaces/IJobScheduler.js
@@ -1,6 +1,6 @@
 /**
  * @typedef {Object} ScheduleConfig
- * @property {'once'|'schedule'|'now'} type - Type of schedule execution
+ * @property {'once'|'schedule'|'now'|'manual'} type - Type of schedule execution
  * @property {string|Date} [value] - Cron expression (for 'schedule') or Date object (for 'once')
  */
 

From c04f249d123b3633632897ea30bd0acde7aae23a Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Sun, 2 Feb 2025 21:17:55 +0530
Subject: [PATCH 81/90] try to handle funnel and flow jobs :)

---
 jobServer/config.js |  2 +-
 jobServer/index.js  | 22 +++++++++++++++++-----
 2 files changed, 18 insertions(+), 6 deletions(-)

diff --git a/jobServer/config.js b/jobServer/config.js
index 22a9125b63a..1c436d09993 100644
--- a/jobServer/config.js
+++ b/jobServer/config.js
@@ -21,7 +21,7 @@ const DEFAULT_PULSE_CONFIG = {
     lockLimit: 1,
     defaultLockLimit: 3,
     defaultLockLifetime: 55 * 60 * 1000, // 55 minutes
-    sort: { nextRunAt: 1, priority: -1 },
+    // sort: { nextRunAt: 1, priority: -1 },
     disableAutoIndex: false,
     resumeOnRestart: true,
     db: {
diff --git a/jobServer/index.js b/jobServer/index.js
index ac8ad9e8ee6..7d43128a090 100644
--- a/jobServer/index.js
+++ b/jobServer/index.js
@@ -41,7 +41,7 @@ const JobServer = require('./JobServer');
 const Logger = require('../api/utils/log.js');
 const log = new Logger('jobServer:index');
 const CountlyRequest = require("countly-request");
-const {ReadBatcher, WriteBatcher} = require('../api/parts/data/batcher');
+const {ReadBatcher, WriteBatcher, InsertBatcher} = require('../api/parts/data/batcher');
 const common = require('../api/utils/common.js');
 const pluginManager = require('../plugins/pluginManager.js');
 
@@ -83,6 +83,7 @@ if (require.main === module) {
         try {
             common.writeBatcher = new WriteBatcher(commonDb);
             common.readBatcher = new ReadBatcher(commonDb);
+            common.insertBatcher = new InsertBatcher(commonDb);
         }
         catch (error) {
             log.e('Failed to override common batcher:', {
@@ -97,7 +98,7 @@ if (require.main === module) {
      * @param {Function} countlyRequest - Countly request function
      * @param {Object} countlyConfig - Countly config object
      */
-    const overrideCommonDispatch = async(countlyRequest, countlyConfig) => {
+    const overrideCommonDispatch = (countlyRequest, countlyConfig) => {
         try {
             if (!countlyRequest || !countlyConfig) {
                 throw new Error('Invalid parameters for dispatch override');
@@ -108,8 +109,9 @@ if (require.main === module) {
             const pathPrefix = countlyConfig.path || "";
 
             // The base URL of the running Countly server
-            const baseUrl = protocol + "://" + hostname + pathPrefix;
+            let baseUrl = protocol + "://" + hostname + pathPrefix;
             log.d('Configuring dispatch override with base URL:', baseUrl);
+            baseUrl = "http://localhost:3001";
 
             /**
              * Keep the same signature, but perform a request to the running Countly server
@@ -118,7 +120,8 @@ if (require.main === module) {
              * @param {Function} callback - Callback function
              * @returns {void} Void
              */
-            pluginManager.dispatch = function(event, params, callback) {
+            const originalDispatch = pluginManager.dispatch;
+            pluginManager.dispatch = async function(event, params, callback) {
                 // Ignore dispatch if event starts with /db
                 if (event.startsWith('/db')) {
                     if (typeof callback === 'function') {
@@ -126,6 +129,15 @@ if (require.main === module) {
                     }
                     return;
                 }
+                else if (event.startsWith('/drill/preprocess_query')) {
+                    // we should not load entire api.js for this
+                    // Need to figure out a better way to call preprocess_query
+                    require('./../plugins/drill/api/api.js');
+                    require('./../plugins/geo/api/api.js');
+
+                    return originalDispatch.call(pluginManager, event, params, callback);
+                }
+
                 const requestBody = {
                     ...(params || {}),
                     source: 'job_server',
@@ -181,7 +193,7 @@ if (require.main === module) {
             await new Promise(resolve => {
                 setTimeout(async() => {
                     const countlyRequest = await initializeCountlyRequest(config);
-                    await overrideCommonDispatch(countlyRequest, config);
+                    overrideCommonDispatch(countlyRequest, config);
                     await overrideCommonBatcher(countlyDb);
                     resolve();
                 }, 3000);

From 3c93269af88c5921450fefc806f13df2c2f69eb1 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Sun, 2 Feb 2025 21:40:12 +0530
Subject: [PATCH 82/90] close server before flushall, and keepalive between
 node and nginx

---
 api/api.js | 26 +++++++++++++++++++++++---
 1 file changed, 23 insertions(+), 3 deletions(-)

diff --git a/api/api.js b/api/api.js
index 4a31b872183..63654feed34 100644
--- a/api/api.js
+++ b/api/api.js
@@ -129,6 +129,19 @@ plugins.connectToAllDatabases().then(function() {
     */
     async function storeBatchedData(code) {
         try {
+
+            await new Promise((resolve) => {
+                server.close((err) => {
+                    if (err) {
+                        console.log("Error closing server:", err);
+                    }
+                    else {
+                        console.log("Server closed successfully");
+                        resolve();
+                    }
+                });
+            });
+
             await common.writeBatcher.flushAll();
             await common.insertBatcher.flushAll();
             console.log("Successfully stored batch state");
@@ -198,16 +211,18 @@ plugins.connectToAllDatabases().then(function() {
     //since process restarted mark running tasks as errored
     taskManager.errorResults({db: common.db});
 
-    // plugins.dispatch("/worker", {common: common});
     plugins.dispatch("/master", {}); // init hook
 
-    http.Server((req, res) => {
+    const server = http.Server((req, res) => {
         const params = {
             qstring: {},
             res: res,
             req: req
         };
 
+        res.setHeader('Connection', 'keep-alive');
+        res.setHeader('Keep-Alive', 'timeout=5, max=1000');
+
         if (req.method.toLowerCase() === 'post') {
             const formidableOptions = {};
             if (countlyConfig.api.maxUploadFileSize) {
@@ -281,7 +296,12 @@ plugins.connectToAllDatabases().then(function() {
         else {
             common.returnMessage(params, 405, "Method not allowed");
         }
-    }).listen(common.config.api.port, common.config.api.host || '').timeout = common.config.api.timeout || 120000;
+    });
+    server.listen(common.config.api.port, common.config.api.host || '');
+    server.timeout = common.config.api.timeout || 120000;
+    server.keepAliveTimeout = common.config.api.timeout || 120000;
+    server.headersTimeout = (common.config.api.timeout || 120000) + 1000; // Slightly higher
+
 
     plugins.loadConfigs(common.db);
 });

From 200cb3c2765a9b3320483d312f4bd195661356bc Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Sun, 2 Feb 2025 23:37:27 +0530
Subject: [PATCH 83/90] fix logger job and simplify

---
 plugins/logger/api/api.js        |  2 +-
 plugins/logger/api/constants.js  |  3 +++
 plugins/logger/api/jobs/clear.js | 11 ++---------
 3 files changed, 6 insertions(+), 10 deletions(-)
 create mode 100644 plugins/logger/api/constants.js

diff --git a/plugins/logger/api/api.js b/plugins/logger/api/api.js
index 246e398a66c..583a54b8d29 100644
--- a/plugins/logger/api/api.js
+++ b/plugins/logger/api/api.js
@@ -5,7 +5,7 @@ var exported = {},
     log = require('../../../api/utils/log.js')('logger:api'),
     { validateRead } = require('../../../api/utils/rights.js');
 // const JOB = require('../../../api/parts/jobs');
-const MAX_NUMBER_OF_LOG_ENTRIES = 1000;
+const {MAX_NUMBER_OF_LOG_ENTRIES} = require('./constants');
 const FEATURE_NAME = 'logger';
 
 var RequestLoggerStateEnum = {
diff --git a/plugins/logger/api/constants.js b/plugins/logger/api/constants.js
new file mode 100644
index 00000000000..87c376b7f6c
--- /dev/null
+++ b/plugins/logger/api/constants.js
@@ -0,0 +1,3 @@
+module.exports = {
+    MAX_NUMBER_OF_LOG_ENTRIES: 1000
+};
\ No newline at end of file
diff --git a/plugins/logger/api/jobs/clear.js b/plugins/logger/api/jobs/clear.js
index cb117945d58..4047599f092 100644
--- a/plugins/logger/api/jobs/clear.js
+++ b/plugins/logger/api/jobs/clear.js
@@ -6,7 +6,7 @@
 const Job = require("../../../../jobServer/Job");
 const log = require("../../../../api/utils/log.js")("job:logger:clear");
 
-const DEFAULT_MAX_ENTRIES = 1000;
+const {MAX_NUMBER_OF_LOG_ENTRIES} = require('../constants');
 
 /**
  * clears logs
@@ -29,14 +29,7 @@ class ClearJob extends Job {
      * @param {Database} db mongodb database instance
      */
     async run(db) {
-        let max = this.data.max;
-        if (typeof max !== "number") {
-            log.e(
-                "Maximum number of log entries required. Falling back to default value:",
-                DEFAULT_MAX_ENTRIES
-            );
-            max = DEFAULT_MAX_ENTRIES;
-        }
+        let max = MAX_NUMBER_OF_LOG_ENTRIES;
 
         log.d("Started: cleaning logs before the last", max);
         const appIds = await db.collection("apps").find().project({ _id: 1 }).toArray();

From 2d5333027c83c3a716565f1d535cf9ca1386879c Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Sun, 2 Feb 2025 23:38:17 +0530
Subject: [PATCH 84/90] config tweak for jobs

---
 jobServer/config.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/jobServer/config.js b/jobServer/config.js
index 1c436d09993..efc44daa9f8 100644
--- a/jobServer/config.js
+++ b/jobServer/config.js
@@ -15,10 +15,10 @@
  * @property {string} db.collection - MongoDB collection name for storing jobs
  */
 const DEFAULT_PULSE_CONFIG = {
-    processEvery: '30 seconds',
+    processEvery: '5 seconds',
     maxConcurrency: 1,
     defaultConcurrency: 1,
-    lockLimit: 1,
+    lockLimit: 3,
     defaultLockLimit: 3,
     defaultLockLifetime: 55 * 60 * 1000, // 55 minutes
     // sort: { nextRunAt: 1, priority: -1 },

From ac28d785d7796e42e3cda52779156661475fed5f Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 4 Feb 2025 11:17:09 +0530
Subject: [PATCH 85/90] add linting to staged files pre commit

---
 .husky/pre-commit | 1 +
 package.json      | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)
 create mode 100644 .husky/pre-commit

diff --git a/.husky/pre-commit b/.husky/pre-commit
new file mode 100644
index 00000000000..4af22ddd479
--- /dev/null
+++ b/.husky/pre-commit
@@ -0,0 +1 @@
+npx lint-staged --verbose
\ No newline at end of file
diff --git a/package.json b/package.json
index 0ffeee5e6c0..1b9ab30bb1d 100644
--- a/package.json
+++ b/package.json
@@ -22,7 +22,7 @@
   },
   "lint-staged": {
     "*.{js,vue}": [
-      "eslint --fix"
+      "eslint --fix --no-quiet"
     ]
   },
   "devDependencies": {

From 1b260af089205d52ed8e06474acdebc9f0919ed1 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 4 Feb 2025 11:17:38 +0530
Subject: [PATCH 86/90] warn when use bluebird

---
 .eslintrc.json | 235 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 235 insertions(+)

diff --git a/.eslintrc.json b/.eslintrc.json
index 5a02c8253c3..41c7c2f28f1 100644
--- a/.eslintrc.json
+++ b/.eslintrc.json
@@ -1,6 +1,7 @@
 {
     "extends": "eslint:recommended",
     "rules": {
+//        "quotes": ["warn", "single", { "avoidEscape": true }],
         "require-atomic-updates": "off",
         "block-spacing": [
             "error",
@@ -118,6 +119,240 @@
         "unicode-bom": [
             "error",
             "never"
+        ],
+        "no-restricted-properties": [
+            "warn",
+            {
+                "object": "Promise",
+                "property": "map",
+                "message": "Bluebird-specific method 'Promise.map' detected. Suggestion: use native arrays with Promise.all or a library like 'p-map'."
+            },
+            {
+                "object": "Promise",
+                "property": "reduce",
+                "message": "Bluebird-specific method 'Promise.reduce' detected. Suggestion: use Array.reduce plus async/await or a concurrency library."
+            },
+            {
+                "object": "Promise",
+                "property": "filter",
+                "message": "Bluebird-specific method 'Promise.filter' detected. Suggestion: use Array.filter plus async/await or a concurrency library."
+            },
+            {
+                "object": "Promise",
+                "property": "each",
+                "message": "Bluebird-specific method 'Promise.each' detected. Suggestion: use a for-loop/forEach with async/await."
+            },
+            {
+                "object": "Promise",
+                "property": "props",
+                "message": "Bluebird-specific method 'Promise.props' detected. Suggestion: use Promise.all with Object.entries or a custom approach."
+            },
+            {
+                "object": "Promise",
+                "property": "join",
+                "message": "Bluebird-specific method 'Promise.join' detected. Suggestion: use Promise.all([...]) and destructuring in .then."
+            },
+            {
+                "object": "Promise",
+                "property": "try",
+                "message": "Bluebird-specific method 'Promise.try' detected. Suggestion: use a try/catch block or an async function."
+            },
+            {
+                "object": "Promise",
+                "property": "attempt",
+                "message": "Bluebird-specific method 'Promise.attempt' detected. Suggestion: same as 'Promise.try'—use try/catch or async."
+            },
+            {
+                "object": "Promise",
+                "property": "method",
+                "message": "Bluebird-specific method 'Promise.method' detected. Suggestion: define an async function or return a native Promise."
+            },
+            {
+                "object": "Promise",
+                "property": "promisify",
+                "message": "Bluebird-specific method 'Promise.promisify' detected. Suggestion: use native 'util.promisify' or wrap in a new Promise."
+            },
+            {
+                "object": "Promise",
+                "property": "promisifyAll",
+                "message": "Bluebird-specific method 'Promise.promisifyAll' detected. Suggestion: consider 'util.promisify' for each function or a similar library."
+            },
+            {
+                "object": "Promise",
+                "property": "fromCallback",
+                "message": "Bluebird-specific method 'Promise.fromCallback' detected. Suggestion: use new Promise(...) or 'util.promisify'."
+            },
+            {
+                "object": "Promise",
+                "property": "coroutine",
+                "message": "Bluebird-specific method 'Promise.coroutine' detected. Suggestion: use native async/await."
+            },
+            {
+                "object": "Promise",
+                "property": "spawn",
+                "message": "Bluebird-specific method 'Promise.spawn' detected. Suggestion: use native async/await."
+            },
+            {
+                "object": "Promise",
+                "property": "using",
+                "message": "Bluebird-specific method 'Promise.using' detected. Suggestion: use try/finally or a resource-management library."
+            },
+            {
+                "object": "Promise",
+                "property": "disposer",
+                "message": "Bluebird-specific method 'Promise.disposer' detected. Suggestion: use try/finally or a resource-management library."
+            },
+            {
+                "object": "Promise",
+                "property": "settle",
+                "message": "Bluebird-specific method 'Promise.settle' detected. Suggestion: use native 'Promise.allSettled'."
+            },
+
+            /* ---------- Same methods on the Bluebird object itself ---------- */
+            {
+                "object": "Bluebird",
+                "property": "map",
+                "message": "Bluebird-specific method 'Bluebird.map' detected. Suggestion: use array mapping + Promise.all or 'p-map'."
+            },
+            {
+                "object": "Bluebird",
+                "property": "reduce",
+                "message": "Bluebird-specific method 'Bluebird.reduce' detected. Suggestion: use array reduce + async/await or concurrency library."
+            },
+            {
+                "object": "Bluebird",
+                "property": "filter",
+                "message": "Bluebird-specific method 'Bluebird.filter' detected. Suggestion: use array filter + async/await or concurrency library."
+            },
+            {
+                "object": "Bluebird",
+                "property": "each",
+                "message": "Bluebird-specific method 'Bluebird.each' detected. Suggestion: use a for-loop or forEach + async/await."
+            },
+            {
+                "object": "Bluebird",
+                "property": "props",
+                "message": "Bluebird-specific method 'Bluebird.props' detected. Suggestion: use Promise.all with object entries or a custom approach."
+            },
+            {
+                "object": "Bluebird",
+                "property": "join",
+                "message": "Bluebird-specific method 'Bluebird.join' detected. Suggestion: use Promise.all([...]) and destructuring."
+            },
+            {
+                "object": "Bluebird",
+                "property": "try",
+                "message": "Bluebird-specific method 'Bluebird.try' detected. Suggestion: use a try/catch block or async function."
+            },
+            {
+                "object": "Bluebird",
+                "property": "attempt",
+                "message": "Bluebird-specific method 'Bluebird.attempt' detected. Suggestion: use a try/catch block or async function."
+            },
+            {
+                "object": "Bluebird",
+                "property": "method",
+                "message": "Bluebird-specific method 'Bluebird.method' detected. Suggestion: define an async function or return a native Promise."
+            },
+            {
+                "object": "Bluebird",
+                "property": "promisify",
+                "message": "Bluebird-specific method 'Bluebird.promisify' detected. Suggestion: use native 'util.promisify' or wrap in a new Promise."
+            },
+            {
+                "object": "Bluebird",
+                "property": "promisifyAll",
+                "message": "Bluebird-specific method 'Bluebird.promisifyAll' detected. Suggestion: consider 'util.promisify' or a similar library."
+            },
+            {
+                "object": "Bluebird",
+                "property": "fromCallback",
+                "message": "Bluebird-specific method 'Bluebird.fromCallback' detected. Suggestion: use new Promise(...) or 'util.promisify'."
+            },
+            {
+                "object": "Bluebird",
+                "property": "coroutine",
+                "message": "Bluebird-specific method 'Bluebird.coroutine' detected. Suggestion: use native async/await."
+            },
+            {
+                "object": "Bluebird",
+                "property": "spawn",
+                "message": "Bluebird-specific method 'Bluebird.spawn' detected. Suggestion: use native async/await."
+            },
+            {
+                "object": "Bluebird",
+                "property": "using",
+                "message": "Bluebird-specific method 'Bluebird.using' detected. Suggestion: use try/finally or a resource-management library."
+            },
+            {
+                "object": "Bluebird",
+                "property": "disposer",
+                "message": "Bluebird-specific method 'Bluebird.disposer' detected. Suggestion: use try/finally or a resource-management library."
+            },
+            {
+                "object": "Bluebird",
+                "property": "settle",
+                "message": "Bluebird-specific method 'Bluebird.settle' detected. Suggestion: use native 'Promise.allSettled'."
+            }
+        ],
+        "no-restricted-syntax": [
+            "warn",
+            {
+                "selector": "CallExpression[callee.property.name='tap']",
+                "message": "Bluebird-specific instance method '.tap()' detected. Suggestion: use '.then(value => { ...; return value; })'."
+            },
+            {
+                "selector": "CallExpression[callee.property.name='tapCatch']",
+                "message": "Bluebird-specific instance method '.tapCatch()' detected. Suggestion: use '.catch(error => { ...; throw error; })'."
+            },
+            {
+                "selector": "CallExpression[callee.property.name='spread']",
+                "message": "Bluebird-specific instance method '.spread()' detected. Suggestion: use '.then(([a, b]) => ... )' with array destructuring."
+            },
+            {
+                "selector": "CallExpression[callee.type='MemberExpression'][callee.property.name='bind'][callee.object.name=/^(Promise|Bluebird|BPromise)$/]",
+                "message": "Bluebird-specific '.bind()' detected on a Bluebird promise. Suggestion: manually bind 'this' or use arrow functions."
+            },
+            {
+                "selector": "CallExpression[callee.property.name='delay']",
+                "message": "Bluebird-specific instance method '.delay()' detected. Suggestion: use setTimeout() or a library (e.g., p-delay)."
+            },
+            {
+                "selector": "CallExpression[callee.property.name='timeout']",
+                "message": "Bluebird-specific instance method '.timeout()' detected. Suggestion: use p-timeout or similar library."
+            },
+            {
+                "selector": "CallExpression[callee.property.name='return']",
+                "message": "Bluebird-specific instance method '.return()' detected. Suggestion: use '.then(() => someValue)' or rewrite chain."
+            },
+            {
+                "selector": "CallExpression[callee.property.name='throw']",
+                "message": "Bluebird-specific instance method '.throw()' detected. Suggestion: use '.then(() => { throw error; })'."
+            },
+            {
+                "selector": "CallExpression[callee.property.name='asCallback']",
+                "message": "Bluebird-specific instance method '.asCallback()' detected. Suggestion: use 'util.callbackify' or rewrite manually."
+            },
+            {
+                "selector": "CallExpression[callee.property.name='nodeify']",
+                "message": "Bluebird-specific instance method '.nodeify()' detected. Suggestion: use 'util.callbackify' or rewrite manually."
+            },
+            {
+                "selector": "CallExpression[callee.property.name='reflect']",
+                "message": "Bluebird-specific instance method '.reflect()' detected. Suggestion: use 'Promise.allSettled' or custom handling."
+            },
+            {
+                "selector": "CallExpression[callee.property.name='caught']",
+                "message": "Bluebird-specific instance method '.caught()' detected. Suggestion: use '.catch()' with condition or separate logic."
+            },
+            {
+                "selector": "CallExpression[callee.property.name='catchReturn']",
+                "message": "Bluebird-specific instance method '.catchReturn()' detected. Suggestion: use '.catch(err => fallbackValue)' or similar."
+            },
+            {
+                "selector": "CallExpression[callee.property.name='catchThrow']",
+                "message": "Bluebird-specific instance method '.catchThrow()' detected. Suggestion: use '.catch(err => { throw newError; })'."
+            }
         ]
     },
     "overrides": [

From 9ba885fd094557023216792de3b5ed59452ad1db Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 4 Feb 2025 11:17:57 +0530
Subject: [PATCH 87/90] allow hard deletion of plugins when disable

---
 plugins/pluginManager.js | 56 ++++++++++++++++++++++++++--------------
 1 file changed, 37 insertions(+), 19 deletions(-)

diff --git a/plugins/pluginManager.js b/plugins/pluginManager.js
index 398efbec306..4b43e9dff90 100644
--- a/plugins/pluginManager.js
+++ b/plugins/pluginManager.js
@@ -1544,6 +1544,7 @@ var pluginManager = function pluginManager() {
 
     /**
     * Procedure to uninstall plugin
+    * Should be used only to hard disabled plugins mainly in dev mode
     * @param {string} plugin - plugin name
     * @param {function} callback - when finished uninstalling plugin
     * @returns {void} void
@@ -1551,28 +1552,45 @@ var pluginManager = function pluginManager() {
     this.uninstallPlugin = function(plugin, callback) {
         console.log('Uninstalling plugin %j...', plugin);
         callback = callback || function() {};
-        var scriptPath = path.join(__dirname, plugin, 'uninstall.js');
-        var errors = false;
-        var args = [scriptPath];
-        if (apiCountlyConfig.symlinked === true) {
-            args.unshift(...["--preserve-symlinks", "--preserve-symlinks-main"]);
-        }
-        var m = cp.spawn("nodejs", args);
+        var self = this;
 
-        m.stdout.on('data', (data) => {
-            console.log(data.toString());
-        });
+        // First update database to disable plugin
+        self.singleDefaultConnection().then((db) => {
+            db.collection("plugins").updateOne(
+                {_id: "plugins"},
+                {$set: {[`plugins.${plugin}`]: false}},
+                function(err) {
+                    if (err) {
+                        console.log("Error updating plugin state:", err);
+                    }
 
-        m.stderr.on('data', (data) => {
-            console.log(data.toString());
-        });
+                    // Then run uninstall script
+                    var scriptPath = path.join(__dirname, plugin, 'uninstall.js');
+                    var errors = false;
+                    var args = [scriptPath];
+                    if (apiCountlyConfig.symlinked === true) {
+                        args.unshift(...["--preserve-symlinks", "--preserve-symlinks-main"]);
+                    }
+                    var m = cp.spawn("nodejs", args);
 
-        m.on('close', (code) => {
-            console.log('Done running uninstall.js with %j', code);
-            if (parseInt(code, 10) !== 0) {
-                errors = true;
-            }
-            callback(errors);
+                    m.stdout.on('data', (data) => {
+                        console.log(data.toString());
+                    });
+
+                    m.stderr.on('data', (data) => {
+                        console.log(data.toString());
+                    });
+
+                    m.on('close', (code) => {
+                        console.log('Done running uninstall.js with %j', code);
+                        if (parseInt(code, 10) !== 0) {
+                            errors = true;
+                        }
+                        db.close();
+                        callback(errors);
+                    });
+                }
+            );
         });
     };
 

From e7dd284b1a83ae7953f9d3be916f87535e1fde0d Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Fri, 28 Feb 2025 21:34:31 +0530
Subject: [PATCH 88/90] init observability and traces across services

---
 api/utils/log.js | 684 +++++++++++++++++++++++------------------------
 package.json     |   3 +-
 2 files changed, 335 insertions(+), 352 deletions(-)

diff --git a/api/utils/log.js b/api/utils/log.js
index 2fbc1a4f736..205141a0570 100644
--- a/api/utils/log.js
+++ b/api/utils/log.js
@@ -1,68 +1,36 @@
-'use strict';
+const pino = require('pino');
 
-/**
- * Log provides a wrapper over debug or console functions with log level filtering, module filtering and ability to store log in database.
- * Uses configuration require('../config.js').logging:
- * {
- *  'info': ['app', 'auth', 'static'],      // log info and higher level for modules 'app*', 'auth*', 'static*'
- *  'debug': ['api.users'],                 // log debug and higher (in fact everything) for modules 'api.users*'
- *  'default': 'warn',                      // log warn and higher for all other modules
- * }
- * Note that log levels supported are ['debug', 'info', 'warn', 'error']
- *
- * Usage is quite simple:
- * var log = require('common.js').log('module[:submodule[:subsubmodule]]');
- * log.i('something happened: %s, %j', 'string', {obj: 'ect'});
- * log.e('something really bad happened: %j', new Error('Oops'));
- *
- * Whenever DEBUG is in process.env, log outputs all filtered messages with debug module instead of console so you could have pretty colors in console.
- * In other cases only log.d is logged using debug module.
- * 
- * To control log level at runtime, call require('common.js').log.setLevel('events', 'debug'). From now on 'events' logger will log everything.
- *
- * There is also a handy method for generating standard node.js callbacks which log error. Only applicable if no actions in case of error needed:
- * collection.find().toArray(log.callback(function(arg1, arg2){ // all good }));
- * - if error didn't happen, function is called
- * - if error happened, it will be logged, but function won't be called
- * - if error happened, arg1 is a first argument AFTER error, it's not an error
- * @module api/utils/log
- */
-
-var prefs = require('../config.js', 'dont-enclose').logging || {};
-prefs.default = prefs.default || "warn";
-var colors = require('colors');
-var deflt = (prefs && prefs.default) ? prefs.default : 'error';
-
-for (let level in prefs) {
-    if (prefs[level].sort) {
-        prefs[level].sort();
-    }
+// Optional OpenTelemetry imports
+let trace;
+let context;
+let metrics;
+let semanticConventions;
+try {
+    trace = require('@opentelemetry/api').trace;
+    context = require('@opentelemetry/api').context;
+    metrics = require('@opentelemetry/api').metrics;
+    semanticConventions = require('@opentelemetry/semantic-conventions');
+    // eslint-disable-next-line no-empty
+}
+catch (e) {
+    // do nothing
 }
 
-var styles = {
-    moduleColors: {
-        //      'push:*api': 0 // green
-        '[last]': -1
-    },
-    colors: ['green', 'yellow', 'blue', 'magenta', 'cyan', 'white', 'gray', 'red'],
-    stylers: {
-        warn: function(args) {
-            for (var i = 0; i < args.length; i++) {
-                if (typeof args[i] === 'string') {
-                    args[i] = colors.bgYellow.black(args[i].black);
-                }
-            }
-        },
-        error: function(args) {
-            for (var i = 0; i < args.length; i++) {
-                if (typeof args[i] === 'string') {
-                    args[i] = colors.bgRed.white(args[i].white);
-                }
-            }
-        }
-    }
+/**
+ * Mapping of short level codes to full level names
+ * @type {Object.<string, string>}
+ */
+const LEVELS = {
+    d: 'debug',
+    i: 'info',
+    w: 'warn',
+    e: 'error'
 };
 
+/**
+ * Mapping of log levels to acceptable log levels
+ * @type {Object.<string, string[]>}
+ */
 const ACCEPTABLE = {
     d: ['debug'],
     i: ['debug', 'info'],
@@ -70,61 +38,97 @@ const ACCEPTABLE = {
     e: ['debug', 'info', 'warn', 'error'],
 };
 
-const NAMES = {
-    d: 'DEBUG',
-    i: 'INFO',
-    w: 'WARN',
-    e: 'ERROR'
-};
+
+// Initialize configuration with defaults
+let prefs = require('../config.js', 'dont-enclose').logging || {};
+prefs.default = prefs.default || "warn";
+let deflt = (prefs && prefs.default) ? prefs.default : 'error';
 
 /**
- * Returns logger function for given preferences
- * @param {string} level - log level
- * @param {string} prefix - add prefix to message
- * @param {boolean} enabled - whether function should log anything
- * @param {object} outer - this for @out
- * @param {function} out - output function (console or debug)
- * @param {function} styler - function to apply styles
- * @returns {function} logger function
+ * Current levels for all modules
+ * @type {Object.<string, string>}
  */
-var log = function(level, prefix, enabled, outer, out, styler) {
-    return function() {
-        // console.log(level, prefix, enabled(), arguments);
-        if (enabled()) {
-            var args = Array.prototype.slice.call(arguments, 0);
-            var color = styles.moduleColors[prefix];
-            if (color === undefined) {
-                color = (++styles.moduleColors['[last]']) % styles.colors.length;
-                styles.moduleColors[prefix] = color;
-            }
-            color = styles.colors[color];
-            if (styler) {
-                args[0] = new Date().toISOString() + ': ' + level + '\t' + '[' + (prefix || '') + ']\t' + args[0];
-                styler(args);
-            }
-            else {
-                args[0] = (new Date().toISOString() + ': ' + level + '\t').gray + colors[color]('[' + (prefix || '') + ']\t') + args[0];
-            }
-            // args[0] = (new Date().toISOString() + ': ' + (prefix || '')).gray + args[0];
-            // console.log('Logging %j', args);
-            if (typeof out === 'function') {
-                out.apply(outer, args);
-            }
-            else {
-                for (var k in out) {
-                    out[k].apply(outer, args);
-                }
-            }
-        }
+const levels = {};
+
+// Metrics setup if OpenTelemetry is available
+let logCounter;
+let logDurationHistogram;
+
+if (metrics) {
+    const meter = metrics.getMeter('logger');
+    logCounter = meter.createCounter('log_entries_total', {
+        description: 'Number of log entries by level and module',
+    });
+    logDurationHistogram = meter.createHistogram('log_duration_seconds', {
+        description: 'Duration of logging operations',
+    });
+}
+
+/**
+ * Gets the current trace context if OpenTelemetry is available
+ * @returns {Object|null} Trace context object or null if unavailable
+ */
+function getTraceContext() {
+    if (!trace) {
+        return null;
+    }
+
+    const currentSpan = trace.getSpan(context.active());
+    if (!currentSpan) {
+        return null;
+    }
+
+    const spanContext = currentSpan.spanContext();
+    return {
+        'trace.id': spanContext.traceId,
+        'span.id': spanContext.spanId,
+        'trace.flags': spanContext.traceFlags.toString(16)
     };
-};
+}
+
+/**
+ * Creates a logging span if OpenTelemetry is available
+ * @param {string} name - The module name
+ * @param {string} level - The log level
+ * @param {string} message - The log message
+ * @returns {Span|null} The created span or null if unavailable
+ */
+function createLoggingSpan(name, level, message) {
+    if (!trace) {
+        return null;
+    }
+
+    const tracer = trace.getTracer('logger');
+    return tracer.startSpan(`log.${level}`, {
+        attributes: {
+            [semanticConventions.SemanticAttributes.CODE_FUNCTION]: name,
+            [semanticConventions.SemanticAttributes.CODE_NAMESPACE]: 'logger',
+            'logging.level': level,
+            'logging.message': message
+        }
+    });
+}
+
+/**
+ * Records metrics for logging operations
+ * @param {string} name - The module name
+ * @param {string} level - The log level
+ */
+function recordMetrics(name, level) {
+    if (logCounter) {
+        logCounter.add(1, {
+            module: name,
+            level: level
+        });
+    }
+}
 
 /**
  * Looks for logging level in config for a particular module
- * @param {string} name - module name
- * @returns {string} log level
+ * @param {string} name - The module name
+ * @returns {string} The configured log level
  */
-var logLevel = function(name) {
+const logLevel = function(name) {
     if (typeof prefs === 'undefined') {
         return 'error';
     }
@@ -132,20 +136,17 @@ var logLevel = function(name) {
         return prefs;
     }
     else {
-        for (var level in prefs) {
+        for (let level in prefs) {
             if (typeof prefs[level] === 'string' && name.indexOf(prefs[level]) === 0) {
                 return level;
             }
             if (typeof prefs[level] === 'object' && prefs[level].length) {
-                for (var i = prefs[level].length - 1; i >= 0; i--) {
-                    var opt = prefs[level][i];
+                for (let i = prefs[level].length - 1; i >= 0; i--) {
+                    let opt = prefs[level][i];
                     if (opt === name || name.indexOf(opt) === 0) {
                         return level;
                     }
                 }
-                // for (var m in prefs[level]) {
-                //  if (name.indexOf(prefs[level][m]) === 0) { return level; }
-                // }
             }
         }
         return deflt;
@@ -153,170 +154,220 @@ var logLevel = function(name) {
 };
 
 /**
- * Current levels for all modules
+ * Creates a Pino logger instance with the appropriate configuration
+ * @param {string} name - The module name
+ * @param {string} [level] - The log level
+ * @returns {Logger} Configured Pino logger instance
  */
-var levels = {
-    // mongo: 'info'
-};
-/**
-* Sets current logging level
-* @static
-* @param {string} module - name of the module for logging
-* @param {string} level -  level of logging, possible values are: debug, info, warn, error
-**/
-var setLevel = function(module, level) {
-    levels[module] = level;
+const createLogger = (name, level) => {
+    return pino({
+        name,
+        level: level || deflt,
+        timestamp: pino.stdTimeFunctions.isoTime,
+        formatters: {
+            level: (label) => {
+                return { level: label.toUpperCase() };
+            },
+            log: (object) => {
+                const traceContext = getTraceContext();
+                return traceContext ? { ...object, ...traceContext } : object;
+            }
+        }
+    });
 };
+
 /**
-* Sets default logging level for all modules, that do not have specific level set
-* @static
-* @param {string} level -  level of logging, possible values are: debug, info, warn, error
-**/
-var setDefault = function(level) {
-    deflt = level;
+ * Creates a logging function for a specific level
+ * @param {Logger} logger - The Pino logger instance
+ * @param {string} name - The module name
+ * @param {string} level - The log level code (d, i, w, e)
+ * @returns {Function} The logging function
+ */
+const createLogFunction = (logger, name, level) => {
+    return function(...args) {
+        const currentLevel = levels[name] || deflt;
+        if (ACCEPTABLE[level].indexOf(currentLevel) !== -1) {
+            const startTime = performance.now();
+            const message = args[0];
+
+            // Create span for this logging operation
+            const span = createLoggingSpan(name, LEVELS[level], message);
+
+            try {
+                if (args.length === 1) {
+                    logger[LEVELS[level]](args[0]);
+                }
+                else {
+                    const msg = args.shift();
+                    logger[LEVELS[level]](msg, ...args);
+                }
+
+                // Record metrics
+                recordMetrics(name, LEVELS[level]);
+
+                // Record duration
+                if (logDurationHistogram) {
+                    const duration = (performance.now() - startTime) / 1000; // Convert to seconds
+                    logDurationHistogram.record(duration, {
+                        module: name,
+                        level: LEVELS[level]
+                    });
+                }
+            }
+            finally {
+                if (span) {
+                    span.end();
+                }
+            }
+        }
+    };
 };
+
 /**
-* Get currently set logging level for module
-* @static
-* @param {string} module - name of the module for logging
-* @returns {string} level of logging, possible values are: debug, info, warn, error
-**/
-var getLevel = function(module) {
-    return levels[module] || deflt;
+ * Sets current logging level for a module
+ * @param {string} module - The module name
+ * @param {string} level - The log level
+ */
+const setLevel = function(module, level) {
+    levels[module] = level;
 };
 
-var getEnabledWithLevel = function(acceptable, module) {
-    return function() {
-        // if (acceptable.indexOf(levels[module]) === -1) {
-        //  console.log('Won\'t log %j because %j doesn\'t have %j (%j)', module, acceptable, levels[module], levels);
-        // }
-        return acceptable.indexOf(levels[module] || deflt) !== -1;
-    };
+/**
+ * Sets default logging level
+ * @param {string} level - The log level
+ */
+const setDefault = function(level) {
+    deflt = level;
 };
 
 /**
- * @typedef {Object} Logger
- * @property {function(): string} id - Get the logger id
- * @example
- * const loggerId = logger.id();
- * console.log(`Current logger ID: ${loggerId}`);
- *
- * @property {function(...*): void} d - Log debug level messages
- * @example
- * logger.d('Debug message: %s', 'Some debug info');
- *
- * @property {function(...*): void} i - Log information level messages
- * @example
- * logger.i('Info message: User %s logged in', username);
- *
- * @property {function(...*): void} w - Log warning level messages
- * @example
- * logger.w('Warning: %d attempts failed', attempts);
- *
- * @property {function(...*): void} e - Log error level messages
- * @example
- * logger.e('Error occurred: %o', errorObject);
- *
- * @property {function(string, function, string, ...*): boolean} f - Log variable level messages
- * @example
- * logger.f('d', (log) => {
- *   const expensiveOperation = performExpensiveCalculation();
- *   log('Debug: Expensive operation result: %j', expensiveOperation);
- * }, 'i', 'Skipped expensive debug logging');
- *
- * @property {function(function=): function} callback - Create a callback function for logging
- * @example
- * const logCallback = logger.callback((result) => {
- *   console.log('Operation completed with result:', result);
- * });
- * someAsyncOperation(logCallback);
- *
- * @property {function(string, function=, function=): function} logdb - Create a callback function for logging database operations
- * @example
- * const dbCallback = logger.logdb('insert user',
- *   (result) => { console.log('User inserted:', result); },
- *   (error) => { console.error('Failed to insert user:', error); }
- * );
- * database.insertUser(userData, dbCallback);
- *
- * @property {function(string): Logger} sub - Create a sub-logger
- * @example
- * const subLogger = logger.sub('database');
- * subLogger.i('Connected to database');
+ * Gets current logging level for a module
+ * @param {string} module - The module name
+ * @returns {string} The current log level
  */
+const getLevel = function(module) {
+    return levels[module] || deflt;
+};
 
 /**
- * Creates a new logger object for the provided module
- * @module api/utils/log
- * @param {string} name - Name of the module
- * @returns {Logger} Logger object
- * @example
- * const logger = require('./log.js')('myModule');
- * logger.i('MyModule initialized');
+ * Creates a new logger instance
+ * @param {string} name - The module name
+ * @returns {Object} Logger instance with various methods
  */
 module.exports = function(name) {
     setLevel(name, logLevel(name));
-    // console.log('Got level for ' + name + ': ' + levels[name] + ' ( prefs ', prefs);
+    const logger = createLogger(name, levels[name]);
+
     /**
-      * @type Logger
-      **/
+     * Creates a sub-logger with the parent's name as prefix
+     * @param {string} subname - The sub-logger name
+     * @returns {Object} Sub-logger instance
+     */
+    const createSubLogger = (subname) => {
+        const full = name + ':' + subname;
+        setLevel(full, logLevel(full));
+        const subLogger = createLogger(full, levels[full]);
+
+        return {
+            /**
+             * Returns the full identifier of this sub-logger
+             * @returns {string} Full logger identifier
+             */
+            id: () => full,
+
+            /**
+             * Logs a debug message
+             * @param {...*} args - Message and optional parameters
+             */
+            d: createLogFunction(subLogger, full, 'd'),
+
+            /**
+             * Logs an info message
+             * @param {...*} args - Message and optional parameters
+             */
+            i: createLogFunction(subLogger, full, 'i'),
+
+            /**
+             * Logs a warning message
+             * @param {...*} args - Message and optional parameters
+             */
+            w: createLogFunction(subLogger, full, 'w'),
+
+            /**
+             * Logs an error message
+             * @param {...*} args - Message and optional parameters
+             */
+            e: createLogFunction(subLogger, full, 'e'),
+
+            /**
+             * Conditionally executes a function based on current log level
+             * @param {string} l - Log level code
+             * @param {Function} fn - Function to execute if level is enabled
+             * @param {string} [fl] - Fallback log level
+             * @param {...*} fargs - Arguments for fallback
+             * @returns {boolean} True if the function was executed
+             */
+            f: function(l, fn, fl, ...fargs) {
+                if (ACCEPTABLE[l].indexOf(levels[full] || deflt) !== -1) {
+                    fn(createLogFunction(subLogger, full, l));
+                    return true;
+                }
+                else if (fl) {
+                    this[fl].apply(this, fargs);
+                }
+            },
+
+            /**
+             * Creates a nested sub-logger
+             * @param {string} subname - The nested sub-logger name
+             * @returns {Object} Nested sub-logger instance
+             */
+            sub: createSubLogger
+        };
+    };
+
     return {
         /**
-           * Get logger id
-           * @returns {string} id of this logger
-           * @example
-           * const loggerId = logger.id();
-           * console.log(`Current logger ID: ${loggerId}`);
-           */
+         * Returns the identifier of this logger
+         * @returns {string} Logger identifier
+         */
         id: () => name,
+
         /**
-           * Log debug level messages
-           * @param {...*} var_args - string and values to format string with
-           * @example
-           * logger.d('Debug message: %s', 'Some debug info');
-           */
-        d: log(NAMES.d, name, getEnabledWithLevel(ACCEPTABLE.d, name), this, console.log),
+         * Logs a debug message
+         * @param {...*} args - Message and optional parameters
+         */
+        d: createLogFunction(logger, name, 'd'),
 
         /**
-         * Log information level messages
-         * @param {...*} var_args - string and values to format string with
-         * @example
-         * logger.i('Info message: User %s logged in', username);
+         * Logs an info message
+         * @param {...*} args - Message and optional parameters
          */
-        i: log(NAMES.i, name, getEnabledWithLevel(ACCEPTABLE.i, name), this, console.info),
+        i: createLogFunction(logger, name, 'i'),
 
         /**
-         * Log warning level messages
-         * @param {...*} var_args - string and values to format string with
-         * @example
-         * logger.w('Warning: %d attempts failed', attempts);
+         * Logs a warning message
+         * @param {...*} args - Message and optional parameters
          */
-        w: log(NAMES.w, name, getEnabledWithLevel(ACCEPTABLE.w, name), this, console.warn, styles.stylers.warn),
+        w: createLogFunction(logger, name, 'w'),
 
         /**
-         * Log error level messages
-         * @param {...*} var_args - string and values to format string with
-         * @example
-         * logger.e('Error occurred: %o', errorObject);
+         * Logs an error message
+         * @param {...*} args - Message and optional parameters
          */
-        e: log(NAMES.e, name, getEnabledWithLevel(ACCEPTABLE.e, name), this, console.error, styles.stylers.error),
+        e: createLogFunction(logger, name, 'e'),
 
         /**
-         * Log variable level messages (for cases when logging parameters calculation are expensive enough and shouldn't be done unless the level is enabled)
-         * @param {string} l - log level (d, i, w, e)
-         * @param {function} fn - function to call with single argument - logging function
-         * @param {string} fl - fallback level if l is disabled
-         * @param {...*} fargs - fallback level arguments
-         * @returns {boolean} true if f() has been called
-         * @example
-         * logger.f('d', (log) => {
-         *   const expensiveOperation = performExpensiveCalculation();
-         *   log('Debug: Expensive operation result: %j', expensiveOperation);
-         * }, 'i', 'Skipped expensive debug logging');
+         * Conditionally executes a function based on current log level
+         * @param {string} l - Log level code
+         * @param {Function} fn - Function to execute if level is enabled
+         * @param {string} [fl] - Fallback log level
+         * @param {...*} fargs - Arguments for fallback
+         * @returns {boolean} True if the function was executed
          */
         f: function(l, fn, fl, ...fargs) {
             if (ACCEPTABLE[l].indexOf(levels[name] || deflt) !== -1) {
-                fn(log(NAMES[l], name, getEnabledWithLevel(ACCEPTABLE[l], name), this, l === 'e' ? console.error : l === 'w' ? console.warn : console.log, l === 'w' ? styles.stylers.warn : l === 'e' ? styles.stylers.error : undefined));
+                fn(createLogFunction(logger, name, l));
                 return true;
             }
             else if (fl) {
@@ -325,42 +376,32 @@ module.exports = function(name) {
         },
 
         /**
-         * Logging inside callbacks
-         * @param {function=} next - next function to call, after callback executed
-         * @returns {function} function to pass as callback
-         * @example
-         * const logCallback = logger.callback((result) => {
-         *   console.log('Operation completed with result:', result);
-         * });
-         * someAsyncOperation(logCallback);
+         * Creates a callback function that logs errors
+         * @param {Function} [next] - Function to call on success
+         * @returns {Function} Callback function
          */
         callback: function(next) {
-            var self = this;
+            const self = this;
             return function(err) {
                 if (err) {
                     self.e(err);
                 }
                 else if (next) {
-                    var args = Array.prototype.slice.call(arguments, 1);
+                    const args = Array.prototype.slice.call(arguments, 1);
                     next.apply(this, args);
                 }
             };
         },
+
         /**
-         * Logging database callbacks
-         * @param {string} opname - name of the performed operation
-         * @param {function=} next - next function to call, after callback executed
-         * @param {function=} nextError - function to pass error to
-         * @returns {function} function to pass as callback
-         * @example
-         * const dbCallback = logger.logdb('insert user',
-         *   (result) => { console.log('User inserted:', result); },
-         *   (error) => { console.error('Failed to insert user:', error); }
-         * );
-         * database.insertUser(userData, dbCallback);
+         * Creates a database operation callback that logs results
+         * @param {string} opname - Operation name
+         * @param {Function} [next] - Function to call on success
+         * @param {Function} [nextError] - Function to call on error
+         * @returns {Function} Database callback function
          */
         logdb: function(opname, next, nextError) {
-            var self = this;
+            const self = this;
             return function(err) {
                 if (err) {
                     self.e('Error while %j: %j', opname, err);
@@ -376,98 +417,39 @@ module.exports = function(name) {
                 }
             };
         },
+
         /**
-         * Add one more level to the logging output while leaving loglevel the same
-         * @param {string} subname - sublogger name
-         * @returns {Logger} new logger
-         * @example
-         * const subLogger = logger.sub('database');
-         * subLogger.i('Connected to database');
+         * Creates a sub-logger with the current logger's name as prefix
+         * @param {string} subname - The sub-logger name
+         * @returns {Object} Sub-logger instance
          */
-
-        sub: function(subname) {
-            let full = name + ':' + subname,
-                self = this;
-
-            setLevel(full, logLevel(full));
-
-            return {
-                /**
-                 * Get logger id
-                 * @returns {string} id of this logger
-                 */
-                id: () => full,
-                /**
-                * Log debug level messages
-                * @memberof module:api/utils/log~Logger
-                * @param {...*} var_args - string and values to format string with
-                **/
-                d: log(NAMES.d, full, getEnabledWithLevel(ACCEPTABLE.d, full), this, console.log),
-
-                /**
-                * Log information level messages
-                * @memberof module:api/utils/log~Logger
-                * @param {...*} var_args - string and values to format string with
-                **/
-                i: log(NAMES.i, full, getEnabledWithLevel(ACCEPTABLE.i, full), this, console.info),
-
-                /**
-                * Log warning level messages
-                * @memberof module:api/utils/log~Logger
-                * @param {...*} var_args - string and values to format string with
-                **/
-                w: log(NAMES.w, full, getEnabledWithLevel(ACCEPTABLE.w, full), this, console.warn, styles.stylers.warn),
-
-                /**
-                * Log error level messages
-                * @memberof module:api/utils/log~Logger
-                * @param {...*} var_args - string and values to format string with
-                **/
-                e: log(NAMES.e, full, getEnabledWithLevel(ACCEPTABLE.e, full), this, console.error, styles.stylers.error),
-
-                /**
-                 * Log variable level messages (for cases when logging parameters calculation are expensive enough and shouldn't be done unless the level is enabled)
-                 * @param {String} l log level (d, i, w, e)
-                 * @param {function} fn function to call with single argument - logging function
-                 * @param {String} fl fallback level if l is disabled
-                 * @param {any[]} fargs fallback level arguments
-                 * @returns {boolean} true if f() has been called
-                 */
-                f: function(l, fn, fl, ...fargs) {
-                    if (ACCEPTABLE[l].indexOf(levels[name] || deflt) !== -1) {
-                        fn(log(NAMES[l], full, getEnabledWithLevel(ACCEPTABLE[l], full), this, l === 'e' ? console.error : l === 'w' ? console.warn : console.log, l === 'w' ? styles.stylers.warn : l === 'e' ? styles.stylers.error : undefined));
-                        return true;
-                    }
-                    else if (fl) {
-                        this[fl].apply(this, fargs);
-                    }
-                },
-
-                /**
-                 * Pass sub one level up
-                 */
-                sub: self.sub.bind(self)
-            };
-        }
+        sub: createSubLogger
     };
-    // return {
-    //  d: log('DEBUG\t', getEnabledWithLevel(ACCEPTABLE.d, name), this, debug(name)),
-    //  i: log('INFO\t', getEnabledWithLevel(ACCEPTABLE.i, name), this, debug(name)),
-    //  w: log('WARN\t', getEnabledWithLevel(ACCEPTABLE.w, name), this, debug(name)),
-    //  e: log('ERROR\t', getEnabledWithLevel(ACCEPTABLE.e, name), this, debug(name)),
-    //  callback: function(next){
-    //      var self = this;
-    //      return function(err) {
-    //          if (err) { self.e(err); }
-    //          else if (next) {
-    //              var args = Array.prototype.slice.call(arguments, 1);
-    //              next.apply(this, args);
-    //          }
-    //      };
-    //  },
-    // };
 };
 
+// Export static methods
+/**
+ * Sets logging level for a specific module
+ * @param {string} module - The module name
+ * @param {string} level - The log level
+ */
 module.exports.setLevel = setLevel;
+
+/**
+ * Sets default logging level for all modules without explicit configuration
+ * @param {string} level - The log level
+ */
 module.exports.setDefault = setDefault;
+
+/**
+ * Gets current logging level for a module
+ * @param {string} module - The module name
+ * @returns {string} The current log level
+ */
 module.exports.getLevel = getLevel;
+
+/**
+ * Indicates if OpenTelemetry integration is available
+ * @type {boolean}
+ */
+module.exports.hasOpenTelemetry = Boolean(trace && metrics);
\ No newline at end of file
diff --git a/package.json b/package.json
index 1b9ab30bb1d..763c4b747f4 100644
--- a/package.json
+++ b/package.json
@@ -28,7 +28,7 @@
   "devDependencies": {
     "@stylistic/eslint-plugin": "^2.11.0",
     "docdash": "^2.0.1",
-    "eslint-plugin-vue": "^9.31.0",
+    "eslint-plugin-vue": "^9.32.0",
     "grunt-contrib-watch": "1.1.0",
     "grunt-eslint": "24.3.0",
     "grunt-mocha-nyc": "1.0.3",
@@ -96,6 +96,7 @@
     "nodemailer": "6.10.0",
     "object-hash": "3.0.0",
     "offline-geocoder": "git+https://github.com/Countly/offline-geocoder.git",
+    "pino": "9.6.0",
     "properties-parser": "0.6.0",
     "puppeteer": "^23.8.0",
     "sass": "1.83.4",

From 386b7872f52c31ad4e4e63ae4013e1519d9cfbd2 Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Tue, 4 Mar 2025 23:34:16 +0530
Subject: [PATCH 89/90] fixes to job ui

---
 .../core/jobs/javascripts/countly.views.js    | 152 ++++--
 .../core/jobs/templates/jobs-details.html     | 330 +++++++-----
 .../public/core/jobs/templates/jobs.html      | 108 ++--
 .../dashboard/dashboard.properties            |  29 +
 jobServer/JobManager.js                       |   2 +-
 jobServer/api.js                              | 504 ++++++++++++++++--
 .../jobRunner/impl/pulse/PulseJobScheduler.js |   4 +-
 7 files changed, 855 insertions(+), 274 deletions(-)

diff --git a/frontend/express/public/core/jobs/javascripts/countly.views.js b/frontend/express/public/core/jobs/javascripts/countly.views.js
index 11bb30ce912..713320a5c67 100644
--- a/frontend/express/public/core/jobs/javascripts/countly.views.js
+++ b/frontend/express/public/core/jobs/javascripts/countly.views.js
@@ -8,18 +8,40 @@
      */
     var getColor = function(row) {
         // row is the merged job object
-        if (row.status === "RUNNING") {
-            return "green";
+        // Use _originalStatus if available, otherwise fall back to status
+        var status = row._originalStatus || row.status;
+        if (status) {
+            status = status.toUpperCase(); // Convert to uppercase for consistent comparison
         }
-        else if (row.status === "SCHEDULED" && row.enabled) {
-            return "yellow";
+
+        // Use _originalLastRunStatus if available, otherwise fall back to lastRunStatus
+        var lastRunStatus = row._originalLastRunStatus || row.lastRunStatus;
+        if (lastRunStatus) {
+            lastRunStatus = lastRunStatus.toUpperCase(); // Convert to uppercase for consistent comparison
         }
-        else if (row.status === "SUSPENDED" || !row.enabled) {
+
+        // Check if job is disabled via config.enabled
+        if (!row.config || !row.config.enabled) {
             return "gray";
         }
-        else if (row.status === "CANCELLED" || row.lastRunStatus === "failed") {
+
+        // Backend uses "COMPLETED", "FAILED", "RUNNING", "SCHEDULED" (see getJobStatus in api.js)
+        // But also "success", "failed", "pending" (see getRunStatus in api.js)
+        switch (status) {
+        case "RUNNING": return "green";
+        case "COMPLETED": return "green";
+        case "SUCCESS": return "green";
+        case "SCHEDULED":
+        case "PENDING": return "yellow";
+        case "SUSPENDED": return "gray";
+        case "CANCELLED": return "red";
+        }
+
+        // If status doesn't match, check lastRunStatus
+        if (lastRunStatus === "FAILED") {
             return "red";
         }
+
         return "gray";
     };
 
@@ -29,9 +51,55 @@
      * @returns {void}
      */
     var updateScheduleRow = function(row) {
-        row.nextRunDate = row.nextRunAt ? moment(row.nextRunAt).format('YYYY-MM-DD') : '';
-        row.nextRunTime = row.nextRunAt ? moment(row.nextRunAt).format('HH:mm:ss') : '';
-        row.lastRun = row.lastFinishedAt ? moment(row.lastFinishedAt).fromNow() : '';
+        // Store original values for sorting
+        if (row.name !== undefined) {
+            row._sortName = String(row.name);
+        }
+
+        if (row.status !== undefined) {
+            // Store the original status value for color determination and sorting
+            row._originalStatus = row.status;
+            row._sortStatus = String(row.status);
+        }
+
+        // Add sortBy properties for date fields to ensure correct sorting
+        if (row.nextRunAt) {
+            var nextRunMoment = moment(row.nextRunAt);
+            row.nextRunDate = nextRunMoment.format('YYYY-MM-DD');
+            row.nextRunTime = nextRunMoment.format('HH:mm:ss');
+
+            // Store original value for sorting
+            row._sortNextRunAt = new Date(row.nextRunAt).getTime();
+        }
+        else {
+            row.nextRunDate = '';
+            row.nextRunTime = '';
+        }
+
+        if (row.lastFinishedAt) {
+            var lastFinishedMoment = moment(row.lastFinishedAt);
+            row.lastRun = lastFinishedMoment.fromNow();
+
+            // Store original value for sorting
+            row._sortLastFinishedAt = new Date(row.lastFinishedAt).getTime();
+        }
+        else {
+            row.lastRun = '';
+        }
+
+        if (row.scheduleLabel !== undefined) {
+            row._sortScheduleLabel = String(row.scheduleLabel);
+        }
+
+        if (row.lastRunStatus !== undefined) {
+            // Store the original lastRunStatus value for color determination and sorting
+            row._originalLastRunStatus = row.lastRunStatus;
+            row._sortLastRunStatus = String(row.lastRunStatus);
+        }
+
+        if (row.total !== undefined) {
+            row._sortTotal = Number(row.total);
+        }
 
         // If the row has .config.defaultConfig.schedule.value, use it as its "default schedule"
         if (row.config && row.config.defaultConfig && row.config.defaultConfig.schedule) {
@@ -53,15 +121,7 @@
             var tableStore = countlyVue.vuex.getLocalStore(
                 countlyVue.vuex.ServerDataTable("jobsTable", {
                     // columns: ['name', "schedule", "next", "finished", "status", "total"],
-                    columns: [
-                        "name",
-                        "status",
-                        "scheduleLabel",
-                        "nextRunAt",
-                        "lastFinishedAt",
-                        "lastRunStatus",
-                        "total"
-                    ],
+                    columns: ["name", "status", "scheduleLabel", "nextRunAt", "lastFinishedAt", "lastRunStatus", "total"],
 
                     onRequest: function() {
                         // Called before making the request
@@ -134,7 +194,7 @@
                     return 'gray';
                 }
                 if (details.currentState.status === 'RUNNING') {
-                    return 'blue';
+                    return 'green';
                 }
                 if (details.currentState.status === 'FAILED') {
                     return 'red';
@@ -142,17 +202,32 @@
                 if (details.currentState.status === 'COMPLETED') {
                     return 'green';
                 }
+                if (details.currentState.status === 'PENDING') {
+                    return 'yellow';
+                }
                 return 'yellow';
             },
             getRunStatusColor(status) {
                 // For run status
-                if (status === 'success') {
-                    return 'green';
+                // Use _originalLastRunStatus if available
+                var statusValue = status && status._originalLastRunStatus ? status._originalLastRunStatus : status;
+
+                // Convert to uppercase for consistent comparison with backend values
+                if (typeof statusValue === 'string') {
+                    statusValue = statusValue.toUpperCase();
                 }
-                if (status === 'failed') {
-                    return 'red';
+
+                // Backend uses "COMPLETED", "FAILED", "RUNNING", "SCHEDULED" (see getJobStatus in api.js)
+                // But also "success", "failed", "pending" (see getRunStatus in api.js)
+                switch (statusValue) {
+                case "SUCCESS":
+                case "COMPLETED": return 'green';
+                case "RUNNING": return 'green';
+                case "FAILED": return 'red';
+                case "PENDING":
+                case "SCHEDULED": return 'yellow';
+                default: return 'gray';
                 }
-                return 'gray';
             },
             refresh: function(force) {
                 if (this.loaded || force) {
@@ -356,12 +431,13 @@
              */
             getStatusColor: function(jobDetails) {
                 if (!jobDetails.config?.enabled) {
-                    return "grey";
+                    return "gray";
                 }
                 switch (jobDetails.currentState.status) {
-                case "RUNNING": return "blue";
+                case "RUNNING": return "green";
                 case "FAILED": return "red";
                 case "COMPLETED": return "green";
+                case "PENDING": return "yellow";
                 default: return "yellow";
                 }
             },
@@ -371,11 +447,25 @@
              * @returns {string} Color code for the status
              */
             getRunStatusColor: function(run) {
-                switch (run.status) {
-                case "success": return "green";
-                case "failed": return "red";
-                case "running": return "blue";
-                default: return "yellow";
+                // Handle both string and object status
+                // Use _originalLastRunStatus if available
+                var status = run._originalLastRunStatus || run.status;
+
+                // Convert to uppercase for consistent comparison with backend values
+                if (typeof status === 'string') {
+                    status = status.toUpperCase();
+                }
+
+                // Backend uses "COMPLETED", "FAILED", "RUNNING", "SCHEDULED" (see getJobStatus in api.js)
+                // But also "success", "failed", "pending" (see getRunStatus in api.js)
+                switch (status) {
+                case "SUCCESS":
+                case "COMPLETED": return "green";
+                case "RUNNING": return "green";
+                case "FAILED": return "red";
+                case "PENDING":
+                case "SCHEDULED": return "yellow";
+                default: return "gray";
                 }
             }
         },
diff --git a/frontend/express/public/core/jobs/templates/jobs-details.html b/frontend/express/public/core/jobs/templates/jobs-details.html
index 46470044516..5a428449037 100644
--- a/frontend/express/public/core/jobs/templates/jobs-details.html
+++ b/frontend/express/public/core/jobs/templates/jobs-details.html
@@ -2,44 +2,54 @@
 <div v-bind:class="[componentId]">
     <cly-header :title="job_name">
         <template v-slot:header-top>
-            <cly-back-link :title="i18n('plugins.back')"></cly-back-link>
+            <cly-back-link :title="i18n('jobs.back-to-jobs-list')"></cly-back-link>
         </template>
     </cly-header>
 
     <cly-main>
         <!-- Wait until jobDetails is loaded -->
         <template v-if="jobDetails">
+            <!-- Job Configuration Header -->
+            <h5 class="bu-is-size-5 bu-mb-4 bu-mt-5">
+                {{ i18n('jobs.job-configuration') }}
+            </h5>
+            
             <!-- Job Overview Section -->
-            <cly-section>
+            <cly-section class="bu-pt-3">
                 <div class="bu-columns">
                     <div class="bu-column">
-                        <h4 class="bu-is-size-4 bu-mb-4">
-                            {{ i18n('jobs.job-configuration') }}
-                        </h4>
 
-                        <div class="bu-box">
+                        <div class="bu-box bu-has-shadow-sm bu-p-5">
                             <!-- Base Configuration -->
                             <div class="bu-mb-4">
-                                <h5 class="bu-is-size-5 bu-mb-2">
+                                <h5 class="bu-is-size-5 bu-mb-3 bu-has-text-weight-medium">
                                     {{ i18n('jobs.base-configuration') }}
                                 </h5>
-                                <p class="bu-mb-2">
-                                    <strong>{{ i18n('jobs.status') }}:</strong>
-                                    <cly-status-tag
-                                        :text="jobDetails.config?.enabled
-                                               ? jobDetails.currentState?.status
-                                               : 'DISABLED'"
-                                        :color="getStatusColor(jobDetails)"
-                                    >
-                                    </cly-status-tag>
-                                </p>
-                                <p class="bu-mb-2">
-                                    <strong>{{ i18n('jobs.default-schedule') }}:</strong>
-                                    {{ jobDetails.config?.defaultConfig?.schedule?.value }}
-                                </p>
-                                <p class="bu-mb-2 bu-has-text-grey">
-                                    {{ jobDetails.config?.scheduleLabel }}
-                                </p>
+                                <div class="bu-field bu-mb-3 bu-is-flex">
+                                    <label class="bu-label bu-has-text-weight-medium bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.status') }}:</label>
+                                    <div class="bu-control bu-is-flex-grow-1">
+                                        <cly-status-tag
+                                            :text="jobDetails.config && jobDetails.config.enabled !== false
+                                                ? (jobDetails.currentState && jobDetails.currentState.status ? jobDetails.currentState.status : i18n('common.unknown'))
+                                                : i18n('jobs.disable').toUpperCase()"
+                                            :color="getStatusColor(jobDetails)"
+                                        >
+                                        </cly-status-tag>
+                                    </div>
+                                </div>
+                                <div class="bu-field bu-mb-3 bu-is-flex">
+                                    <label class="bu-label bu-has-text-weight-medium bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.default-schedule') }}:</label>
+                                    <div class="bu-control bu-is-flex-grow-1 bu-is-flex bu-is-align-items-center">
+                                        <span class="bu-tag bu-is-info bu-is-light bu-mr-2">
+                                            {{ jobDetails.config && jobDetails.config && jobDetails.config.schedule
+                                                ? jobDetails.config.schedule
+                                                : i18n('common.never') }}
+                                        </span>
+                                        <p class="bu-has-text-grey bu-is-italic" v-if="jobDetails.config && jobDetails.config.scheduleLabel">
+                                            {{ jobDetails.config.scheduleLabel }}
+                                        </p>
+                                    </div>
+                                </div>
                             </div>
 
                             <!-- Configuration Overrides -->
@@ -47,140 +57,184 @@ <h5 class="bu-is-size-5 bu-mb-2">
                                 class="bu-mb-4"
                                 v-if="hasOverrides"
                             >
-                                <h5 class="bu-is-size-5 bu-mb-2">
+                                <h5 class="bu-is-size-5 bu-mb-3 bu-has-text-weight-medium">
                                     {{ i18n('jobs.active-overrides') }}
                                 </h5>
-                                <p
-                                    class="bu-mb-2"
-                                    v-if="jobDetails.config?.scheduleOverride"
+                                <div
+                                    class="bu-field bu-mb-3 bu-is-flex"
+                                    v-if="jobDetails.config && jobDetails.config.scheduleOverride"
                                 >
-                                    <strong>{{ i18n('jobs.schedule-override') }}:</strong>
-                                    <el-tag
-                                        size="small"
-                                        type="warning"
-                                    >
-                                        {{ jobDetails.config.scheduleOverride }}
-                                    </el-tag>
-                                </p>
-                                <p
-                                    class="bu-mb-2"
-                                    v-if="jobDetails.config?.retryOverride"
+                                    <label class="bu-label bu-has-text-weight-medium bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.schedule-override') }}:</label>
+                                    <div class="bu-control bu-is-flex-grow-1 bu-is-flex bu-is-align-items-center">
+                                        <span class="bu-tag bu-is-warning bu-is-light bu-mr-2">
+                                            {{ jobDetails.config.scheduleOverride }}
+                                        </span>
+                                        <p class="bu-has-text-grey bu-is-italic" v-if="jobDetails.config && jobDetails.config.scheduleOverrideLabel">
+                                            {{ jobDetails.config.scheduleOverrideLabel }}
+                                        </p>
+                                    </div>
+                                </div>
+                                <div
+                                    class="bu-field bu-mb-3"
+                                    v-if="jobDetails.config && jobDetails.config.retryOverride"
                                 >
-                                    <strong>{{ i18n('jobs.retry-override') }}:</strong>
-                                    <br>
-                                    {{ i18n('jobs.attempts') }}:
-                                    {{ jobDetails.config.retryOverride.attempts }}
-                                    <br>
-                                    {{ i18n('jobs.delay') }}:
-                                    {{ jobDetails.config.retryOverride.delay }}ms
-                                </p>
+                                    <div class="bu-is-flex">
+                                        <label class="bu-label bu-has-text-weight-medium bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.retry-override') }}:</label>
+                                        <div class="bu-control bu-is-flex-grow-1">
+                                            <div class="bu-notification bu-is-light bu-is-info bu-p-3 bu-mt-2">
+                                                <div class="bu-mb-2 bu-is-flex bu-is-align-items-center">
+                                                    <strong class="bu-mr-2">{{ i18n('jobs.attempts') }}:</strong>
+                                                    <span class="bu-tag bu-is-info bu-is-light">{{ jobDetails.config.retryOverride.attempts }}</span>
+                                                </div>
+                                                <div class="bu-is-flex bu-is-align-items-center">
+                                                    <strong class="bu-mr-2">{{ i18n('jobs.delay') }}:</strong>
+                                                    <span class="bu-tag bu-is-info bu-is-light">{{ jobDetails.config.retryOverride.delay }}ms</span>
+                                                </div>
+                                            </div>
+                                        </div>
+                                    </div>
+                                </div>
                             </div>
 
                             <!-- Current State -->
                             <div class="bu-mb-4">
-                                <h5 class="bu-is-size-5 bu-mb-2">
+                                <h5 class="bu-is-size-5 bu-mb-3 bu-has-text-weight-medium">
                                     {{ i18n('jobs.current-state') }}
                                 </h5>
-                                <p class="bu-mb-2">
-                                    <strong>{{ i18n('jobs.last-run') }}:</strong>
-                                    {{ jobDetails.currentState?.lastRun }}
-                                </p>
-                                <p class="bu-mb-2">
-                                    <strong>{{ i18n('jobs.next-run') }}:</strong>
-                                    {{ jobDetails.currentState?.nextRun }}
-                                </p>
-                                <p
-                                    class="bu-mb-2"
-                                    v-if="jobDetails.currentState?.failReason"
+                                <div class="bu-field bu-mb-3 bu-is-flex">
+                                    <label class="bu-label bu-has-text-weight-medium bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.last-run') }}:</label>
+                                    <div class="bu-control bu-is-flex-grow-1">
+                                        <span class="bu-tag bu-is-light">
+                                            {{ jobDetails.currentState && jobDetails.currentState.lastRun ? formatDateTime(jobDetails.currentState.lastRun) : i18n('common.never') }}
+                                        </span>
+                                    </div>
+                                </div>
+                                <div class="bu-field bu-mb-3 bu-is-flex">
+                                    <label class="bu-label bu-has-text-weight-medium bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.next-run') }}:</label>
+                                    <div class="bu-control bu-is-flex-grow-1">
+                                        <span class="bu-tag bu-is-light">
+                                            {{ jobDetails.currentState && jobDetails.currentState.nextRun ? formatDateTime(jobDetails.currentState.nextRun) : i18n('common.not-scheduled') }}
+                                        </span>
+                                    </div>
+                                </div>
+                                <div class="bu-field bu-mb-3 bu-is-flex">
+                                    <label class="bu-label bu-has-text-weight-medium bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.duration') }}:</label>
+                                    <div class="bu-control bu-is-flex-grow-1">
+                                        <span class="bu-tag bu-is-light">
+                                            {{ jobDetails.currentState && jobDetails.currentState.lastRunDuration ? jobDetails.currentState.lastRunDuration + 's' : '-' }}
+                                        </span>
+                                    </div>
+                                </div>
+                                <div
+                                    class="bu-field bu-mb-3"
+                                    v-if="jobDetails.currentState && jobDetails.currentState.failReason"
                                 >
-                                    <strong>{{ i18n('jobs.last-failure') }}:</strong>
-                                    <span class="bu-has-text-danger">
-                                        {{ jobDetails.currentState.failReason }}
-                                    </span>
-                                </p>
+                                    <div class="bu-is-flex">
+                                        <label class="bu-label bu-has-text-weight-medium bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.last-failure') }}:</label>
+                                        <div class="bu-control bu-is-flex-grow-1">
+                                            <div class="bu-notification bu-is-light bu-is-danger bu-p-3 bu-mt-2">
+                                                {{ jobDetails.currentState.failReason }}
+                                            </div>
+                                        </div>
+                                    </div>
+                                </div>
                             </div>
                         </div>
                     </div>
                 </div>
             </cly-section>
 
-            <!-- Job History Section (table of type='normal' runs) -->
-            <cly-section>
-                <h4 class="bu-is-size-4 bu-mb-4">
-                    {{ i18n('jobs.job-history') }}
-
-                </h4>
-                <h3>Single runs of jobs </h3>
-                <cly-datatable-n
-                    :rows="jobRuns"
-                    :columns="jobRunColumns"
-                    :force-loading="isLoading"
-                >
-                    <template v-slot="scope">
-                        <!-- Column: lastRunAt -->
-                        <el-table-column
-                            prop="lastRunAt"
-                            :label="i18n('jobs.run-time')"
-                            sortable="custom"
-                        >
-                            <template v-slot="scope">
-                                {{ formatDateTime(scope.row.lastRunAt) }}
-                            </template>
-                        </el-table-column>
+            <h5 class="bu-is-size-5 bu-mb-4 bu-mt-5">
+                {{ i18n('jobs.manual-job-history') }}
+            </h5>
+            
+            <cly-section class="bu-pt-3">
+                <div class="bu-box bu-has-shadow-sm bu-p-0">
+                    <cly-datatable-n
+                        :rows="jobRuns"
+                        :columns="jobRunColumns"
+                        :force-loading="isLoading"
+                        class="jobs-history-table"
+                    >
+                        <template v-slot="scope">
+                            <!-- Column: lastRunAt -->
+                            <el-table-column
+                                prop="lastRunAt"
+                                :label="i18n('jobs.run-time')"
+                                sortable="custom"
+                                width="180"
+                                header-align="left"
+                            >
+                                <template v-slot="scope">
+                                    <span class="bu-has-text-weight-medium">
+                                        {{ formatDateTime(scope.row.lastRunAt) }}
+                                    </span>
+                                </template>
+                            </el-table-column>
 
-                        <!-- Column: status -->
-                        <el-table-column
-                            prop="status"
-                            :label="i18n('jobs.status')"
-                            sortable="custom"
-                        >
-                            <template v-slot="scope">
-                                <cly-status-tag
-                                    :text="scope.row.status"
-                                    :color="getRunStatusColor(scope.row)"
-                                >
-                                </cly-status-tag>
-                                <el-tooltip
-                                    v-if="scope.row.failReason"
-                                    :content="scope.row.failReason"
-                                    placement="top"
-                                >
-                                    <i
-                                        class="ion-alert-circled bu-ml-2"
-                                        style="color: #d23f00"
-                                    ></i>
-                                </el-tooltip>
-                            </template>
-                        </el-table-column>
+                            <!-- Column: status -->
+                            <el-table-column
+                                prop="status"
+                                :label="i18n('jobs.status')"
+                                sortable="custom"
+                                width="150"
+                                header-align="left"
+                            >
+                                <template v-slot="scope">
+                                    <div class="bu-is-flex bu-is-align-items-center">
+                                        <cly-status-tag
+                                            :text="scope.row.status"
+                                            :color="getRunStatusColor(scope.row)"
+                                        >
+                                        </cly-status-tag>
+                                        <el-tooltip
+                                            v-if="scope.row.failReason"
+                                            :content="scope.row.failReason"
+                                            placement="top"
+                                        >
+                                            <i
+                                                class="ion-alert-circled bu-ml-2"
+                                                style="color: #d23f00"
+                                            ></i>
+                                        </el-tooltip>
+                                    </div>
+                                </template>
+                            </el-table-column>
 
-                        <!-- Column: duration -->
-                        <el-table-column
-                            prop="duration"
-                            :label="i18n('jobs.duration')"
-                            sortable="custom"
-                        >
-                            <template v-slot="scope">
-                                {{ scope.row.duration }}s
-                            </template>
-                        </el-table-column>
+                            <!-- Column: duration -->
+                            <el-table-column
+                                prop="duration"
+                                :label="i18n('jobs.duration')"
+                                sortable="custom"
+                                width="120"
+                                header-align="left"
+                            >
+                                <template v-slot="scope">
+                                    <span class="bu-tag bu-is-info bu-is-light" v-if="scope.row.duration">
+                                        {{ scope.row.duration }}s
+                                    </span>
+                                    <span class="bu-has-text-grey" v-else>-</span>
+                                </template>
+                            </el-table-column>
 
-                        <!-- Column: result -->
-                        <el-table-column
-                            prop="result"
-                            :label="i18n('jobs.result')"
-                        >
-                            <template v-slot="scope">
-                                <pre
-                                    v-if="scope.row.dataAsString"
-                                    class="bu-p-2 bu-has-background-grey-lighter bu-is-family-code"
-                                >
-{{ scope.row.dataAsString }}
-                                </pre>
-                                <span v-else>-</span>
-                            </template>
-                        </el-table-column>
-                    </template>
-                </cly-datatable-n>
+                            <!-- Column: result -->
+                            <el-table-column
+                                prop="result"
+                                :label="i18n('jobs.result')"
+                                header-align="left"
+                            >
+                                <template v-slot="scope">
+                                    <pre
+                                        v-if="scope.row.dataAsString && scope.row.dataAsString !== '{}'"
+                                        class="bu-p-3 bu-has-background-grey-lighter bu-is-family-code bu-has-radius bu-has-shadow-inner"
+                                        style="max-height: 200px; overflow: auto;"
+                                    >{{ scope.row.dataAsString }}</pre>
+                                    <span class="bu-has-text-grey" v-else>{{ i18n('common.no-data') }}</span>
+                                </template>
+                            </el-table-column>
+                        </template>
+                    </cly-datatable-n>
+                </div>
             </cly-section>
         </template>
     </cly-main>
diff --git a/frontend/express/public/core/jobs/templates/jobs.html b/frontend/express/public/core/jobs/templates/jobs.html
index b9d46df961e..cde6f3078ce 100644
--- a/frontend/express/public/core/jobs/templates/jobs.html
+++ b/frontend/express/public/core/jobs/templates/jobs.html
@@ -16,6 +16,8 @@
                 v-on:row-click="goTo"
                 :isClickable="true"
                 :default-sort="{ prop: 'name', order: 'ascending' }"
+                persistKey="jobs-table"
+                class="jobs-table"
             >
                 <template v-slot="scope">
                     <!-- Job Name Column -->
@@ -71,14 +73,19 @@
                                     size="mini"
                                     type="warning"
                                 >
-                                    Overridden
+                                    {{ i18n('jobs.schedule-overridden') }}
                                 </el-tag>
                             </p>
                             <p
                                 :data-test-id="'datatable-jobs-schedule-detail-' + scope.$index"
                                 class="bu-has-text-grey bu-is-size-7"
                             >
-                                {{ scope.row.configuredSchedule || scope.row.schedule }}
+                                <template v-if="scope.row.scheduleOverridden">
+                                    <span class="bu-has-text-weight-bold">{{ i18n('jobs.override') }}:</span> {{ scope.row.configuredSchedule }}
+                                </template>
+                                <template v-else>
+                                    {{ scope.row.schedule }}
+                                </template>
                             </p>
                         </template>
                     </el-table-column>
@@ -139,9 +146,7 @@
                             <span :data-test-id="'datatable-jobs-last-run-status-' + scope.$index">
                                 <cly-status-tag
                                     :text="scope.row.lastRunStatus"
-                                    :color="scope.row.lastRunStatus === 'success'
-                                        ? 'green'
-                                        : (scope.row.lastRunStatus === 'failed' ? 'red' : 'gray')"
+                                    :color="getRunStatusColor(scope.row)"
                                 >
                                 </cly-status-tag>
                             </span>
@@ -152,56 +157,11 @@
                     <el-table-column
                         align="center"
                         type="options"
+                        header-align="center"
+                        width="100"
                     >
                         <template v-slot="scope">
-                            <!--
-                                This is your schedule dialog.
-                                It's placed here so you can reuse the same dialog instance
-                                for each row, opening it via 'handleCommand'.
-                            -->
-                            <el-dialog
-                                :title="i18n('jobs.schedule-configuration')"
-                                :visible.sync="scheduleDialogVisible"
-                                width="30%"
-                            >
-                                <div class="bu-columns">
-                                    <div class="bu-column">
-                                        <div class="config-item">
-                                            <label>{{ i18n('jobs.schedule') }}:</label>
-                                            <el-input
-                                                v-model="selectedJobConfig.schedule"
-                                                :placeholder="selectedJobConfig.defaultSchedule"
-                                            ></el-input>
-
-                                            <div class="bu-mt-2 bu-is-size-7 bu-has-text-grey">
-                                                {{ i18n('jobs.schedule-hint') }}
-                                            </div>
-
-                                            <div
-                                                v-if="selectedJobConfig.scheduleLabel"
-                                                class="bu-mt-2 bu-is-size-7"
-                                            >
-                                                {{ selectedJobConfig.scheduleLabel }}
-                                            </div>
-                                        </div>
-                                    </div>
-                                </div>
 
-                                <span slot="footer" class="dialog-footer">
-                                    <el-button @click="scheduleDialogVisible = false">
-                                        {{ i18n('common.cancel') }}
-                                    </el-button>
-                                    <el-button
-                                        type="primary"
-                                        @click="saveSchedule"
-                                        :loading="saving"
-                                    >
-                                        {{ i18n('common.save') }}
-                                    </el-button>
-                                </span>
-                            </el-dialog>
-
-                            <!-- The 'More Options' dropdown -->
                             <cly-more-options
                                 v-if="canSuspendJob"
                                 @command="handleCommand($event, scope.row)"
@@ -231,6 +191,50 @@
                     </el-table-column>
                 </template>
             </cly-datatable-n>
+            
+            <!-- Schedule dialog moved outside of the table structure -->
+            <el-dialog
+                :title="i18n('jobs.schedule-configuration')"
+                :visible.sync="scheduleDialogVisible"
+                width="30%"
+                append-to-body
+            >
+                <div class="bu-columns">
+                    <div class="bu-column">
+                        <div class="config-item">
+                            <label>{{ i18n('jobs.schedule') }}:</label>
+                            <el-input
+                                v-model="selectedJobConfig.schedule"
+                                :placeholder="selectedJobConfig.defaultSchedule"
+                            ></el-input>
+
+                            <div class="bu-mt-2 bu-is-size-7 bu-has-text-grey">
+                                {{ i18n('jobs.schedule-hint') }}
+                            </div>
+
+                            <div
+                                v-if="selectedJobConfig.scheduleLabel"
+                                class="bu-mt-2 bu-is-size-7"
+                            >
+                                {{ selectedJobConfig.scheduleLabel }}
+                            </div>
+                        </div>
+                    </div>
+                </div>
+
+                <span slot="footer" class="dialog-footer">
+                    <el-button @click="scheduleDialogVisible = false">
+                        {{ i18n('common.cancel') }}
+                    </el-button>
+                    <el-button
+                        type="primary"
+                        @click="saveSchedule"
+                        :loading="saving"
+                    >
+                        {{ i18n('common.save') }}
+                    </el-button>
+                </span>
+            </el-dialog>
         </cly-section>
     </cly-main>
 </div>
diff --git a/frontend/express/public/localization/dashboard/dashboard.properties b/frontend/express/public/localization/dashboard/dashboard.properties
index 0b1a5340f46..7f7529556ff 100644
--- a/frontend/express/public/localization/dashboard/dashboard.properties
+++ b/frontend/express/public/localization/dashboard/dashboard.properties
@@ -169,6 +169,8 @@ common.show-items = Display
 common.items-per-page = Items per page
 common.try-later = Can't perform this operation now, try again later.
 common.never = Never
+common.not-scheduled = Not scheduled
+common.not-configured = Not configured
 common.in.days = in {0} days
 common.in.hours = in {0} hours
 common.in.minutes = in {0} minutes
@@ -200,6 +202,7 @@ common.no-dont-change = No, don't change
 common.no-dont-delete = No, don't delete
 common.no-dont-continue = No, don't continue
 common.no-dont-do-that  = No, don't do that
+common.no-data = No data
 common.last-updated = Last updated
 common.location = Location
 common.enable = Enable
@@ -1159,6 +1162,32 @@ jobs.run-duration = Duration
 jobs.suspend = Suspend
 jobs.suspend-successful-message = {0} suspended successfully
 jobs.schedule = Schedule
+jobs.manual-runs = Manual Job Runs
+jobs.run-time = Run Time
+jobs.duration = Duration
+jobs.result = Result
+jobs.job-configuration = Job Configuration
+jobs.base-configuration = Base Configuration
+jobs.active-overrides = Active Overrides
+jobs.current-state = Current State
+jobs.status = Status
+jobs.default-schedule = Default Schedule
+jobs.schedule-override = Schedule Override
+jobs.retry-override = Retry Override
+jobs.attempts = Attempts
+jobs.delay = Delay
+jobs.next-run = Next Run
+jobs.last-run = Last Run
+jobs.last-failure = Last Failure
+jobs.manual-job-history = Manual Job History
+jobs.schedule-configuration = Schedule Configuration
+jobs.schedule-hint = Enter a cron expression (e.g. "0 */10 * * * *" for every 10 minutes)
+jobs.change-schedule = Change Schedule
+jobs.run-now = Run Now
+jobs.enable = Enable
+jobs.disable = Disable
+jobs.last-run-status = Last Run Status
+jobs.schedule-overridden = Overridden
 
 systemlogs.action.task_manager_task_deleted = Report deleted
 systemlogs.action.task_manager_task_updated = Report updated
diff --git a/jobServer/JobManager.js b/jobServer/JobManager.js
index a7f822f7d52..6b8fcf76b3b 100644
--- a/jobServer/JobManager.js
+++ b/jobServer/JobManager.js
@@ -134,7 +134,7 @@ class JobManager {
             }
 
             if (jobConfig.schedule) {
-                await this.#jobRunner.updateSchedule(jobName, jobConfig.schedule);
+                await this.#jobRunner.updateSchedule(jobName, {type: "schedule", value: jobConfig.schedule});
             }
 
             if (jobConfig.retry) {
diff --git a/jobServer/api.js b/jobServer/api.js
index a012397d31b..3ea724a095b 100644
--- a/jobServer/api.js
+++ b/jobServer/api.js
@@ -21,12 +21,28 @@ function getJobStatus(job) {
     if (job.lockedAt) {
         return "RUNNING";
     }
+
+    // If both failedAt and lastFinishedAt are present, use the most recent one
+    if (job.failedAt && job.lastFinishedAt) {
+        const failedDate = new Date(job.failedAt);
+        const finishedDate = new Date(job.lastFinishedAt);
+
+        if (failedDate > finishedDate) {
+            return "FAILED";
+        }
+        else {
+            return "COMPLETED";
+        }
+    }
+
+    // If only one is present
     if (job.failedAt) {
         return "FAILED";
     }
     if (job.lastFinishedAt) {
         return "COMPLETED";
     }
+
     return "SCHEDULED";
 }
 
@@ -36,12 +52,27 @@ function getJobStatus(job) {
  * @returns {String} "failed", "success", or "pending"
  */
 function getRunStatus(job) {
+    // If both failedAt and lastFinishedAt are present, use the most recent one
+    if (job.failedAt && job.lastFinishedAt) {
+        const failedDate = new Date(job.failedAt);
+        const finishedDate = new Date(job.lastFinishedAt);
+
+        if (failedDate > finishedDate) {
+            return "failed";
+        }
+        else {
+            return "success";
+        }
+    }
+
+    // If only one is present
     if (job.failedAt) {
         return "failed";
     }
     if (job.lastFinishedAt) {
         return "success";
     }
+
     return "pending";
 }
 
@@ -80,15 +111,26 @@ function getScheduleLabel(schedule) {
  * Build the "jobDetails" object for a scheduled doc
  * @param {Object} scheduledDoc doc from pulseJobs, type='single'
  * @param {Object} jobConfig doc from jobConfigs
+ * @param {Object} latestRunData Optional data from the latest run (aggregated from normal docs)
  * @returns {Object} e.g. { config: {...}, currentState: {...} }
  */
-function buildJobDetails(scheduledDoc, jobConfig) {
+function buildJobDetails(scheduledDoc, jobConfig, latestRunData = null) {
     if (!scheduledDoc && !jobConfig) {
         return {};
     }
 
     const enabled = jobConfig?.enabled ?? true;
 
+    // Get the default schedule value
+    const defaultScheduleValue = jobConfig?.defaultConfig?.schedule?.value || scheduledDoc?.repeatInterval;
+
+    // Get the override schedule value if it exists
+    const overrideScheduleValue = jobConfig?.schedule;
+
+    // Generate human-readable labels for both default and override schedules
+    const defaultScheduleLabel = getScheduleLabel(defaultScheduleValue);
+    const overrideScheduleLabel = overrideScheduleValue ? getScheduleLabel(overrideScheduleValue) : null;
+
     // Combine default config from the scheduled doc + overrides from jobConfigs
     const details = {
         config: {
@@ -104,30 +146,43 @@ function buildJobDetails(scheduledDoc, jobConfig) {
             },
 
             enabled,
-            // The "override" fields
-            scheduleOverride: jobConfig?.schedule || null,
+            // Keep original field names for API compatibility
+            schedule: overrideScheduleValue || null,
+            scheduleOverride: overrideScheduleValue || null,
             retryOverride: jobConfig?.retry || null,
-            // The human-friendly label from the scheduled doc's cron
-            scheduleLabel: getScheduleLabel(
-                jobConfig?.defaultConfig?.schedule?.value ||
-                scheduledDoc?.repeatInterval
-            )
+            // The human-friendly labels
+            scheduleLabel: defaultScheduleLabel,
+            scheduleOverrideLabel: overrideScheduleLabel,
+            // Flag to indicate if schedule is overridden
+            hasScheduleOverride: !!overrideScheduleValue
         },
         currentState: {}
     };
 
-    // If we have a scheduledDoc, fill out currentState
-    if (scheduledDoc) {
+    // If we have a scheduledDoc or latestRunData, fill out currentState
+    if (scheduledDoc || latestRunData) {
+        // If we have latestRunData, combine it with scheduledDoc for status determination
+        // This ensures we're using the latest run's status
+        const statusDoc = latestRunData ? {
+            // Prioritize latest run data for status determination
+            lockedAt: latestRunData.lockedAt || scheduledDoc?.lockedAt,
+            failedAt: latestRunData.failedAt || scheduledDoc?.failedAt,
+            lastFinishedAt: latestRunData.lastFinishedAt || scheduledDoc?.lastFinishedAt,
+            lastRunAt: latestRunData.lastRunAt || scheduledDoc?.lastRunAt,
+            failReason: latestRunData.failReason || scheduledDoc?.failReason
+        } : scheduledDoc;
+
+        // Use scheduledDoc for nextRunAt, but latestRunData for everything else when available
         details.currentState = {
-            status: getJobStatus(scheduledDoc),
-            nextRun: scheduledDoc.nextRunAt,
-            lastRun: scheduledDoc.lastFinishedAt,
-            lastRunStatus: getRunStatus(scheduledDoc),
-            failReason: scheduledDoc.failReason,
-            lastRunDuration: formatJobDuration(scheduledDoc.lastRunAt, scheduledDoc.lastFinishedAt),
+            status: getJobStatus(statusDoc),
+            nextRun: scheduledDoc?.nextRunAt,
+            lastRun: statusDoc.lastFinishedAt,
+            lastRunStatus: getRunStatus(statusDoc),
+            failReason: statusDoc.failReason,
+            lastRunDuration: formatJobDuration(statusDoc.lastRunAt, statusDoc.lastFinishedAt),
             // Additional fields if needed
-            finishedCount: scheduledDoc.finishedCount,
-            totalRuns: scheduledDoc.runCount
+            finishedCount: (scheduledDoc?.finishedCount || 0) + (latestRunData?.manualFinishedCount || 0),
+            totalRuns: (scheduledDoc?.runCount || 0) + (latestRunData?.manualRunCount || 0)
         };
     }
 
@@ -136,17 +191,23 @@ function buildJobDetails(scheduledDoc, jobConfig) {
 
 /**
  * Process a "normal" (type='normal') doc for job details table
+ * Ensures all fields needed for status-based sorting are included
  * @param {Object} doc doc from pulseJobs, type='normal'
- * @returns {Object} e.g. { lastRunAt: Date, status: String, duration: String, result: Object, failReason: String, dataAsString: String }
+ * @returns {Object} Job run details with all required timestamps
  */
 function processRunDoc(doc) {
     return {
+        // Include all timestamp fields needed for accurate status-based sorting
         lastRunAt: doc.lastRunAt,
+        lastFinishedAt: doc.lastFinishedAt,
+        lockedAt: doc.lockedAt,
+        failedAt: doc.failedAt,
+        // Computed fields
         status: getJobStatus(doc),
         duration: formatJobDuration(doc.lastRunAt, doc.lastFinishedAt),
         result: doc.result,
         failReason: doc.failReason,
-        dataAsString: JSON.stringify(doc.result || {}, null, 2)
+        dataAsString: doc.result ? JSON.stringify(doc.result, null, 2) : null
     };
 }
 
@@ -163,32 +224,70 @@ function buildListViewJob(mainDoc, jobConfig, groupItem) {
         return { job: {}, config: {} };
     }
 
+    // Get default schedule value
+    const defaultScheduleValue = mainDoc.repeatInterval;
+
+    // Get override schedule value if it exists
+    const overrideScheduleValue = jobConfig?.schedule;
+
+    // Generate human-readable labels
+    const defaultScheduleLabel = getScheduleLabel(defaultScheduleValue);
+    const overrideScheduleLabel = overrideScheduleValue ? getScheduleLabel(overrideScheduleValue) : null;
+
+    // Use the appropriate label based on which schedule is active
+    const effectiveScheduleLabel = overrideScheduleValue ? overrideScheduleLabel : defaultScheduleLabel;
+
     // Merge in config overrides
     const finalConfig = {
         enabled: jobConfig?.enabled ?? true,
 
         // The default config is taken from the doc
         defaultConfig: {
-            schedule: { value: mainDoc.repeatInterval },
+            schedule: { value: defaultScheduleValue },
             retry: { attempts: mainDoc.attempts, delay: mainDoc.backoff }
         },
 
-        // The user overrides
-        schedule: jobConfig?.schedule,
+        // Keep the original field names for API compatibility 
+        schedule: overrideScheduleValue,
         retry: jobConfig?.retry
     };
 
+    // Create a composite document with data from the latest run
+    // This ensures status is determined from the most recent run
+    const latestRunData = {
+        // Take the lockedAt value from either mainDoc or the group data
+        lockedAt: groupItem.lockedAt || mainDoc.lockedAt || null,
+        // Prioritize failedAt and lastFinishedAt from groupItem as they represent max values
+        failedAt: groupItem.failedAt || mainDoc.failedAt || null,
+        lastFinishedAt: groupItem.lastFinishedAt || mainDoc.lastFinishedAt || null,
+        // Use the latest lastRunAt value
+        lastRunAt: groupItem.lastRunAt || mainDoc.lastRunAt || null,
+        // Include failReason if available
+        failReason: groupItem.failedAt ? (groupItem.failReason || mainDoc.failReason) : mainDoc.failReason,
+        // Include counts of normal docs (manual runs)
+        manualRunCount: groupItem.normalDocCount || 0,
+        manualFinishedCount: groupItem.normalFinishedCount || 0
+    };
+
     const jobObj = {
         name: mainDoc.name,
-        status: getJobStatus(mainDoc),
-        schedule: finalConfig.defaultConfig.schedule.value,
-        scheduleLabel: getScheduleLabel(finalConfig.defaultConfig.schedule.value),
-        nextRunAt: mainDoc.nextRunAt,
-        lastFinishedAt: groupItem.lastFinishedAt,
-        lastRunStatus: getRunStatus(mainDoc),
-        failReason: mainDoc.failReason,
-        // "total" can be included if you want. Here it's arbitrary:
-        total: 0
+        // Determine status using the latest run data
+        status: getJobStatus(latestRunData),
+        // Include both the effective schedule and the original values
+        schedule: defaultScheduleValue,
+        configuredSchedule: overrideScheduleValue,
+        scheduleLabel: effectiveScheduleLabel,
+        scheduleOverridden: !!overrideScheduleValue,
+        // Include all timestamp fields from the latest run
+        lockedAt: latestRunData.lockedAt,
+        failedAt: latestRunData.failedAt,
+        lastRunAt: latestRunData.lastRunAt,
+        nextRunAt: mainDoc.nextRunAt || null,
+        lastFinishedAt: latestRunData.lastFinishedAt,
+        lastRunStatus: getRunStatus(latestRunData),
+        failReason: latestRunData.failReason,
+        // Calculate total runs (scheduled + manual)
+        total: (mainDoc.runCount || 0) + (latestRunData.manualRunCount || 0)
     };
 
     return {
@@ -267,7 +366,7 @@ plugins.register('/o/jobs', async function(ob) {
         const db = common.db;
         const jobsCollection = db.collection('pulseJobs');
         const jobConfigsCollection = db.collection('jobConfigs');
-
+        const columns = ["name", "status", "scheduleLabel", "nextRunAt", "lastFinishedAt", "lastRunStatus", "total" ];
         try {
             const {
                 name: jobName,
@@ -281,7 +380,7 @@ plugins.register('/o/jobs', async function(ob) {
             // Basic pagination & sorting
             const skip = Number(iDisplayStart) || 0;
             const limit = Number(iDisplayLength) || 50;
-            const sortColumn = iSortCol_0 || 'nextRunAt';
+            const sortColumn = columns[iSortCol_0] || 'name';
             const sortDir = (sSortDir_0 === 'desc') ? -1 : 1;
 
             // If a specific job name is provided -> detail view
@@ -294,11 +393,72 @@ plugins.register('/o/jobs', async function(ob) {
 
                 // The "normal" docs (type: 'normal')
                 const query = { name: jobName, type: 'normal' };
+
+                // Determine appropriate sorting for the job details view
+                let detailSortQuery = {};
+
+                // If status is requested as sort column, sort by the relevant timestamps
+                if (sortColumn === 'status') {
+                    if (sortDir === 1) { // ascending: SCHEDULED, COMPLETED, FAILED, RUNNING
+                        detailSortQuery = {
+                            lockedAt: -1, // RUNNING last
+                            failedAt: -1, // FAILED before RUNNING
+                            lastFinishedAt: -1 // COMPLETED before FAILED
+                        };
+                    }
+                    else { // descending: RUNNING, FAILED, COMPLETED, SCHEDULED
+                        detailSortQuery = {
+                            lockedAt: 1, // RUNNING first
+                            failedAt: 1, // FAILED after RUNNING
+                            lastFinishedAt: 1 // COMPLETED after FAILED
+                        };
+                    }
+                    // For runs with the same status, sort by lastRunAt to show most recent first
+                    detailSortQuery.lastRunAt = -1;
+                    // Add _id as final sort for absolute stability
+                    detailSortQuery._id = sortDir;
+                }
+                // Handle direct date field sorting
+                else if (['lastRunAt', 'lastFinishedAt'].includes(sortColumn)) {
+                    // For date fields, handle null values consistently
+                    // This can be implemented directly in MongoDB sort
+                    detailSortQuery[sortColumn] = sortDir;
+
+                    // Add consistent secondary sort fields to ensure stable ordering
+                    // This ensures runs with the same value in the sorted column always appear in the same order
+                    if (sortColumn !== 'lastRunAt') {
+                        detailSortQuery.lastRunAt = -1;
+                    }
+                    if (sortColumn !== 'lastFinishedAt') {
+                        detailSortQuery.lastFinishedAt = -1;
+                    }
+
+                    // Add _id as final sort for absolute stability
+                    detailSortQuery._id = sortDir;
+                }
+                // Default to sorting by lastRunAt (most recent first) with consistent secondary sort
+                else {
+                    detailSortQuery.lastRunAt = -1;
+                    detailSortQuery.lastFinishedAt = -1;
+                    detailSortQuery._id = -1; // Consistent secondary sort
+                }
+
+                // Add options for case-insensitive sorting if sorting by name
+                const cursorOptions = {};
+                if (sortColumn === 'name') {
+                    cursorOptions.collation = { locale: 'en', strength: 2 }; // Case-insensitive
+                }
+
                 const normalDocsCursor = jobsCollection.find(query)
-                    .sort({ lastRunAt: -1 })
+                    .sort(detailSortQuery)
                     .skip(skip)
                     .limit(limit);
 
+                // Apply collation if needed
+                if (cursorOptions.collation) {
+                    normalDocsCursor.collation(cursorOptions.collation);
+                }
+
                 const [normalDocs, total] = await Promise.all([
                     normalDocsCursor.toArray(),
                     jobsCollection.countDocuments(query)
@@ -307,8 +467,120 @@ plugins.register('/o/jobs', async function(ob) {
                 // Fetch config override
                 const jobConfig = await jobConfigsCollection.findOne({ jobName });
 
-                // Build jobDetails
-                const jobDetails = buildJobDetails(scheduledDoc, jobConfig);
+                // Aggregate the latest run data with focus on the newest manual run
+                let latestRunData = null;
+
+                if (normalDocs.length > 0 || scheduledDoc) {
+                    // First, find the newest manual run document (type='normal') based on lastRunAt
+                    let newestManualRun = null;
+
+                    if (normalDocs.length > 0) {
+                        // Sort normalDocs by lastRunAt in descending order
+                        const sortedDocs = [...normalDocs].sort((a, b) => {
+                            if (!a.lastRunAt) {
+                                return 1;
+                            } // null/undefined lastRunAt goes last
+                            if (!b.lastRunAt) {
+                                return -1;
+                            }
+                            // If timestamps are exactly equal, use _id for stable sorting
+                            const timeCompare = new Date(b.lastRunAt) - new Date(a.lastRunAt);
+                            if (timeCompare === 0 && a._id && b._id) {
+                                return a._id.toString().localeCompare(b._id.toString());
+                            }
+                            return timeCompare; // newest first
+                        });
+
+                        // The first document is the newest manual run
+                        newestManualRun = sortedDocs[0];
+                    }
+
+                    // Create base latestRunData object
+                    const latestByStatus = {
+                        lockedAt: null,
+                        failedAt: null,
+                        lastFinishedAt: null,
+                        lastRunAt: null,
+                        failReason: null,
+                        // Count manual runs
+                        manualRunCount: normalDocs.length,
+                        manualFinishedCount: normalDocs.filter(doc => doc.lastFinishedAt).length
+                    };
+
+                    // First, apply data from the newest manual run if available
+                    if (newestManualRun) {
+                        if (newestManualRun.lockedAt) {
+                            latestByStatus.lockedAt = newestManualRun.lockedAt;
+                        }
+                        if (newestManualRun.failedAt) {
+                            // Only update failedAt if it's more recent than any existing lastFinishedAt
+                            if (!latestByStatus.lastFinishedAt || new Date(newestManualRun.failedAt) > new Date(latestByStatus.lastFinishedAt)) {
+                                latestByStatus.failedAt = newestManualRun.failedAt;
+                                latestByStatus.failReason = newestManualRun.failReason;
+                            }
+                        }
+                        if (newestManualRun.lastFinishedAt) {
+                            // Update lastFinishedAt and clear failedAt if the successful completion is more recent
+                            latestByStatus.lastFinishedAt = newestManualRun.lastFinishedAt;
+                            if (latestByStatus.failedAt && new Date(newestManualRun.lastFinishedAt) > new Date(latestByStatus.failedAt)) {
+                                latestByStatus.failedAt = null;
+                                latestByStatus.failReason = null;
+                            }
+                        }
+                        if (newestManualRun.lastRunAt) {
+                            latestByStatus.lastRunAt = newestManualRun.lastRunAt;
+                        }
+                    }
+
+                    // Then check if the scheduled doc has more recent timestamps
+                    if (scheduledDoc) {
+                        if (scheduledDoc.lockedAt && (!latestByStatus.lockedAt || new Date(scheduledDoc.lockedAt) > new Date(latestByStatus.lockedAt))) {
+                            latestByStatus.lockedAt = scheduledDoc.lockedAt;
+                        }
+                        if (scheduledDoc.failedAt) {
+                            // Only update failedAt if it's more recent than any existing lastFinishedAt
+                            const failedDate = new Date(scheduledDoc.failedAt);
+                            if ((!latestByStatus.failedAt || failedDate > new Date(latestByStatus.failedAt)) &&
+                                (!latestByStatus.lastFinishedAt || failedDate > new Date(latestByStatus.lastFinishedAt))) {
+                                latestByStatus.failedAt = scheduledDoc.failedAt;
+                                latestByStatus.failReason = scheduledDoc.failReason;
+                            }
+                        }
+                        if (scheduledDoc.lastFinishedAt) {
+                            const finishedDate = new Date(scheduledDoc.lastFinishedAt);
+                            if (!latestByStatus.lastFinishedAt || finishedDate > new Date(latestByStatus.lastFinishedAt)) {
+                                latestByStatus.lastFinishedAt = scheduledDoc.lastFinishedAt;
+
+                                // Clear failed status if the successful completion is more recent
+                                if (latestByStatus.failedAt && finishedDate > new Date(latestByStatus.failedAt)) {
+                                    latestByStatus.failedAt = null;
+                                    latestByStatus.failReason = null;
+                                }
+                            }
+                        }
+                        if (scheduledDoc.lastRunAt && (!latestByStatus.lastRunAt || new Date(scheduledDoc.lastRunAt) > new Date(latestByStatus.lastRunAt))) {
+                            latestByStatus.lastRunAt = scheduledDoc.lastRunAt;
+                        }
+                    }
+
+                    // Look for any running job (lockedAt is set) across all documents
+                    // This takes precedence as we want to show running status immediately
+                    for (const doc of normalDocs) {
+                        if (doc.lockedAt && (!latestByStatus.lockedAt || new Date(doc.lockedAt) > new Date(latestByStatus.lockedAt))) {
+                            latestByStatus.lockedAt = doc.lockedAt;
+                            // Update other fields from this running document
+                            if (doc.lastRunAt) {
+                                latestByStatus.lastRunAt = doc.lastRunAt;
+                            }
+                            // Note: we don't update failedAt/lastFinishedAt because a running job hasn't failed/finished yet
+                        }
+                    }
+
+                    latestRunData = latestByStatus;
+                }
+
+                // Build jobDetails with latest run data
+                const jobDetails = buildJobDetails(scheduledDoc, jobConfig, latestRunData);
 
                 // Process normal docs
                 const processedRuns = normalDocs.map(doc => processRunDoc(doc));
@@ -323,7 +595,7 @@ plugins.register('/o/jobs', async function(ob) {
                 return;
             }
             else {
-                // Listing all unique job names
+                // Set up search based on query parameter
                 const search = sSearch ? { name: new RegExp(sSearch, 'i') } : {};
 
                 // We'll create a pipeline that groups by name. 
@@ -331,11 +603,87 @@ plugins.register('/o/jobs', async function(ob) {
                 const pipeline = [
                     { $match: search },
                     {
-                        $sort: {
-
-                            type: -1, // ensures 'single' (0) is sorted before 'normal' (1)
-                            [sortColumn]: sortDir
-                        }
+                        $sort: (() => {
+                            // Always sort by type first to prioritize 'single' jobs
+                            const sortObj = { type: -1 };
+
+                            // Handle special sorting for status (which is derived from timestamps)
+                            if (sortColumn === 'status') {
+                                // For status sorting, we need to sort by the relevant timestamps
+                                // Status priority: RUNNING (lockedAt) > FAILED (failedAt) > COMPLETED (lastFinishedAt) > SCHEDULED
+                                if (sortDir === 1) { // ascending
+                                    // Prioritize in natural order: SCHEDULED, COMPLETED, FAILED, RUNNING
+                                    sortObj.lockedAt = -1; // RUNNING last (if lockedAt exists)
+                                    sortObj.failedAt = -1; // FAILED before RUNNING
+                                    sortObj.lastFinishedAt = -1; // COMPLETED before FAILED
+                                }
+                                else { // descending
+                                    // Prioritize in reverse order: RUNNING, FAILED, COMPLETED, SCHEDULED
+                                    sortObj.lockedAt = 1; // RUNNING first (if lockedAt exists)
+                                    sortObj.failedAt = 1; // FAILED after RUNNING
+                                    sortObj.lastFinishedAt = 1; // COMPLETED after FAILED
+                                }
+                                // For jobs with the same status, sort by lastRunAt to show most recent first
+                                sortObj.lastRunAt = -1;
+                                // Add name as secondary sort for consistent ordering when status is the same
+                                sortObj.name = sortDir;
+                                // Add _id as tertiary sort for absolute stability
+                                sortObj._id = sortDir;
+                            }
+                            // Handle sorting for timestamp-based fields used in lastRunStatus
+                            else if (sortColumn === 'lastRunStatus') {
+                                if (sortDir === 1) { // ascending: pending, success, failed
+                                    sortObj.failedAt = -1;
+                                    sortObj.lastFinishedAt = -1;
+                                }
+                                else { // descending: failed, success, pending
+                                    sortObj.failedAt = 1;
+                                    sortObj.lastFinishedAt = 1;
+                                }
+                                // For jobs with the same run status, sort by lastRunAt
+                                sortObj.lastRunAt = -1;
+                                // Add name as secondary sort for consistent ordering when status is the same
+                                sortObj.name = sortDir;
+                                // Add _id as tertiary sort for absolute stability
+                                sortObj._id = sortDir;
+                            }
+                            // Handle direct sorting for actual date fields
+                            else if (sortColumn === 'lastFinishedAt' || sortColumn === 'nextRunAt' || sortColumn === 'lastRunAt') {
+                                // For date fields, always sort nulls last regardless of direction
+                                // MongoDB can't do this directly in the sort stage, but we can handle nulls in the group stage
+                                sortObj[sortColumn] = sortDir;
+                                // Add name as secondary sort for consistent ordering when date is the same
+                                sortObj.name = sortDir;
+                                // Add _id as tertiary sort for absolute stability
+                                sortObj._id = sortDir;
+                            }
+                            // Handle sorting when total is selected
+                            else if (sortColumn === 'total') {
+                                // Sort by the combined run count (scheduled + manual)
+                                // This will be calculated later, but we can sort by the initial factors here
+                                sortObj.runCount = sortDir; // From the single doc
+                                sortObj.normalDocCount = sortDir; // From the aggregation
+                                // Add name for secondary sort
+                                sortObj.name = sortDir;
+                                // Add _id as tertiary sort
+                                sortObj._id = sortDir;
+                            }
+                            // Default to name sorting if column isn't directly sortable at db level
+                            else if (sortColumn !== 'name' && sortColumn !== 'scheduleLabel') {
+                                // Sort by name in a case-insensitive manner for consistency
+                                sortObj.name = sortDir;
+                                // Add _id as secondary sort for absolute stability when names are the same
+                                sortObj._id = sortDir;
+                            }
+                            else {
+                                // Sort by name in a case-insensitive manner for consistency
+                                sortObj.name = sortDir;
+                                // Add _id as secondary sort for absolute stability when names are the same
+                                sortObj._id = sortDir;
+                            }
+
+                            return sortObj;
+                        })()
                     },
                     {
                         $group: {
@@ -343,7 +691,38 @@ plugins.register('/o/jobs', async function(ob) {
                             doc: { $first: "$$ROOT" }, // the first doc in each group
                             lastRunAt: { $max: "$lastRunAt" },
                             lastFinishedAt: { $max: "$lastFinishedAt" },
-                            failedAt: { $max: "$failedAt" }
+                            failedAt: { $max: "$failedAt" },
+                            // Add lockedAt to track running jobs
+                            lockedAt: { $max: "$lockedAt" },
+                            // Count documents to help with run counts
+                            normalDocCount: {
+                                $sum: { $cond: [{ $eq: ["$type", "normal"] }, 1, 0] }
+                            },
+                            // Count finished normal docs
+                            normalFinishedCount: {
+                                $sum: {
+                                    $cond: [
+                                        {
+                                            $and: [
+                                                { $eq: ["$type", "normal"] },
+                                                { $ne: ["$lastFinishedAt", null] }
+                                            ]
+                                        },
+                                        1,
+                                        0
+                                    ]
+                                }
+                            },
+                            // Also gather failReason from the document with the latest failedAt
+                            failReason: {
+                                $first: {
+                                    $cond: [
+                                        { $eq: ["$failedAt", { $max: "$failedAt" }] },
+                                        "$failReason",
+                                        null
+                                    ]
+                                }
+                            }
                         }
                     },
                     {
@@ -353,7 +732,11 @@ plugins.register('/o/jobs', async function(ob) {
                             doc: 1,
                             lastRunAt: 1,
                             lastFinishedAt: 1,
-                            failedAt: 1
+                            failedAt: 1,
+                            lockedAt: 1,
+                            failReason: 1,
+                            normalDocCount: 1,
+                            normalFinishedCount: 1
                         }
                     },
                     { $skip: skip },
@@ -364,23 +747,42 @@ plugins.register('/o/jobs', async function(ob) {
                 const allNames = await jobsCollection.distinct('name', search);
                 const totalCount = allNames.length;
 
+                // Check if we need to use a collation for case-insensitive name sorting
+                const options = {
+                    allowDiskUse: true
+                };
+
                 // Run pipeline
-                const rawJobs = await jobsCollection.aggregate(pipeline).toArray();
+                const rawJobs = await jobsCollection.aggregate(pipeline, options).toArray();
 
                 // Fetch jobConfigs in one pass
                 const jobConfigs = await jobConfigsCollection.find({}).toArray();
                 const configMap = {};
                 jobConfigs.forEach(cfg => {
+                    // Store by jobName to match with the name field in pulseJobs
                     configMap[cfg.jobName] = cfg;
                 });
 
                 // Merge data
                 const processed = rawJobs.map(item => {
-                    const name = item.name; // Changed from jobName to name
+                    const name = item.name;
                     const mainDoc = item.doc; // the doc from $first
-                    const jobConfig = configMap[name] || null; // Use name here
-
-                    return buildListViewJob(mainDoc, jobConfig, item);
+                    const jobConfig = configMap[name] || null; // Use name to look up in configMap
+
+                    // Ensure we're getting the latest status data, especially for manual runs
+                    // The group aggregation already provides max values for timestamps,
+                    // but we need to make sure lockedAt reflects current running state
+                    const status = {
+                        lockedAt: item.lockedAt,
+                        failedAt: item.failedAt,
+                        lastFinishedAt: item.lastFinishedAt,
+                        lastRunAt: item.lastRunAt,
+                        failReason: item.failReason,
+                        normalDocCount: item.normalDocCount,
+                        normalFinishedCount: item.normalFinishedCount
+                    };
+
+                    return buildListViewJob(mainDoc, jobConfig, status);
                 });
 
                 // Each entry in final array should be { job: {...}, config: {...} }
@@ -404,4 +806,4 @@ plugins.register('/o/jobs', async function(ob) {
         }
     });
     return true;
-});
+});
\ No newline at end of file
diff --git a/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js b/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js
index 9b5344313cb..f1e3731c955 100644
--- a/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js
+++ b/jobServer/jobRunner/impl/pulse/PulseJobScheduler.js
@@ -126,7 +126,9 @@ class PulseJobScheduler extends IJobScheduler {
             this.#validateScheduleConfig(schedule);
 
             // First remove the existing job
-            await this.pulseRunner.remove({ name: jobName });
+            let job = this.pulseRunner.create(jobName);
+            await job.remove();
+            // await this.pulseRunner.remove({ name: jobName });
 
             // Then create a new schedule based on the type
             switch (schedule.type) {

From 739318ca375e1704716108f0c059b5be715669af Mon Sep 17 00:00:00 2001
From: Kanwar Ujjaval Singh <4216199+kanwarujjaval@users.noreply.github.com>
Date: Wed, 5 Mar 2025 13:02:37 +0530
Subject: [PATCH 90/90] fixes to table in jobs

---
 .../core/jobs/javascripts/countly.views.js    |   3 +-
 .../core/jobs/templates/jobs-details.html     |  28 +-
 .../public/core/jobs/templates/jobs.html      |  17 +-
 .../dashboard/dashboard.properties            |   3 +-
 jobServer/api.js                              | 640 +++++++++---------
 5 files changed, 328 insertions(+), 363 deletions(-)

diff --git a/frontend/express/public/core/jobs/javascripts/countly.views.js b/frontend/express/public/core/jobs/javascripts/countly.views.js
index 713320a5c67..e80e152976f 100644
--- a/frontend/express/public/core/jobs/javascripts/countly.views.js
+++ b/frontend/express/public/core/jobs/javascripts/countly.views.js
@@ -172,7 +172,8 @@
                     enabled: true
                 },
                 tableStore: tableStore,
-                remoteTableDataSource: countlyVue.vuex.getServerDataSource(tableStore, "jobsTable")
+                remoteTableDataSource: countlyVue.vuex.getServerDataSource(tableStore, "jobsTable"),
+                jobsTablePersistKey: "cly-jobs-table"
             };
         },
         computed: {
diff --git a/frontend/express/public/core/jobs/templates/jobs-details.html b/frontend/express/public/core/jobs/templates/jobs-details.html
index 5a428449037..e67a5184964 100644
--- a/frontend/express/public/core/jobs/templates/jobs-details.html
+++ b/frontend/express/public/core/jobs/templates/jobs-details.html
@@ -22,11 +22,11 @@ <h5 class="bu-is-size-5 bu-mb-4 bu-mt-5">
                         <div class="bu-box bu-has-shadow-sm bu-p-5">
                             <!-- Base Configuration -->
                             <div class="bu-mb-4">
-                                <h5 class="bu-is-size-5 bu-mb-3 bu-has-text-weight-medium">
+                                <h5 class="bu-is-size-5 bu-mb-3">
                                     {{ i18n('jobs.base-configuration') }}
                                 </h5>
                                 <div class="bu-field bu-mb-3 bu-is-flex">
-                                    <label class="bu-label bu-has-text-weight-medium bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.status') }}:</label>
+                                    <label class="bu-label bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.status') }}:</label>
                                     <div class="bu-control bu-is-flex-grow-1">
                                         <cly-status-tag
                                             :text="jobDetails.config && jobDetails.config.enabled !== false
@@ -38,11 +38,11 @@ <h5 class="bu-is-size-5 bu-mb-3 bu-has-text-weight-medium">
                                     </div>
                                 </div>
                                 <div class="bu-field bu-mb-3 bu-is-flex">
-                                    <label class="bu-label bu-has-text-weight-medium bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.default-schedule') }}:</label>
+                                    <label class="bu-label bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.default-schedule') }}:</label>
                                     <div class="bu-control bu-is-flex-grow-1 bu-is-flex bu-is-align-items-center">
                                         <span class="bu-tag bu-is-info bu-is-light bu-mr-2">
-                                            {{ jobDetails.config && jobDetails.config && jobDetails.config.schedule
-                                                ? jobDetails.config.schedule
+                                            {{ jobDetails.config && jobDetails.config.defaultConfig && jobDetails.config.defaultConfig.schedule && jobDetails.config.defaultConfig.schedule.value
+                                                ? jobDetails.config.defaultConfig.schedule.value
                                                 : i18n('common.never') }}
                                         </span>
                                         <p class="bu-has-text-grey bu-is-italic" v-if="jobDetails.config && jobDetails.config.scheduleLabel">
@@ -57,14 +57,14 @@ <h5 class="bu-is-size-5 bu-mb-3 bu-has-text-weight-medium">
                                 class="bu-mb-4"
                                 v-if="hasOverrides"
                             >
-                                <h5 class="bu-is-size-5 bu-mb-3 bu-has-text-weight-medium">
+                                <h5 class="bu-is-size-5 bu-mb-3">
                                     {{ i18n('jobs.active-overrides') }}
                                 </h5>
                                 <div
                                     class="bu-field bu-mb-3 bu-is-flex"
                                     v-if="jobDetails.config && jobDetails.config.scheduleOverride"
                                 >
-                                    <label class="bu-label bu-has-text-weight-medium bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.schedule-override') }}:</label>
+                                    <label class="bu-label bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.schedule-override') }}:</label>
                                     <div class="bu-control bu-is-flex-grow-1 bu-is-flex bu-is-align-items-center">
                                         <span class="bu-tag bu-is-warning bu-is-light bu-mr-2">
                                             {{ jobDetails.config.scheduleOverride }}
@@ -79,7 +79,7 @@ <h5 class="bu-is-size-5 bu-mb-3 bu-has-text-weight-medium">
                                     v-if="jobDetails.config && jobDetails.config.retryOverride"
                                 >
                                     <div class="bu-is-flex">
-                                        <label class="bu-label bu-has-text-weight-medium bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.retry-override') }}:</label>
+                                        <label class="bu-label bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.retry-override') }}:</label>
                                         <div class="bu-control bu-is-flex-grow-1">
                                             <div class="bu-notification bu-is-light bu-is-info bu-p-3 bu-mt-2">
                                                 <div class="bu-mb-2 bu-is-flex bu-is-align-items-center">
@@ -98,11 +98,11 @@ <h5 class="bu-is-size-5 bu-mb-3 bu-has-text-weight-medium">
 
                             <!-- Current State -->
                             <div class="bu-mb-4">
-                                <h5 class="bu-is-size-5 bu-mb-3 bu-has-text-weight-medium">
+                                <h5 class="bu-is-size-5 bu-mb-3">
                                     {{ i18n('jobs.current-state') }}
                                 </h5>
                                 <div class="bu-field bu-mb-3 bu-is-flex">
-                                    <label class="bu-label bu-has-text-weight-medium bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.last-run') }}:</label>
+                                    <label class="bu-label bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.last-run') }}:</label>
                                     <div class="bu-control bu-is-flex-grow-1">
                                         <span class="bu-tag bu-is-light">
                                             {{ jobDetails.currentState && jobDetails.currentState.lastRun ? formatDateTime(jobDetails.currentState.lastRun) : i18n('common.never') }}
@@ -110,7 +110,7 @@ <h5 class="bu-is-size-5 bu-mb-3 bu-has-text-weight-medium">
                                     </div>
                                 </div>
                                 <div class="bu-field bu-mb-3 bu-is-flex">
-                                    <label class="bu-label bu-has-text-weight-medium bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.next-run') }}:</label>
+                                    <label class="bu-label bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.next-run') }}:</label>
                                     <div class="bu-control bu-is-flex-grow-1">
                                         <span class="bu-tag bu-is-light">
                                             {{ jobDetails.currentState && jobDetails.currentState.nextRun ? formatDateTime(jobDetails.currentState.nextRun) : i18n('common.not-scheduled') }}
@@ -118,7 +118,7 @@ <h5 class="bu-is-size-5 bu-mb-3 bu-has-text-weight-medium">
                                     </div>
                                 </div>
                                 <div class="bu-field bu-mb-3 bu-is-flex">
-                                    <label class="bu-label bu-has-text-weight-medium bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.duration') }}:</label>
+                                    <label class="bu-label bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.duration') }}:</label>
                                     <div class="bu-control bu-is-flex-grow-1">
                                         <span class="bu-tag bu-is-light">
                                             {{ jobDetails.currentState && jobDetails.currentState.lastRunDuration ? jobDetails.currentState.lastRunDuration + 's' : '-' }}
@@ -130,7 +130,7 @@ <h5 class="bu-is-size-5 bu-mb-3 bu-has-text-weight-medium">
                                     v-if="jobDetails.currentState && jobDetails.currentState.failReason"
                                 >
                                     <div class="bu-is-flex">
-                                        <label class="bu-label bu-has-text-weight-medium bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.last-failure') }}:</label>
+                                        <label class="bu-label bu-mr-2 bu-is-flex-grow-0 bu-is-flex-shrink-0 bu-is-flex bu-is-align-items-center" style="width: 120px;">{{ i18n('jobs.last-failure') }}:</label>
                                         <div class="bu-control bu-is-flex-grow-1">
                                             <div class="bu-notification bu-is-light bu-is-danger bu-p-3 bu-mt-2">
                                                 {{ jobDetails.currentState.failReason }}
@@ -166,7 +166,7 @@ <h5 class="bu-is-size-5 bu-mb-4 bu-mt-5">
                                 header-align="left"
                             >
                                 <template v-slot="scope">
-                                    <span class="bu-has-text-weight-medium">
+                                    <span>
                                         {{ formatDateTime(scope.row.lastRunAt) }}
                                     </span>
                                 </template>
diff --git a/frontend/express/public/core/jobs/templates/jobs.html b/frontend/express/public/core/jobs/templates/jobs.html
index cde6f3078ce..54dc18c0600 100644
--- a/frontend/express/public/core/jobs/templates/jobs.html
+++ b/frontend/express/public/core/jobs/templates/jobs.html
@@ -16,7 +16,7 @@
                 v-on:row-click="goTo"
                 :isClickable="true"
                 :default-sort="{ prop: 'name', order: 'ascending' }"
-                persistKey="jobs-table"
+                :persist-key="jobsTablePersistKey"
                 class="jobs-table"
             >
                 <template v-slot="scope">
@@ -68,20 +68,19 @@
                         <template v-slot="scope">
                             <p :data-test-id="'datatable-jobs-schedule-' + scope.$index">
                                 {{ scope.row.scheduleLabel }}
-                                <el-tag
-                                    v-if="scope.row.scheduleOverridden"
-                                    size="mini"
-                                    type="warning"
-                                >
-                                    {{ i18n('jobs.schedule-overridden') }}
-                                </el-tag>
                             </p>
                             <p
                                 :data-test-id="'datatable-jobs-schedule-detail-' + scope.$index"
                                 class="bu-has-text-grey bu-is-size-7"
                             >
                                 <template v-if="scope.row.scheduleOverridden">
-                                    <span class="bu-has-text-weight-bold">{{ i18n('jobs.override') }}:</span> {{ scope.row.configuredSchedule }}
+                                    <el-tooltip
+                                    :content= "'Default: ' + scope.row.schedule + (scope.row.configuredSchedule ? ' | Override: ' + scope.row.configuredSchedule : '')"
+                                    placement="top">
+                                        <i class="ion-alert-circled bu-ml-2""></i>
+                                    <span class="bu-has-text-weight-bold">{{ i18n('jobs.override') }}:</span>
+                                    </el-tooltip>
+                                    {{scope.row.configuredSchedule}}
                                 </template>
                                 <template v-else>
                                     {{ scope.row.schedule }}
diff --git a/frontend/express/public/localization/dashboard/dashboard.properties b/frontend/express/public/localization/dashboard/dashboard.properties
index 7f7529556ff..a9aa2c65ccd 100644
--- a/frontend/express/public/localization/dashboard/dashboard.properties
+++ b/frontend/express/public/localization/dashboard/dashboard.properties
@@ -1187,7 +1187,8 @@ jobs.run-now = Run Now
 jobs.enable = Enable
 jobs.disable = Disable
 jobs.last-run-status = Last Run Status
-jobs.schedule-overridden = Overridden
+jobs.override = Override
+jobs.default = Default
 
 systemlogs.action.task_manager_task_deleted = Report deleted
 systemlogs.action.task_manager_task_updated = Report updated
diff --git a/jobServer/api.js b/jobServer/api.js
index 3ea724a095b..3837161cc52 100644
--- a/jobServer/api.js
+++ b/jobServer/api.js
@@ -131,19 +131,22 @@ function buildJobDetails(scheduledDoc, jobConfig, latestRunData = null) {
     const defaultScheduleLabel = getScheduleLabel(defaultScheduleValue);
     const overrideScheduleLabel = overrideScheduleValue ? getScheduleLabel(overrideScheduleValue) : null;
 
+    // Get default config from the scheduled doc or use existing defaultConfig from jobConfig
+    const defaultConfig = jobConfig?.defaultConfig || {
+        schedule: { value: scheduledDoc?.repeatInterval },
+        retry: {
+            attempts: scheduledDoc?.attempts,
+            delay: scheduledDoc?.backoff
+        }
+    };
+
     // Combine default config from the scheduled doc + overrides from jobConfigs
     const details = {
         config: {
             ...(jobConfig || {}),
 
-            // We ensure we have defaultConfig if jobConfig is missing it
-            defaultConfig: {
-                schedule: { value: scheduledDoc?.repeatInterval },
-                retry: {
-                    attempts: scheduledDoc?.attempts,
-                    delay: scheduledDoc?.backoff
-                }
-            },
+            // Ensure we have the correct defaultConfig
+            defaultConfig: defaultConfig,
 
             enabled,
             // Keep original field names for API compatibility
@@ -211,14 +214,110 @@ function processRunDoc(doc) {
     };
 }
 
+// Helper functions are defined above
+
+/**
+ * Get the most recent status information between scheduled and manual runs
+ * @param {Object} scheduledJob The scheduled job document (type='single')
+ * @param {Array} manualRuns Array of manual run documents (type='normal')
+ * @returns {Object} Combined status information with the most recent data
+ */
+function getLatestStatusInfo(scheduledJob, manualRuns) {
+    // Initialize with scheduled job data or empty object
+    const statusInfo = {
+        lockedAt: scheduledJob?.lockedAt || null,
+        failedAt: scheduledJob?.failedAt || null,
+        lastFinishedAt: scheduledJob?.lastFinishedAt || null,
+        lastRunAt: scheduledJob?.lastRunAt || null,
+        failReason: scheduledJob?.failReason || null,
+        // Count manual runs
+        manualRunCount: manualRuns.length,
+        manualFinishedCount: manualRuns.filter(doc => doc.lastFinishedAt).length
+    };
+
+    // If no manual runs, return scheduled job data
+    if (!manualRuns.length) {
+        return statusInfo;
+    }
+
+    // Find the most recent manual run by lastRunAt
+    const sortedManualRuns = [...manualRuns].sort((a, b) => {
+        if (!a.lastRunAt) {
+            return 1;
+        }
+        if (!b.lastRunAt) {
+            return -1;
+        }
+        return new Date(b.lastRunAt) - new Date(a.lastRunAt);
+    });
+
+    const latestManualRun = sortedManualRuns[0];
+
+    // Update with the latest manual run data
+    if (latestManualRun.lastRunAt && (!statusInfo.lastRunAt || new Date(latestManualRun.lastRunAt) > new Date(statusInfo.lastRunAt))) {
+        statusInfo.lastRunAt = latestManualRun.lastRunAt;
+    }
+
+    // Check for running jobs (lockedAt present)
+    const runningJobs = manualRuns.filter(job => job.lockedAt);
+    if (runningJobs.length) {
+        // Find the most recently started running job
+        const latestRunningJob = runningJobs.sort((a, b) => {
+            return new Date(b.lockedAt) - new Date(a.lockedAt);
+        })[0];
+
+        statusInfo.lockedAt = latestRunningJob.lockedAt;
+        // Update lastRunAt for the running job if available
+        if (latestRunningJob.lastRunAt) {
+            statusInfo.lastRunAt = latestRunningJob.lastRunAt;
+        }
+    }
+
+    // Find the most recent failure across all manual runs
+    const failedJobs = manualRuns.filter(job => job.failedAt);
+    if (failedJobs.length) {
+        const latestFailedJob = failedJobs.sort((a, b) => {
+            return new Date(b.failedAt) - new Date(a.failedAt);
+        })[0];
+
+        // Only update if this failure is more recent than our current failure and completion
+        if ((!statusInfo.failedAt || new Date(latestFailedJob.failedAt) > new Date(statusInfo.failedAt)) &&
+            (!statusInfo.lastFinishedAt || new Date(latestFailedJob.failedAt) > new Date(statusInfo.lastFinishedAt))) {
+            statusInfo.failedAt = latestFailedJob.failedAt;
+            statusInfo.failReason = latestFailedJob.failReason;
+        }
+    }
+
+    // Find the most recent successful completion
+    const completedJobs = manualRuns.filter(job => job.lastFinishedAt);
+    if (completedJobs.length) {
+        const latestCompletedJob = completedJobs.sort((a, b) => {
+            return new Date(b.lastFinishedAt) - new Date(a.lastFinishedAt);
+        })[0];
+
+        // Update if this is the most recent completion
+        if (!statusInfo.lastFinishedAt || new Date(latestCompletedJob.lastFinishedAt) > new Date(statusInfo.lastFinishedAt)) {
+            statusInfo.lastFinishedAt = latestCompletedJob.lastFinishedAt;
+
+            // Clear failure status if completion is more recent
+            if (statusInfo.failedAt && new Date(latestCompletedJob.lastFinishedAt) > new Date(statusInfo.failedAt)) {
+                statusInfo.failedAt = null;
+                statusInfo.failReason = null;
+            }
+        }
+    }
+
+    return statusInfo;
+}
+
 /**
  * Build a job row for the listing page
  * @param {Object} mainDoc doc from pipeline ($first)
  * @param {Object} jobConfig from jobConfigs
- * @param {Object} groupItem aggregator object (with lastRunAt, lastFinishedAt, etc.)
+ * @param {Object} statusInfo Combined status info from scheduled and manual runs
  * @returns {Object} { job: {...}, config: {...} }
  */
-function buildListViewJob(mainDoc, jobConfig, groupItem) {
+function buildListViewJob(mainDoc, jobConfig, statusInfo) {
     // If there's no doc, return something minimal
     if (!mainDoc) {
         return { job: {}, config: {} };
@@ -237,57 +336,43 @@ function buildListViewJob(mainDoc, jobConfig, groupItem) {
     // Use the appropriate label based on which schedule is active
     const effectiveScheduleLabel = overrideScheduleValue ? overrideScheduleLabel : defaultScheduleLabel;
 
+    // Get default config values from the main doc
+    const defaultConfig = jobConfig?.defaultConfig || {
+        schedule: { value: defaultScheduleValue },
+        retry: { attempts: mainDoc.attempts, delay: mainDoc.backoff }
+    };
+
     // Merge in config overrides
     const finalConfig = {
         enabled: jobConfig?.enabled ?? true,
 
-        // The default config is taken from the doc
-        defaultConfig: {
-            schedule: { value: defaultScheduleValue },
-            retry: { attempts: mainDoc.attempts, delay: mainDoc.backoff }
-        },
+        // The default config is taken from either jobConfig or the doc
+        defaultConfig: defaultConfig,
 
         // Keep the original field names for API compatibility 
         schedule: overrideScheduleValue,
         retry: jobConfig?.retry
     };
 
-    // Create a composite document with data from the latest run
-    // This ensures status is determined from the most recent run
-    const latestRunData = {
-        // Take the lockedAt value from either mainDoc or the group data
-        lockedAt: groupItem.lockedAt || mainDoc.lockedAt || null,
-        // Prioritize failedAt and lastFinishedAt from groupItem as they represent max values
-        failedAt: groupItem.failedAt || mainDoc.failedAt || null,
-        lastFinishedAt: groupItem.lastFinishedAt || mainDoc.lastFinishedAt || null,
-        // Use the latest lastRunAt value
-        lastRunAt: groupItem.lastRunAt || mainDoc.lastRunAt || null,
-        // Include failReason if available
-        failReason: groupItem.failedAt ? (groupItem.failReason || mainDoc.failReason) : mainDoc.failReason,
-        // Include counts of normal docs (manual runs)
-        manualRunCount: groupItem.normalDocCount || 0,
-        manualFinishedCount: groupItem.normalFinishedCount || 0
-    };
-
     const jobObj = {
         name: mainDoc.name,
         // Determine status using the latest run data
-        status: getJobStatus(latestRunData),
+        status: getJobStatus(statusInfo),
         // Include both the effective schedule and the original values
         schedule: defaultScheduleValue,
         configuredSchedule: overrideScheduleValue,
         scheduleLabel: effectiveScheduleLabel,
         scheduleOverridden: !!overrideScheduleValue,
         // Include all timestamp fields from the latest run
-        lockedAt: latestRunData.lockedAt,
-        failedAt: latestRunData.failedAt,
-        lastRunAt: latestRunData.lastRunAt,
+        lockedAt: statusInfo.lockedAt,
+        failedAt: statusInfo.failedAt,
+        lastRunAt: statusInfo.lastRunAt,
         nextRunAt: mainDoc.nextRunAt || null,
-        lastFinishedAt: latestRunData.lastFinishedAt,
-        lastRunStatus: getRunStatus(latestRunData),
-        failReason: latestRunData.failReason,
+        lastFinishedAt: statusInfo.lastFinishedAt,
+        lastRunStatus: getRunStatus(statusInfo),
+        failReason: statusInfo.failReason,
         // Calculate total runs (scheduled + manual)
-        total: (mainDoc.runCount || 0) + (latestRunData.manualRunCount || 0)
+        total: (mainDoc.runCount || 0) + (statusInfo.manualRunCount || 0)
     };
 
     return {
@@ -421,11 +506,9 @@ plugins.register('/o/jobs', async function(ob) {
                 // Handle direct date field sorting
                 else if (['lastRunAt', 'lastFinishedAt'].includes(sortColumn)) {
                     // For date fields, handle null values consistently
-                    // This can be implemented directly in MongoDB sort
                     detailSortQuery[sortColumn] = sortDir;
 
                     // Add consistent secondary sort fields to ensure stable ordering
-                    // This ensures runs with the same value in the sorted column always appear in the same order
                     if (sortColumn !== 'lastRunAt') {
                         detailSortQuery.lastRunAt = -1;
                     }
@@ -467,119 +550,10 @@ plugins.register('/o/jobs', async function(ob) {
                 // Fetch config override
                 const jobConfig = await jobConfigsCollection.findOne({ jobName });
 
-                // Aggregate the latest run data with focus on the newest manual run
-                let latestRunData = null;
-
-                if (normalDocs.length > 0 || scheduledDoc) {
-                    // First, find the newest manual run document (type='normal') based on lastRunAt
-                    let newestManualRun = null;
-
-                    if (normalDocs.length > 0) {
-                        // Sort normalDocs by lastRunAt in descending order
-                        const sortedDocs = [...normalDocs].sort((a, b) => {
-                            if (!a.lastRunAt) {
-                                return 1;
-                            } // null/undefined lastRunAt goes last
-                            if (!b.lastRunAt) {
-                                return -1;
-                            }
-                            // If timestamps are exactly equal, use _id for stable sorting
-                            const timeCompare = new Date(b.lastRunAt) - new Date(a.lastRunAt);
-                            if (timeCompare === 0 && a._id && b._id) {
-                                return a._id.toString().localeCompare(b._id.toString());
-                            }
-                            return timeCompare; // newest first
-                        });
-
-                        // The first document is the newest manual run
-                        newestManualRun = sortedDocs[0];
-                    }
-
-                    // Create base latestRunData object
-                    const latestByStatus = {
-                        lockedAt: null,
-                        failedAt: null,
-                        lastFinishedAt: null,
-                        lastRunAt: null,
-                        failReason: null,
-                        // Count manual runs
-                        manualRunCount: normalDocs.length,
-                        manualFinishedCount: normalDocs.filter(doc => doc.lastFinishedAt).length
-                    };
-
-                    // First, apply data from the newest manual run if available
-                    if (newestManualRun) {
-                        if (newestManualRun.lockedAt) {
-                            latestByStatus.lockedAt = newestManualRun.lockedAt;
-                        }
-                        if (newestManualRun.failedAt) {
-                            // Only update failedAt if it's more recent than any existing lastFinishedAt
-                            if (!latestByStatus.lastFinishedAt || new Date(newestManualRun.failedAt) > new Date(latestByStatus.lastFinishedAt)) {
-                                latestByStatus.failedAt = newestManualRun.failedAt;
-                                latestByStatus.failReason = newestManualRun.failReason;
-                            }
-                        }
-                        if (newestManualRun.lastFinishedAt) {
-                            // Update lastFinishedAt and clear failedAt if the successful completion is more recent
-                            latestByStatus.lastFinishedAt = newestManualRun.lastFinishedAt;
-                            if (latestByStatus.failedAt && new Date(newestManualRun.lastFinishedAt) > new Date(latestByStatus.failedAt)) {
-                                latestByStatus.failedAt = null;
-                                latestByStatus.failReason = null;
-                            }
-                        }
-                        if (newestManualRun.lastRunAt) {
-                            latestByStatus.lastRunAt = newestManualRun.lastRunAt;
-                        }
-                    }
-
-                    // Then check if the scheduled doc has more recent timestamps
-                    if (scheduledDoc) {
-                        if (scheduledDoc.lockedAt && (!latestByStatus.lockedAt || new Date(scheduledDoc.lockedAt) > new Date(latestByStatus.lockedAt))) {
-                            latestByStatus.lockedAt = scheduledDoc.lockedAt;
-                        }
-                        if (scheduledDoc.failedAt) {
-                            // Only update failedAt if it's more recent than any existing lastFinishedAt
-                            const failedDate = new Date(scheduledDoc.failedAt);
-                            if ((!latestByStatus.failedAt || failedDate > new Date(latestByStatus.failedAt)) &&
-                                (!latestByStatus.lastFinishedAt || failedDate > new Date(latestByStatus.lastFinishedAt))) {
-                                latestByStatus.failedAt = scheduledDoc.failedAt;
-                                latestByStatus.failReason = scheduledDoc.failReason;
-                            }
-                        }
-                        if (scheduledDoc.lastFinishedAt) {
-                            const finishedDate = new Date(scheduledDoc.lastFinishedAt);
-                            if (!latestByStatus.lastFinishedAt || finishedDate > new Date(latestByStatus.lastFinishedAt)) {
-                                latestByStatus.lastFinishedAt = scheduledDoc.lastFinishedAt;
-
-                                // Clear failed status if the successful completion is more recent
-                                if (latestByStatus.failedAt && finishedDate > new Date(latestByStatus.failedAt)) {
-                                    latestByStatus.failedAt = null;
-                                    latestByStatus.failReason = null;
-                                }
-                            }
-                        }
-                        if (scheduledDoc.lastRunAt && (!latestByStatus.lastRunAt || new Date(scheduledDoc.lastRunAt) > new Date(latestByStatus.lastRunAt))) {
-                            latestByStatus.lastRunAt = scheduledDoc.lastRunAt;
-                        }
-                    }
-
-                    // Look for any running job (lockedAt is set) across all documents
-                    // This takes precedence as we want to show running status immediately
-                    for (const doc of normalDocs) {
-                        if (doc.lockedAt && (!latestByStatus.lockedAt || new Date(doc.lockedAt) > new Date(latestByStatus.lockedAt))) {
-                            latestByStatus.lockedAt = doc.lockedAt;
-                            // Update other fields from this running document
-                            if (doc.lastRunAt) {
-                                latestByStatus.lastRunAt = doc.lastRunAt;
-                            }
-                            // Note: we don't update failedAt/lastFinishedAt because a running job hasn't failed/finished yet
-                        }
-                    }
-
-                    latestRunData = latestByStatus;
-                }
+                // Get the latest run information combining scheduled and manual runs
+                const latestRunData = getLatestStatusInfo(scheduledDoc, normalDocs);
 
-                // Build jobDetails with latest run data
+                // Build job details using the combined latest run data
                 const jobDetails = buildJobDetails(scheduledDoc, jobConfig, latestRunData);
 
                 // Process normal docs
@@ -595,203 +569,193 @@ plugins.register('/o/jobs', async function(ob) {
                 return;
             }
             else {
+                // OPTIMIZED LIST VIEW APPROACH
+
                 // Set up search based on query parameter
-                const search = sSearch ? { name: new RegExp(sSearch, 'i') } : {};
-
-                // We'll create a pipeline that groups by name. 
-                // We prefer the doc with type='single' if it exists, because we sort by type ascending first.
-                const pipeline = [
-                    { $match: search },
-                    {
-                        $sort: (() => {
-                            // Always sort by type first to prioritize 'single' jobs
-                            const sortObj = { type: -1 };
-
-                            // Handle special sorting for status (which is derived from timestamps)
-                            if (sortColumn === 'status') {
-                                // For status sorting, we need to sort by the relevant timestamps
-                                // Status priority: RUNNING (lockedAt) > FAILED (failedAt) > COMPLETED (lastFinishedAt) > SCHEDULED
-                                if (sortDir === 1) { // ascending
-                                    // Prioritize in natural order: SCHEDULED, COMPLETED, FAILED, RUNNING
-                                    sortObj.lockedAt = -1; // RUNNING last (if lockedAt exists)
-                                    sortObj.failedAt = -1; // FAILED before RUNNING
-                                    sortObj.lastFinishedAt = -1; // COMPLETED before FAILED
-                                }
-                                else { // descending
-                                    // Prioritize in reverse order: RUNNING, FAILED, COMPLETED, SCHEDULED
-                                    sortObj.lockedAt = 1; // RUNNING first (if lockedAt exists)
-                                    sortObj.failedAt = 1; // FAILED after RUNNING
-                                    sortObj.lastFinishedAt = 1; // COMPLETED after FAILED
-                                }
-                                // For jobs with the same status, sort by lastRunAt to show most recent first
-                                sortObj.lastRunAt = -1;
-                                // Add name as secondary sort for consistent ordering when status is the same
-                                sortObj.name = sortDir;
-                                // Add _id as tertiary sort for absolute stability
-                                sortObj._id = sortDir;
-                            }
-                            // Handle sorting for timestamp-based fields used in lastRunStatus
-                            else if (sortColumn === 'lastRunStatus') {
-                                if (sortDir === 1) { // ascending: pending, success, failed
-                                    sortObj.failedAt = -1;
-                                    sortObj.lastFinishedAt = -1;
-                                }
-                                else { // descending: failed, success, pending
-                                    sortObj.failedAt = 1;
-                                    sortObj.lastFinishedAt = 1;
-                                }
-                                // For jobs with the same run status, sort by lastRunAt
-                                sortObj.lastRunAt = -1;
-                                // Add name as secondary sort for consistent ordering when status is the same
-                                sortObj.name = sortDir;
-                                // Add _id as tertiary sort for absolute stability
-                                sortObj._id = sortDir;
-                            }
-                            // Handle direct sorting for actual date fields
-                            else if (sortColumn === 'lastFinishedAt' || sortColumn === 'nextRunAt' || sortColumn === 'lastRunAt') {
-                                // For date fields, always sort nulls last regardless of direction
-                                // MongoDB can't do this directly in the sort stage, but we can handle nulls in the group stage
-                                sortObj[sortColumn] = sortDir;
-                                // Add name as secondary sort for consistent ordering when date is the same
-                                sortObj.name = sortDir;
-                                // Add _id as tertiary sort for absolute stability
-                                sortObj._id = sortDir;
-                            }
-                            // Handle sorting when total is selected
-                            else if (sortColumn === 'total') {
-                                // Sort by the combined run count (scheduled + manual)
-                                // This will be calculated later, but we can sort by the initial factors here
-                                sortObj.runCount = sortDir; // From the single doc
-                                sortObj.normalDocCount = sortDir; // From the aggregation
-                                // Add name for secondary sort
-                                sortObj.name = sortDir;
-                                // Add _id as tertiary sort
-                                sortObj._id = sortDir;
-                            }
-                            // Default to name sorting if column isn't directly sortable at db level
-                            else if (sortColumn !== 'name' && sortColumn !== 'scheduleLabel') {
-                                // Sort by name in a case-insensitive manner for consistency
-                                sortObj.name = sortDir;
-                                // Add _id as secondary sort for absolute stability when names are the same
-                                sortObj._id = sortDir;
-                            }
-                            else {
-                                // Sort by name in a case-insensitive manner for consistency
-                                sortObj.name = sortDir;
-                                // Add _id as secondary sort for absolute stability when names are the same
-                                sortObj._id = sortDir;
-                            }
+                const searchQuery = sSearch ? { name: new RegExp(sSearch, 'i') } : {};
 
-                            return sortObj;
-                        })()
-                    },
-                    {
-                        $group: {
-                            _id: "$name",
-                            doc: { $first: "$$ROOT" }, // the first doc in each group
-                            lastRunAt: { $max: "$lastRunAt" },
-                            lastFinishedAt: { $max: "$lastFinishedAt" },
-                            failedAt: { $max: "$failedAt" },
-                            // Add lockedAt to track running jobs
-                            lockedAt: { $max: "$lockedAt" },
-                            // Count documents to help with run counts
-                            normalDocCount: {
-                                $sum: { $cond: [{ $eq: ["$type", "normal"] }, 1, 0] }
-                            },
-                            // Count finished normal docs
-                            normalFinishedCount: {
-                                $sum: {
-                                    $cond: [
-                                        {
-                                            $and: [
-                                                { $eq: ["$type", "normal"] },
-                                                { $ne: ["$lastFinishedAt", null] }
-                                            ]
-                                        },
-                                        1,
-                                        0
-                                    ]
-                                }
-                            },
-                            // Also gather failReason from the document with the latest failedAt
-                            failReason: {
-                                $first: {
-                                    $cond: [
-                                        { $eq: ["$failedAt", { $max: "$failedAt" }] },
-                                        "$failReason",
-                                        null
-                                    ]
-                                }
-                            }
-                        }
-                    },
-                    {
-                        $project: {
-                            _id: 0,
-                            name: "$_id",
-                            doc: 1,
-                            lastRunAt: 1,
-                            lastFinishedAt: 1,
-                            failedAt: 1,
-                            lockedAt: 1,
-                            failReason: 1,
-                            normalDocCount: 1,
-                            normalFinishedCount: 1
-                        }
-                    },
-                    { $skip: skip },
-                    { $limit: limit }
-                ];
-
-                // total distinct job names
-                const allNames = await jobsCollection.distinct('name', search);
-                const totalCount = allNames.length;
-
-                // Check if we need to use a collation for case-insensitive name sorting
-                const options = {
-                    allowDiskUse: true
+                // 1. Count total distinct job names for pagination info
+                const totalCount = (await jobsCollection.distinct('name', {
+                    ...searchQuery,
+                    type: 'single'
+                })).length;
+
+                // 2. Fetch single (scheduled) jobs with pagination and sorting
+                const singleJobsQuery = {
+                    ...searchQuery,
+                    type: 'single'
                 };
 
-                // Run pipeline
-                const rawJobs = await jobsCollection.aggregate(pipeline, options).toArray();
+                // Handle sorting
+                let singleJobsSort = {};
+
+                // Default to name sorting
+                if (sortColumn === 'name') {
+                    singleJobsSort.name = sortDir;
+                }
+                // Handle status sorting (depends on timestamp fields)
+                else if (sortColumn === 'status') {
+                    if (sortDir === 1) { // ascending: SCHEDULED, COMPLETED, FAILED, RUNNING
+                        singleJobsSort = {
+                            lockedAt: -1,
+                            failedAt: -1,
+                            lastFinishedAt: -1
+                        };
+                    }
+                    else { // descending
+                        singleJobsSort = {
+                            lockedAt: 1,
+                            failedAt: 1,
+                            lastFinishedAt: 1
+                        };
+                    }
+                    // Secondary sort by name for consistent ordering
+                    singleJobsSort.name = 1;
+                }
+                // Handle timestamp sorting
+                else if (['lastFinishedAt', 'nextRunAt', 'lastRunAt'].includes(sortColumn)) {
+                    singleJobsSort[sortColumn] = sortDir;
+                    // Secondary sort by name for consistent ordering
+                    singleJobsSort.name = 1;
+                }
+                // Handle total sorting (runCount)
+                else if (sortColumn === 'total') {
+                    singleJobsSort.runCount = sortDir;
+                    // Secondary sort by name for consistent ordering
+                    singleJobsSort.name = 1;
+                }
+                // For other columns, default to name sort
+                else {
+                    singleJobsSort.name = sortDir;
+                }
+
+                // Add _id as final sort for absolute stability
+                singleJobsSort._id = 1;
+
+                // Set collation for case-insensitive sorting when sorting by name
+                const sortOptions = {};
+                if (sortColumn === 'name' || !columns.includes(sortColumn)) {
+                    sortOptions.collation = { locale: 'en', strength: 2 };
+                }
+
+                // Fetch scheduled (single) jobs with pagination and sorting
+                const singleJobsCursor = jobsCollection.find(singleJobsQuery)
+                    .sort(singleJobsSort)
+                    .skip(skip)
+                    .limit(limit);
+
+                // Apply collation if needed
+                if (sortOptions.collation) {
+                    singleJobsCursor.collation(sortOptions.collation);
+                }
+
+                // Get scheduled jobs
+                const singleJobs = await singleJobsCursor.toArray();
+
+                // If no jobs found, return empty result
+                if (singleJobs.length === 0) {
+                    common.returnOutput(ob.params, {
+                        sEcho: ob.params.qstring.sEcho,
+                        iTotalRecords: totalCount,
+                        iTotalDisplayRecords: totalCount,
+                        aaData: []
+                    });
+                    return;
+                }
+
+                // Extract job names to fetch related data
+                const jobNames = singleJobs.map(job => job.name);
+
+                // 3. Fetch all relevant job configs at once
+                const jobConfigs = await jobConfigsCollection.find({
+                    jobName: { $in: jobNames }
+                }).toArray();
 
-                // Fetch jobConfigs in one pass
-                const jobConfigs = await jobConfigsCollection.find({}).toArray();
-                const configMap = {};
-                jobConfigs.forEach(cfg => {
-                    // Store by jobName to match with the name field in pulseJobs
-                    configMap[cfg.jobName] = cfg;
+                // Create a map for faster config lookups
+                const jobConfigMap = {};
+                jobConfigs.forEach(config => {
+                    jobConfigMap[config.jobName] = config;
                 });
 
-                // Merge data
-                const processed = rawJobs.map(item => {
-                    const name = item.name;
-                    const mainDoc = item.doc; // the doc from $first
-                    const jobConfig = configMap[name] || null; // Use name to look up in configMap
-
-                    // Ensure we're getting the latest status data, especially for manual runs
-                    // The group aggregation already provides max values for timestamps,
-                    // but we need to make sure lockedAt reflects current running state
-                    const status = {
-                        lockedAt: item.lockedAt,
-                        failedAt: item.failedAt,
-                        lastFinishedAt: item.lastFinishedAt,
-                        lastRunAt: item.lastRunAt,
-                        failReason: item.failReason,
-                        normalDocCount: item.normalDocCount,
-                        normalFinishedCount: item.normalFinishedCount
-                    };
-
-                    return buildListViewJob(mainDoc, jobConfig, status);
+                // 4. Fetch all manual runs for these jobs at once
+                const normalJobs = await jobsCollection.find({
+                    name: { $in: jobNames },
+                    type: 'normal'
+                }).toArray();
+
+                // Group manual jobs by name for faster processing
+                const normalJobsByName = {};
+                normalJobs.forEach(job => {
+                    if (!normalJobsByName[job.name]) {
+                        normalJobsByName[job.name] = [];
+                    }
+                    normalJobsByName[job.name].push(job);
+                });
+
+                // 5. Process each job with its manual runs and config
+                const processedJobs = singleJobs.map(singleJob => {
+                    const name = singleJob.name;
+                    const jobConfig = jobConfigMap[name] || null;
+                    const relatedManualJobs = normalJobsByName[name] || [];
+
+                    // Get combined status from scheduled job and all its manual runs
+                    const latestStatusInfo = getLatestStatusInfo(singleJob, relatedManualJobs);
+
+                    // Build the list view job with correct status information
+                    return buildListViewJob(singleJob, jobConfig, latestStatusInfo);
                 });
 
-                // Each entry in final array should be { job: {...}, config: {...} }
-                const finalOutput = processed.map(data => ({
+                // Format the final output array with { job, config } structure
+                const finalOutput = processedJobs.map(data => ({
                     job: data.job,
                     config: data.config
                 }));
 
-                // Return
+                // Post-sort the results if we're sorting by a computed field
+                // This ensures that status sorting works correctly since it depends on multiple fields
+                if (sortColumn === 'status' || sortColumn === 'lastRunStatus') {
+                    finalOutput.sort((a, b) => {
+                        // For status sorting
+                        if (sortColumn === 'status') {
+                            const statusOrder = {
+                                'RUNNING': 0,
+                                'FAILED': 1,
+                                'COMPLETED': 2,
+                                'SCHEDULED': 3
+                            };
+
+                            const statusA = statusOrder[a.job.status];
+                            const statusB = statusOrder[b.job.status];
+
+                            if (statusA !== statusB) {
+                                return sortDir === 1 ? statusA - statusB : statusB - statusA;
+                            }
+                        }
+
+                        // For lastRunStatus sorting
+                        if (sortColumn === 'lastRunStatus') {
+                            const runStatusOrder = {
+                                'failed': 0,
+                                'success': 1,
+                                'pending': 2
+                            };
+
+                            const runStatusA = runStatusOrder[a.job.lastRunStatus];
+                            const runStatusB = runStatusOrder[b.job.lastRunStatus];
+
+                            if (runStatusA !== runStatusB) {
+                                return sortDir === 1 ? runStatusA - runStatusB : runStatusB - runStatusA;
+                            }
+                        }
+
+                        // Fallback to name sorting for consistent ordering
+                        return sortDir === 1
+                            ? a.job.name.localeCompare(b.job.name)
+                            : b.job.name.localeCompare(a.job.name);
+                    });
+                }
+
+                // Return the final result
                 common.returnOutput(ob.params, {
                     sEcho: ob.params.qstring.sEcho,
                     iTotalRecords: totalCount,