Sync exacaster/lighter

.github/workflows/build-frontend.yml

@@ -6,13 +6,13 @@ jobs:
   build-frontend:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: '18'
+          node-version: '20'
       - name: Cache Dependencies
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: '**/node_modules'
           key: ${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}

.github/workflows/build-java.yml

@@ -6,15 +6,15 @@ jobs:
   build-java:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '17'
           distribution: 'temurin'
           cache: 'gradle'
       - name: Validate Gradle wrapper
-        uses: gradle/wrapper-validation-action@e6e38bacfdf1a337459f332974bb2327a31aaf4b
+        uses: gradle/actions/wrapper-validation@v4
       - name: Build with Gradle
         run: |
           cd ./server

.github/workflows/publish-registry-pr.yml

@@ -11,13 +11,13 @@ jobs:
     strategy:
       matrix:
         include:
-          - spark_version: 3.4.2
+          - spark_version: 3.5.6
             hadoop_version: 3
-          - spark_version: 3.5.1
+          - spark_version: 4.0.0
             hadoop_version: 3
     steps:
       - name: Checkout pull request
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           ref: refs/pull/${{ github.event.issue.number }}/head
       - uses: docker/setup-buildx-action@v2

.github/workflows/publish-registry.yml

@@ -8,18 +8,18 @@ jobs:
     strategy:
       matrix:
         include:
-          - spark_version: 3.4.2
+          - spark_version: 3.5.6
             hadoop_version: 3
-          - spark_version: 3.5.1
+          - spark_version: 4.0.0
             hadoop_version: 3
     steps:
-      - uses: docker/setup-buildx-action@v2
-      - uses: docker/login-action@v2
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ github.token }}
-      - uses: docker/build-push-action@v3
+      - uses: docker/build-push-action@v6
         with:
           push: true
           platforms: linux/amd64,linux/arm64

Dockerfile

@@ -1,6 +1,6 @@
-FROM eclipse-temurin:17-jdk-jammy as server
+FROM eclipse-temurin:17-jdk-jammy AS server
 
-ARG SPARK_VERSION=3.5.1
+ARG SPARK_VERSION=3.5.6
 ARG HADOOP_VERSION=3
 
 WORKDIR /home/app/
@@ -9,9 +9,9 @@ COPY server/ ./server/
 WORKDIR /home/app/server/
 RUN ./gradlew build -x test -PSPARK_VERSION=${SPARK_VERSION}
 
-FROM node:lts-alpine3.18 as frontend
+FROM node:lts-alpine3.18 AS frontend
 
-ARG SPARK_VERSION=3.5.1
+ARG SPARK_VERSION=3.5.6
 ARG HADOOP_VERSION=3
 
 ENV APP_BASE_URL='/lighter'
@@ -21,11 +21,11 @@ COPY frontend/ ./frontend/
 RUN wget "https://downloads.apache.org/spark/spark-${SPARK_VERSION}/spark-${SPARK_VERSION}-bin-hadoop${HADOOP_VERSION}.tgz" -O - | tar -xz
 
 WORKDIR /home/app/frontend/
-RUN yarn install && yarn build
+RUN yarn install --network-timeout 1000000 && yarn build
 
 FROM eclipse-temurin:17-jre-jammy
 
-ARG SPARK_VERSION=3.5.1
+ARG SPARK_VERSION=3.5.6
 ARG HADOOP_VERSION=3
 
 ENV FRONTEND_PATH=/home/app/frontend/

README.md

@@ -1,12 +1,13 @@
 # Lighter
 
-Lighter is an opensource application for interacting with [Apache Spark](https://spark.apache.org/) on [Kubernetes](https://kubernetes.io/) or [Apache Hadoop YARN](https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/YARN.html). It is heavily inspired by [Apache Livy](https://livy.incubator.apache.org/) and has some overlapping features.
+Lighter is an opensource application for interacting with [Apache Spark](https://spark.apache.org/) on [Kubernetes](https://kubernetes.io/) or [Apache Hadoop YARN](https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/YARN.html).
+It is heavily inspired by [Apache Livy](https://livy.incubator.apache.org/) and has some overlapping features.
 
-Lighter supports:
+Lighter support:
 - Interactive Python Sessions through [Sparkmagic](https://github.com/jupyter-incubator/sparkmagic) kernel
 - Batch application submissions through the REST API
 
-You can read a breaf description on how Lighter works [here](./docs/architecture.md).
+You can read a description on how Lighter works [here](./docs/architecture.md).
 
 ## Using Lighter
 - [Quickstart with Minikube](./quickstart/README.md)
@@ -28,7 +29,7 @@ docker build -t lighter .
 
 ### Spark versions
 
-Lighter will always try to support the latest patch version for all officially recommended [Apache Spark](https://spark.apache.org/) releases i.e.: 3.4.2 and 3.5.1.
+Lighter will always try to support the latest patch version for all officially recommended [Apache Spark](https://spark.apache.org/) releases i.e.: 3.5.6 and 4.0.0.
 
 ### Contributing
 

docs/configuration.md

@@ -4,25 +4,28 @@ Lighter can be configured by using environment variables. Currently, Lighter sup
 
 ## Global properties
 
-| Property                               | Description                                                                                                                                      | Default                         |
-|----------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------|
-| LIGHTER_MAX_RUNNING_JOBS               | Max running Batch jobs in parallel                                                                                                               | 5                               |
-| LIGHTER_MAX_STARTING_JOBS              | Max starting Batch jobs in parallel                                                                                                              | 5                               |
-| LIGHTER_SPARK_HISTORY_SERVER_URL       | Spark history server URL used on the Lighter UI                                                                                                  | http://localhost/spark-history/ |
-| LIGHTER_EXTERNAL_LOGS_URL_TEMPLATE     | Template for link to external logs (Grafana, Graylog, etc.) used on the Lighter UI. Allowed placeholders: `{{id}}`, `{{appId}}`, `{{createdTs}}` |                                 |
-| LIGHTER_PY_GATEWAY_PORT                | Port for live Spark session communication                                                                                                        | 25333                           |
-| LIGHTER_URL                            | URL which can be used to access Lighter form Spark Job                                                                                           | http://lighter.spark:8080       |
-| LIGHTER_SESSION_TIMEOUT_INTERVAL       | `java.time.Duration` representing session lifetime (from last statement creation). Use `0m` value to disable                                     | 90m                             |
-| LIGHTER_SESSION_TIMEOUT_ACTIVE         | Should Lighter kill sessions with waiting statements (obsolete when `LIGHTER_SESSION_TIMEOUT_INTERVAL` is `0m`)                                  | false                           |
-| LIGHTER_SESSION_SCHEDULE_INTERVAL      | `java.time.Duration` representing the interval at which a task is triggered to initiate scheduled sessions                                       | 1m                              |
-| LIGHTER_SESSION_TRACK_RUNNING_INTERVAL | `java.time.Duration` representing the interval at which a task is triggered to process and update running session state                          | 2m                              |
-| LIGHTER_STORAGE_JDBC_URL               | JDBC url for lighter storage                                                                                                                     | jdbc:h2:mem:lighter             |
-| LIGHTER_STORAGE_JDBC_USERNAME          | JDBC username                                                                                                                                    | sa                              |
-| LIGHTER_STORAGE_JDBC_PASSWORD          | JDBC password                                                                                                                                    |                                 |
-| LIGHTER_STORAGE_JDBC_DRIVER_CLASS_NAME | JDBC driver class name                                                                                                                           | org.h2.Driver                   |
-| LIGHTER_BATCH_DEFAULT_CONF             | Default `conf` props for batch applications (JSON)<sup>*</sup>                                                                                   |                                 |
-| LIGHTER_SESSION_DEFAULT_CONF           | Default `conf` props for session applications (JSON)                                                                                             |                                 |
-| LIGHTER_SESSION_PERMANENT_SESSIONS     | List of configurations for [permanent sessions](./permanent_sessions.md)                                                                         | "[]"                            |
+| Property                               | Description                                                                                                                                                                                            | Default                         |
+|----------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------|
+| LIGHTER_MAX_RUNNING_JOBS               | Max running Batch jobs in parallel                                                                                                                                                                     | 5                               |
+| LIGHTER_MAX_STARTING_JOBS              | Max starting Batch jobs in parallel                                                                                                                                                                    | 5                               |
+| LIGHTER_MAX_RUNNING_SESSIONS           | Max running sessions in parallel                                                                                                                                                                       | 5                               |
+| LIGHTER_SPARK_HISTORY_SERVER_URL       | Spark history server URL used on the Lighter UI                                                                                                                                                        | http://localhost/spark-history/ |
+| LIGHTER_EXTERNAL_LOGS_URL_TEMPLATE     | Template for link to external logs (Grafana, Graylog, etc.) used on the Lighter UI. Allowed placeholders: `{{id}}`, `{{appId}}`, `{{createdTs}}`                                                       |                                 |
+| LIGHTER_PY_GATEWAY_PORT                | Port for live Spark session communication                                                                                                                                                              | 25333                           |
+| LIGHTER_URL                            | URL which can be used to access Lighter form Spark Job                                                                                                                                                 | http://lighter.spark:8080       |
+| LIGHTER_ZOMBIE_INTERVAL                | How long for Lighter to try to fetch the status of the job before marking it as a zombie. (For jobs that "disappear" before Lighter could determine their final status)                                | 30m                             |
+| LIGHTER_SESSION_TIMEOUT_INTERVAL       | `java.time.Duration` representing session lifetime (from last statement creation). Use `0m` value to disable                                                                                           | 90m                             |
+| LIGHTER_STATE_RETAIN_INTERVAL          | `java.time.Duration` representing how long finished session and batch history will be retained before automatic cleanup. If not set, cleanup is disabled.                                              |                                 |
+| LIGHTER_SESSION_TIMEOUT_ACTIVE         | Should Lighter kill sessions with waiting statements (obsolete when `LIGHTER_SESSION_TIMEOUT_INTERVAL` is `0m`)                                                                                        | false                           |
+| LIGHTER_SESSION_SCHEDULE_INTERVAL      | `java.time.Duration` representing the interval at which a task is triggered to initiate scheduled sessions                                                                                             | 1m                              |
+| LIGHTER_SESSION_TRACK_RUNNING_INTERVAL | `java.time.Duration` representing the interval at which a task is triggered to process and update running session state                                                                                | 2m                              |
+| LIGHTER_STORAGE_JDBC_URL               | JDBC url for lighter storage                                                                                                                                                                           | jdbc:h2:mem:lighter             |
+| LIGHTER_STORAGE_JDBC_USERNAME          | JDBC username                                                                                                                                                                                          | sa                              |
+| LIGHTER_STORAGE_JDBC_PASSWORD          | JDBC password                                                                                                                                                                                          |                                 |
+| LIGHTER_STORAGE_JDBC_DRIVER_CLASS_NAME | JDBC driver class name                                                                                                                                                                                 | org.h2.Driver                   |
+| LIGHTER_BATCH_DEFAULT_CONF             | Default `conf` props for batch applications (JSON)<sup>*</sup>                                                                                                                                         |                                 |
+| LIGHTER_SESSION_DEFAULT_CONF           | Default `conf` props for session applications (JSON)                                                                                                                                                   |                                 |
+| LIGHTER_CONFIG_JSON                    | Any set of Lighter config in JSON format. For permanent sessions use: `{"lighter":{"session":{"permanent-sessions":[...]}}}`. List of configurations for [permanent sessions](./permanent_sessions.md) |                                 |
 
 <sup>*</sup> default configs will be merged with configss provided in submit request, if property is defined in submit request, default will be ignored.
 Example of `LIGHTER_BATCH_DEFAULT_CONF`: `{"spark.kubernetes.driverEnv.TEST1":"test1"}`.
@@ -49,4 +52,4 @@ Example of `LIGHTER_BATCH_DEFAULT_CONF`: `{"spark.kubernetes.driverEnv.TEST1":"t
 | LIGHTER_YARN_KERBEROS_KEYTAB    | Kerberos keytab used for job management                           |         |
 
 <sup>*</sup> Principal & Keytab provided in `LIGHTER_YARN_KERBEROS_PRINCIPAL` and `LIGHTER_YARN_KERBEROS_KEYTAB` will be used by spark job
-as well, if `spark.kerberos.keytab` is not explicitly declared in `LIGHTER_BATCH_DEFAULT_CONF` or provided on submit request.
+as well, if `spark.kerberos.keytab` is not explicitly declared in `LIGHTER_BATCH_DEFAULT_CONF` or provided on submit request.

docs/docker.md

@@ -3,7 +3,7 @@
 If you're thinking of running Lighter on docker, you are probably using it with YARN backend. When you are running it, you should expose two ports (REST API port and Python Gateway port), as shown in this example:
 
 ```bash
-docker run ghcr.io/exacaster/lighter:0.1.1-spark3.5.1 \
+docker run ghcr.io/exacaster/lighter:0.1.5-spark3.5.6 \
   -p 8080:8080 \
   -p 25333:25333 \
   -e LIGHTER_KUBERNETES_ENABLED=false \

docs/rest.md

@@ -71,7 +71,8 @@ Response example:
         }
       },
       "createdAt":"2021-10-11T08:12:43.50739",
-      "contactedAt":"2021-10-11T08:15:23.335098"
+      "contactedAt":"2021-10-11T08:15:23.335098",
+      "finishedAt":"2021-10-11T08:20:15.123456"
     }
   ]
 }
@@ -108,7 +109,8 @@ Response example:
     }
   },
   "createdAt":"2021-10-11T08:12:43.50739",
-  "contactedAt":"2021-10-11T08:15:23.335098"
+  "contactedAt":"2021-10-11T08:15:23.335098",
+  "finishedAt":"2021-10-11T08:20:15.123456"
 }
 ```
 

frontend/eslint.config.mjs

@@ -0,0 +1,22 @@
+import eslint from '@eslint/js';
+import eslintConfigPrettier from 'eslint-config-prettier';
+import tseslint from 'typescript-eslint';
+import react from 'eslint-plugin-react';
+import importPlugin from 'eslint-plugin-import';
+// TODO: Does not work with eslint 9
+// import reactHooks from 'eslint-plugin-react-hooks';
+
+export default tseslint.config(eslint.configs.recommended, eslintConfigPrettier, tseslint.configs.recommended, react.configs.flat.recommended, {
+  settings: {
+    react: {
+      version: 'detect',
+    },
+  },
+  files: ['**/*.{ts,tsx}'],
+  extends: [importPlugin.flatConfigs.recommended, importPlugin.flatConfigs.typescript],
+  rules: {
+    'react/react-in-jsx-scope': 'off',
+    'react/jsx-uses-react': 'off',
+    'react/prop-types': 'off',
+  },
+});