From ea8e8de1bbeb3b7523b1959cd21750958e6aca8b Mon Sep 17 00:00:00 2001 From: hategan Date: Thu, 13 Feb 2025 22:10:03 -0800 Subject: [PATCH 01/33] Added a TUI installer for the CI tests. --- .flake8 | 20 +- .github/dependabot.yml | 1 + psij-ci-run | 33 +- psij-ci-setup | 100 ++- requirements-tests.txt | 3 + tests/installer/__init__.py | 0 tests/installer/dialogs.py | 305 ++++++++ tests/installer/install_methods.py | 264 +++++++ tests/installer/log.py | 1 + tests/installer/main.py | 258 ++++++ tests/installer/panels/basic_info_panel.py | 100 +++ .../installer/panels/batch_scheduler_panel.py | 418 ++++++++++ tests/installer/panels/complete_panel.py | 30 + tests/installer/panels/intro_panel.py | 44 ++ tests/installer/panels/key_panel.py | 228 ++++++ tests/installer/panels/panel.py | 39 + tests/installer/panels/schedule_panel.py | 127 +++ tests/installer/py.typed | 0 tests/installer/state.py | 172 ++++ tests/installer/style.tcss | 738 ++++++++++++++++++ tests/installer/terminal.py | 329 ++++++++ tests/installer/widgets.py | 53 ++ tests/run_installer.py | 5 + 23 files changed, 3240 insertions(+), 28 deletions(-) create mode 100644 tests/installer/__init__.py create mode 100644 tests/installer/dialogs.py create mode 100644 tests/installer/install_methods.py create mode 100644 tests/installer/log.py create mode 100644 tests/installer/main.py create mode 100644 tests/installer/panels/basic_info_panel.py create mode 100644 tests/installer/panels/batch_scheduler_panel.py create mode 100644 tests/installer/panels/complete_panel.py create mode 100644 tests/installer/panels/intro_panel.py create mode 100644 tests/installer/panels/key_panel.py create mode 100644 tests/installer/panels/panel.py create mode 100644 tests/installer/panels/schedule_panel.py create mode 100644 tests/installer/py.typed create mode 100644 tests/installer/state.py create mode 100644 tests/installer/style.tcss create mode 100644 tests/installer/terminal.py create mode 100644 tests/installer/widgets.py create mode 100644 tests/run_installer.py diff --git a/.flake8 b/.flake8 index fe2f46ac..c24af0b3 100644 --- a/.flake8 +++ b/.flake8 @@ -55,7 +55,7 @@ max-line-length = 100 # One can omit the description on the __init__ docstring and simply # document the parameters, but then flake8 complains about the # structure of the __init__ doctsring. The D205 and D400 errors are -# precisely those complains. We disable these in order to have a +# precisely those complaints. We disable these in order to have a # sane way of documenting classes with Sphinx' autoclass. @@ -63,6 +63,20 @@ ignore = B902, D205, D400, D401, D100, W503 # D103 - Missing docstring in public function # -# Ignore docstrings requirement in tests +# Ignore docstrings requirement in tests. +# +# D101, D102, D103, D107 +# +# We don't quite document the installer. +# +# E402 - module level import not at top of file +# +# In the installer, we need to patch Textual for non-unicode terminal +# support, and that needs to be the first thing done before we import +# other stuff. +# -per-file-ignores = tests/*:D103 +per-file-ignores = + tests/*: D103 + tests/installer/*: D101, D102, D103, D104, D107 + tests/installer/main.py: E402, D101, D102, D103, D107 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ed04926b..cf382e3e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -16,4 +16,5 @@ updates: - dependency-name: "pystache" - dependency-name: "typeguard" - dependency-name: "packaging" + - dependency-name: "requests" diff --git a/psij-ci-run b/psij-ci-run index 3783bfba..f9b9b4af 100755 --- a/psij-ci-run +++ b/psij-ci-run @@ -2,9 +2,17 @@ usage() { cat </dev/null 2>&1 ; pwd -P )" +cd "$MYPATH" + if [ -f psij-ci-env ]; then source psij-ci-env fi REPEAT=0 RESCHEDULE=0 +NO_OUT=0 while [ "$1" != "" ]; do case "$1" in @@ -38,13 +50,26 @@ while [ "$1" != "" ]; do TIME="$2" shift 2 ;; + --log) + NO_OUT=1 + shift + ;; *) echo "Unrecognized command line option: $1." usage exit 1 + ;; esac done +if [ "$NO_OUT" == "1" ]; then + exec 1<&- + exec 2<&- + + exec 1>>./testing.log + exec 2>&1 +fi + if python --version 2>&1 | egrep -q 'Python 3\..*' >/dev/null 2>&1 ; then PYTHON="python" else @@ -68,7 +93,7 @@ if [ "$REPEAT" == "1" ]; then sleep $TO_SLEEP done elif [ "$RESCHEDULE" == "1" ]; then - CMD="./psij-ci-run --reschedule $TIME >> testing.log 2>&1" + CMD="./psij-ci-run --reschedule $TIME --log" echo "$CMD" | at $TIME $PYTHON tests/ci_runner.py $MODE else diff --git a/psij-ci-setup b/psij-ci-setup index c080c70a..3355544c 100755 --- a/psij-ci-setup +++ b/psij-ci-setup @@ -1,11 +1,20 @@ #!/bin/bash +set -o pipefail + if pip --version 2>&1 | egrep -q 'python 3\..*' >/dev/null 2>&1 ; then PIP="pip" else PIP="pip3" fi +if python --version 2>&1 | egrep -q 'Python 3\..*' >/dev/null 2>&1 ; then + PYTHON="python" +else + PYTHON="python3" +fi + + existing_error_trailer() { echo "If you are certain that you want to install multiple entries, " echo "you can re-run this script with the \"-f\" flag. " @@ -50,7 +59,7 @@ cron_existing_error() { } cron_install() { - CMD="$CMD >> testing.log 2>&1" + CMD="$CMD --log" LINE="$MINUTE $HOUR * * * cd \"$MYPATH\" && $CMD" echo echo "================================================================" @@ -109,7 +118,7 @@ at_existing_error() { } at_install() { - CMD="$CMD --reschedule $HOUR:$MINUTE >> testing.log 2>&1" + CMD="$CMD --reschedule $HOUR:$MINUTE --log" echo echo "================================================================" echo "The following will be executed:" @@ -141,7 +150,7 @@ screen_existing_error() { } screen_install() { - CMD="$CMD --repeat >> testing.log 2>&1" + CMD="$CMD --repeat --log" echo echo "================================================================" echo "WARNING: Screen sessions do not persist across reboots. Please " @@ -166,7 +175,7 @@ manual_existing_error() { } manual_install() { - CMD="$CMD --repeat >> testing.log 2>&1" + CMD="$CMD --repeat --log" echo echo "================================================================" echo "Please run the following command in the background: " @@ -271,14 +280,76 @@ check_key() { fi } + +not_cached() { + if [ ! -d .packages ]; then + return 0 + fi + if [ ! -f .packages/req_csum.txt ]; then + return 0 + fi + REQ_CSUM=`cat .packages/req_csum.txt` + CRT_CSUM=`cat requirements*.txt | md5sum` + + if [ "$REQ_CSUM" != "$CRT_CSUM" ]; then + return 0 + else + return 1 + fi +} + +dots_for_lines() { + while read LINE; do + echo -n . + done +} + +install_deps() { + if not_cached; then + echo -n "Installing dependencies..." + + exec 3> >(dots_for_lines >> /dev/stdout) + OUT=`$PIP install --target .packages --upgrade -r requirements-tests.txt --no-compile 2>&1 1>&3` + + if [ "$?" != "0" ]; then + echo "FAILED" + echo "$OUT" + exit 2 + else + echo "Done" + cat requirements*.txt | md5sum >.packages/req_csum.txt + fi + fi + + export PYTHONPATH=`pwd`/.packages:$PYTHONPATH +} + + FORCE=0 +PLAIN=0 -if [ "$1" == "-f" ]; then - FORCE=1 +while [ "$1" != "" ]; do + if [ "$1" == "-f" ]; then + FORCE=1 + elif [ "$1" == "--plain" ]; then + PLAIN=1 + else + echo "Unknown option \"$1\"" + exit 1 + fi shift -fi +done MYPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +cd "$MYPATH" + +if [ "$PLAIN" != "1" ]; then + install_deps + + echo "Running TUI installer. To run the normal installer, use \"psij-ci-setup --plain\"" + + exec $PYTHON tests/run_installer.py +fi echo echo "================================================================" @@ -297,21 +368,8 @@ check_email check_key -cd "$MYPATH" - -echo -n "Installing dependencies..." - -OUT=`$PIP install --target .packages --upgrade -r requirements-tests.txt 2>&1` - -if [ "$?" != "0" ]; then - echo "FAILED" - echo $OUT - exit 2 -else - echo "Done" -fi +install_deps -export PYTHONPATH=`pwd`/.packages:$PYTHONPATH HOUR=`echo $(($RANDOM % 24))` MINUTE=`echo $(($RANDOM % 60))` diff --git a/requirements-tests.txt b/requirements-tests.txt index cf9ed31a..ef9572d5 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,3 +8,6 @@ requests >= 2.25.1 pytest-cov pytest-timeout filelock >= 3.4, < 3.18 +textual +tree-sitter +tree-sitter-bash diff --git a/tests/installer/__init__.py b/tests/installer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/installer/dialogs.py b/tests/installer/dialogs.py new file mode 100644 index 00000000..d079bbe5 --- /dev/null +++ b/tests/installer/dialogs.py @@ -0,0 +1,305 @@ +import asyncio +from typing import cast + +from textual import on +from textual.app import App, ComposeResult +from textual.containers import Horizontal, Vertical +from textual.screen import ModalScreen +from textual.widget import Widget +from textual.widgets import Button, Label, RichLog, ProgressBar + +from installer.widgets import DottedLoadingIndicator, ShortcutButton + + +class RunnableDialog[T](ModalScreen[None]): + def __init__(self) -> None: + super().__init__() + self.done = asyncio.get_running_loop().create_future() + + def set_result(self, result: T) -> None: + self.done.set_result(result) + + async def wait(self) -> bool: + return cast(bool, await self.done) + + async def run(self, app: App[object]) -> T: + app.push_screen(self) + try: + await self.done + return cast(T, self.done.result()) + finally: + app.pop_screen() + + +class HelpDialog(RunnableDialog[None]): + BINDINGS = [ + ('escape', 'close', 'Close'), + ('c', 'close', 'Close'), + ('alt+c', 'close', 'Close') + ] + + def __init__(self, title: str, widget: Widget) -> None: + super().__init__() + self.title = title + self.widget = widget + + def compose(self) -> ComposeResult: + assert self.title is not None + yield Vertical( + Label(self.title, classes='header'), + Vertical(self.widget, classes='v-scrollable'), + Horizontal( + ShortcutButton('&Close', variant='error', id='btn-close'), + classes='action-area' + ), + id='help-dialog', classes='dialog help-dialog' + ) + + def on_button_pressed(self, event: Button.Pressed) -> None: + self.action_close() + + def action_close(self) -> None: + self.set_result(None) + + +class ExitConfirmDialog(RunnableDialog[str]): + BINDINGS = [ + ('escape', 'close'), + ('c', 'close', 'Close'), + ('alt+c', 'close', 'Close') + ] + + def __init__(self) -> None: + super().__init__() + + def compose(self) -> ComposeResult: + self.btn_quit = ShortcutButton('&Quit', variant='error', id='btn-quit') + self.btn_cancel = ShortcutButton('&Cancel', variant='warning', id='btn-cancel') + yield Vertical( + Label('Confirm exit', classes='header'), + Label('Are you sure you want to exit the installer?', + classes='main-text', + shrink=True, expand=True), + Horizontal(self.btn_quit, self.btn_cancel, classes='action-area'), + id='replace-dialog', classes='dialog error-dialog' + ) + + def action_close(self) -> None: + self.set_result('cancel') + + def on_button_pressed(self, event: Button.Pressed) -> None: + if event.button == self.btn_quit: + self.set_result('quit') + else: + self.set_result('cancel') + + +class ProgressDialog(ModalScreen[None]): + def __init__(self, message: str) -> None: + super().__init__() + self.message = message + + def compose(self) -> ComposeResult: + yield Vertical( + Label(self.message, classes='header', shrink=True, expand=True), + ProgressBar(id='progress-indicator-indeterminate', + show_percentage=False, show_bar=True, show_eta=False), + id='progress-dialog', classes='dialog' + ) + + +class TestJobsDialog(RunnableDialog[bool]): + BINDINGS = [ + ('b', 'back', 'Back'), + ('alt+b', 'back', 'Back'), + ('c', 'continue', 'Continue'), + ('alt+c', 'continue', 'Continue'), + ] + + def __init__(self) -> None: + super().__init__() + self.result = None + + def compose(self) -> ComposeResult: + yield Vertical( + Label('Running test jobs', classes='header', shrink=True, expand=True), + Horizontal( + Label('Single node job', id='label-job-1', classes='test-job-label'), + Label('[ ', classes='test-job-marker'), + DottedLoadingIndicator(id='indicator-job-1', classes='test-job-indicator hidden'), + Label('', id='status-job-1', classes='test-job-status'), + Label(' ]', classes='test-job-marker'), + classes='job-progress-row' + ), + Horizontal( + Label('Multi node job ', id='label-job-2', classes='test-job-label'), + Label('[ ', classes='test-job-marker'), + DottedLoadingIndicator(id='indicator-job-2', classes='test-job-indicator hidden'), + Label('', id='status-job-2', classes='test-job-status'), + Label(' ]', classes='test-job-marker'), + classes='job-progress-row' + ), + RichLog(id='test-job-errors', wrap=True), + Horizontal( + ShortcutButton('Go &back', variant='error', id='btn-back'), + ShortcutButton('&Continue', id='btn-continue'), + classes='action-area' + ), + id='test-jobs-dialog', classes='dialog' + ) + + def on_button_pressed(self, event: Button.Pressed) -> None: + if event.button.id == 'btn-back': + self.action_back() + else: + self.action_continue() + + def action_back(self) -> None: + self.app.pop_screen() + self.set_result(False) + + def action_continue(self) -> None: + self.app.pop_screen() + self.set_result(True) + + def set_running(self, job_no: int) -> None: + indicator = self.get_widget_by_id(f'indicator-job-{job_no}') + indicator.remove_class('hidden') + + label = self.get_widget_by_id(f'status-job-{job_no}') + label.add_class('hidden') + + def set_status(self, job_no: int, status: str, cls: str) -> None: + indicator = self.get_widget_by_id(f'indicator-job-{job_no}') + indicator.add_class('hidden') + + label = self.get_widget_by_id(f'status-job-{job_no}') + assert isinstance(label, Label) + label.update(status) + label.remove_class('hidden') + label.add_class(cls) + + def log_error(self, label: str, ex: Exception) -> None: + log = self.get_widget_by_id('test-job-errors') + assert isinstance(log, RichLog) + + log.write(f'----------- {label} -----------') + log.write(str(ex)) + log.write('') + + def focus_back_button(self) -> None: + self.get_widget_by_id('btn-back').focus() + + def focus_continue_button(self) -> None: + self.get_widget_by_id('btn-continue').focus() + + +class ErrorDialog(RunnableDialog[bool]): + BINDINGS = [ + ('c', 'close', 'Close'), + ('alt+c', 'close', 'Close') + ] + + def __init__(self, title: str, message: str) -> None: + super().__init__() + self.title = title + self.message = message + + def compose(self) -> ComposeResult: + assert self.title is not None + yield Vertical( + Label(self.title, classes='header'), + Label(self.message, classes='main-text', shrink=True, expand=True), + Horizontal( + ShortcutButton('&Close', variant='error', id='btn-close'), + classes='action-area' + ), + id='error-dialog', classes='dialog error-dialog' + ) + + def on_button_pressed(self, event: Button.Pressed) -> None: + self.action_close() + + def action_close(self) -> None: + self.set_result(True) + + +class KeyRequestDialog(ModalScreen[None]): + BINDINGS = [ + ('c', 'cancel', 'Cancel'), + ('alt+c', 'cancel', 'Cancel') + ] + + def __init__(self, url: str) -> None: + super().__init__() + self.url = url + + def compose(self) -> ComposeResult: + yield Vertical( + Label('Key request', classes='header'), + Label('Please enter the following URL in a web browser:', classes='main-text'), + Label(self.url, classes='main-text', id='key-request-url'), + Label('Then follow the instructions the browser. The URL is valid for 10 minutes.', + classes='main-text', expand=True, shrink=True), + Horizontal( + ProgressBar(id='progress-indicator-indeterminate'), + Label('Waiting', classes='main-text', id='status-label'), + ), + Horizontal( + ShortcutButton('&Cancel', variant='error', id='btn-cancel'), + classes='action-area' + ), + id='key-request-dialog', classes='dialog' + ) + + def update_status(self, status_str: str) -> None: + status_label = self.get_widget_by_id('status-label') + assert isinstance(status_label, Label) + if status_str == 'initialized': + pass + elif status_str == 'seen': + status_label.update('Page opened') + elif status_str == 'email_sent': + status_label.update('Email sent') + elif status_str == 'verified': + status_label.update('Email verified') + + @on(Button.Pressed, selector='#btn-cancel') + def action_cancel(self) -> None: + self.app.pop_screen() + + +class ExistingInstallConfirmDialog(RunnableDialog[str]): + BINDINGS = [ + ('c', 'close', 'Close'), + ('alt+c', 'close', 'Close') + ] + + def __init__(self, method: str) -> None: + super().__init__() + self.method = method + + def compose(self) -> ComposeResult: + assert self.method is not None + yield Vertical( + Label('Existing installation detected', classes='header'), + Label(f'An existing {self.method} installation of the tests was detected. Continuing ' + 'will update the settings used by the existing installation.', + classes='main-text', + shrink=True, expand=True), + Horizontal( + ShortcutButton('&Quit', variant='error', id='btn-quit'), + ShortcutButton('&Update', variant='warning', id='btn-update'), + ShortcutButton('&Reinstall', variant='warning', id='btn-reinstall'), + classes='action-area' + ), + id='replace-dialog', classes='dialog error-dialog' + ) + + def on_button_pressed(self, event: Button.Pressed) -> None: + if event.button.id == 'btn-quit': + self.set_result('quit') + elif event.button.id == 'btn-update': + self.set_result('update') + else: + self.set_result('reinstall') diff --git a/tests/installer/install_methods.py b/tests/installer/install_methods.py new file mode 100644 index 00000000..2da9adcb --- /dev/null +++ b/tests/installer/install_methods.py @@ -0,0 +1,264 @@ +import os +import subprocess +import time +from abc import ABC, abstractmethod +import random +from tempfile import mkstemp +from typing import Optional, Tuple + +from installer.log import log + + +def _run(cmd: str, input: Optional[str] = None) -> Tuple[int, str]: + p = subprocess.run(['/bin/bash', '-c', cmd], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + input=input, text=True) + return p.returncode, p.stdout + + +def _must_succeed(cmd: str, input: Optional[str] = None) -> str: + ec, out = _run(cmd, input) + if ec != 0: + raise Exception(f'Command {cmd} failed: {out}') + return out + + +def _succeeds(cmd: str, input: Optional[str] = None) -> bool: + ec, _ = _run(cmd, input) + return ec == 0 + + +def _save_env() -> None: + _must_succeed('declare -x | egrep -v "^declare -x ' + '(BASH_VERSINFO|DISPLAY|EUID|GROUPS|SHELLOPTS|TERM|UID|_)=" >psij-ci-env') + + +class InstallMethod(ABC): + @abstractmethod + def is_available(self) -> Tuple[bool, str | None]: + pass + + @abstractmethod + def install(self) -> None: + pass + + @abstractmethod + def already_installed(self) -> bool: + pass + + @property + @abstractmethod + def preview(self) -> str: + pass + + @property + @abstractmethod + def name(self) -> str: + pass + + @property + @abstractmethod + def label(self) -> str: + pass + + @property + @abstractmethod + def help_message(self) -> str: + pass + + +class Crontab(InstallMethod): + def __init__(self) -> None: + cwd = os.getcwd() + hour = random.randint(0, 23) + minute = random.randint(0, 59) + self.line = f'{minute} {hour} * * * "{cwd}/psij-ci-run" --log' + + def is_available(self) -> Tuple[bool, str | None]: + if not _succeeds("ps -eo command | awk '{print $1}' | grep cron"): + return False, 'not found' + if _succeeds('crontab -l 2>&1 | grep "not allowed"'): + return False, 'not allowed' + return True, None + + def _crt_crontab(self) -> str: + ec, out = _run('crontab -l') + if ec != 0: + if 'no crontab for' in out: + return '' + else: + raise Exception(f'Error getting crontab: {out}') + else: + return out + + def already_installed(self) -> bool: + cwd = os.getcwd() + out = self._crt_crontab() + if f'cd "{cwd}" && ./psij-ci-run' in out or '"{cwd}/psij-ci-run"' in out: + return True + else: + return False + + def install(self) -> None: + crt = self._crt_crontab() + _save_env() + _must_succeed('crontab -', input=f'{crt}\n{self.line}\n') + + @property + def preview(self) -> str: + return self.line + + @property + def name(self) -> str: + return 'cron' + + @property + def label(self) -> str: + return 'Cron - the standard UNIX job scheduler' + + @property + def help_message(self) -> str: + return 'Uses the Cron scheduler to schedule daily runs of the tests.' + + +class At(InstallMethod): + def __init__(self) -> None: + self.hour = random.randint(0, 23) + self.minute = random.randint(0, 59) + self.cmd = f'psij-ci-run --reschedule {self.hour}:{self.minute} --log' + + def is_available(self) -> Tuple[bool, str | None]: + fd, path = mkstemp() + os.close(fd) + + try: + ec, out = _run('at now', input=f'rm {path}') + if ec != 0: + return False, 'not found' + time.sleep(0.2) + if out.startswith('job'): + job_no = out.split()[1] + _run(f'atrm {job_no}') + if 'No atd' in out: + return False, 'not running' + if os.path.exists(path): + os.remove(path) + return False, 'unknown error' + return True, None + finally: + try: + os.remove(path) + except FileNotFoundError: + pass + + def already_installed(self) -> bool: + out = _must_succeed('atq') + out = out.strip() + if len(out) == 0: + return False + for line in out.split('\n'): + job_no = line.split()[0] + job = _must_succeed(f'at -c {job_no}') + if 'psij-ci-run' in job: + return False + return True + + def install(self) -> None: + _must_succeed(f'at {self.hour}:{self.minute}', input=self.cmd) + + @property + def preview(self) -> str: + return f'echo "{self.cmd}" | at {self.hour}:{self.minute}' + + @property + def name(self) -> str: + return 'at' + + @property + def label(self) -> str: + return 'at - the standard UNIX "at" command' + + @property + def help_message(self) -> str: + return ('Uses the "at" job scheduler to schedule the tests which then re-schedule ' + 'themselves.') + + +class Screen(InstallMethod): + def __init__(self) -> None: + self.cmd = 'screen -d -m bash -c "./psij-ci-run --repeat --log"' + + def is_available(self) -> Tuple[bool, str | None]: + if _succeeds('which screen'): + return True, None + else: + return False, 'not found' + + def already_installed(self) -> bool: + ec, out = _run('screen -list | grep psij-ci-run') + return ec == 0 + + def install(self) -> None: + _must_succeed(self.cmd) + + @property + def preview(self) -> str: + return self.cmd + + @property + def name(self) -> str: + return 'screen' + + @property + def label(self) -> str: + return 'Screen - the Screen terminal multiplexer' + + @property + def help_message(self) -> str: + return ('Uses GNU Screen to run the tests in a Screen session. Does not persist across ' + 'reboots.') + + +class Custom(InstallMethod): + def is_available(self) -> Tuple[bool, str | None]: + return True, None + + def already_installed(self) -> bool: + return False + + def install(self) -> None: + _save_env() + + @property + def preview(self) -> str: + cwd = os.getcwd() + return f'"{cwd}/psij-ci-run" --log' + + @property + def name(self) -> str: + return 'custom' + + @property + def label(self) -> str: + return 'Custom - run with custom tool' + + @property + def help_message(self) -> str: + return ('Prints a command that can be used to run the tests, which can then be used with ' + 'a custom scheduler.') + + +METHODS = [ + Crontab(), + At(), + Screen(), + Custom() +] + + +def existing() -> InstallMethod | None: + for method in METHODS: + if method.already_installed(): + return method + else: + log.write(f'Not installed {method.name}\n') + return None diff --git a/tests/installer/log.py b/tests/installer/log.py new file mode 100644 index 00000000..bb5bc75c --- /dev/null +++ b/tests/installer/log.py @@ -0,0 +1 @@ +log = open('psij-ci-setup.log', 'w', buffering=1) diff --git a/tests/installer/main.py b/tests/installer/main.py new file mode 100644 index 00000000..5b938b78 --- /dev/null +++ b/tests/installer/main.py @@ -0,0 +1,258 @@ +from .terminal import run_patches, terminal_supports_unicode +run_patches() + +import asyncio +import types + +from .dialogs import ExitConfirmDialog +from .state import State +from .log import log +from .panels.basic_info_panel import BasicInfoPanel +from .panels.batch_scheduler_panel import BatchSchedulerPanel +from .panels.intro_panel import IntroPanel +from .panels.key_panel import KeyPanel +from .panels.schedule_panel import SchedulePanel +from .panels.complete_panel import CompletePanel +from .widgets import ShortcutButton + +from textual import on, events +from textual.app import App, ComposeResult +from textual.containers import Horizontal, Vertical +from textual.widgets import Footer, Header, Static, Label, Button +from typing import Optional, Dict, cast, List + + +class PSIJCIInstallWizard(App[object]): + CSS_PATH = 'style.tcss' + + ENABLE_COMMAND_PALETTE = False + + BINDINGS = [ + ('down', 'scroll_down'), + ('up', 'scroll_up'), + ('pageup', 'prev_page'), + ('pagedown', 'next_page'), + ('ctrl+q', 'confirm_quit', 'Quit'), + ('ctrl+c', 'quit', 'Quit'), + ('x', 'confirm_quit'), + ('alt+x', 'confirm_quit'), + ('ctrl+x', 'confirm_quit'), + ('escape', 'confirm_quit') + ] + + def __init__(self, conftest: types.ModuleType, ci_runner: types.ModuleType) -> None: + super().__init__() + self.state = State(conftest, ci_runner) + self.body = None + self.shortcuts: Dict[str, List[ShortcutButton]] = {} + self.key_panel = KeyPanel(self.state) + self.scheduler_panel = BatchSchedulerPanel(self.state) + self.install_panel = SchedulePanel(self.state) + + self.panels = [ + IntroPanel(self.state), + BasicInfoPanel(self.state), + self.key_panel, + self.scheduler_panel, + self.install_panel, + CompletePanel(self.state) + ] + + def disable_install(self) -> None: + self.state.disable_install = True + self.install_panel.disabled = True + self.get_widget_by_id('panel-label-install').disabled = True + + def compose(self) -> ComposeResult: + self.prev_button = ShortcutButton('&Previous', disabled=True, id='btn-prev') + self.next_button = ShortcutButton('&Next', variant='primary', id='btn-next') + + yield Header(icon='=') + with Horizontal(id='root'): + with Vertical(id='sidebar', classes='sidebar') as sidebar: + self.sidebar = sidebar + for panel in self.panels: + yield Label(' ' + panel.label, id=f'panel-label-{panel.name}', + classes='panel-label') + with Vertical(id='main'): + with Vertical(id='body', classes='v-scrollable'): + for panel in self.panels: + yield panel + with Horizontal(id='bottom'): + yield Static('', id='left-padding') + with Horizontal(id='buttons'): + yield self.prev_button + yield self.next_button + yield Static('', id='right-padding') + yield Footer() + + async def on_mount(self) -> None: + for panel in self.panels: + panel.widgets.disabled = True + if self.size.height < 35: + self.add_class('small') + else: + self.add_class('large') + if not terminal_supports_unicode(): + self.add_class('no-unicode') + self.set_default_executor() + + body = self.get_widget_by_id('body') + + self.watch(body, 'scroll_y', self._on_y_scroll) + self.next_button.focus() + self.activate_panel(0) + self.copy_to_clipboard('Testing, 1, 2, 3') + + async def _on_y_scroll(self, y: int) -> None: + log.write(f'on_y_scroll({y})\n') + cy = y + self.size.height / 2 + for i in range(len(self.panels)): + # parent because all panels are wrapped + panel = self.panels[i] + region = panel.widgets.virtual_region + if cy >= region.y and cy <= region.bottom: + self.activate_panel(i, False) + break + + def done(self) -> None: + super().exit() + + def activate_panel(self, n: int, scroll: Optional[bool] = True) -> None: + if n == len(self.panels): + self.done() + asyncio.create_task(self.a_activate_panel(n, scroll)) + + async def a_activate_panel(self, n: int, scroll: Optional[bool] = True) -> None: + try: + if n == self.state.active_panel: + return + new_panel = self.panels[n] + log.write(f'activate_panel({n}: {new_panel}, scroll={scroll})\n') + labels = cast(List[Label], self.sidebar.children) + if self.state.active_panel is not None: + old_panel = self.panels[self.state.active_panel] + if scroll and n > self.state.active_panel: + if not await old_panel.validate(): + return + label = labels[self.state.active_panel] + label.update(' ' + old_panel.label) + old_panel.widgets.disabled = True + old_panel.active = False + for i in range(n): + labels[i].styles.text_style = 'bold' + for i in range(n, len(labels)): + labels[i].styles.text_style = 'none' + label = labels[n] + label.update('> ' + new_panel.label) + self.prev_button.disabled = n == 0 + self.set_next_button(n == len(self.panels) - 1) + if scroll: + new_panel.anchor() + + self.state.active_panel = n + new_panel.active = True + await new_panel.activate() + new_panel.widgets.disabled = False + except Exception as ex: + import traceback + log.write(f'Ex: {ex}\n') + traceback.print_exc(file=log) + + def set_next_button(self, exit: bool) -> None: + btn_next = self.get_widget_by_id('btn-next') + assert isinstance(btn_next, ShortcutButton) + if exit: + btn_next.set_label('E&xit') + else: + btn_next.set_label('&Next') + + def register_button_shortcut(self, char: str, btn: ShortcutButton) -> None: + char = char.lower() + if char not in self.shortcuts: + self.shortcuts[char] = [] + self.shortcuts[char].append(btn) + self.bind(f'{char}, alt+{char}', 'on_key', show=False) + + def on_key(self, event: events.Key) -> None: + k = event.key + if k.startswith('alt+'): + k = k[4:] + if k in self.shortcuts: + btns = self.shortcuts[k] + for btn in btns: + if not btn.disabled and btn.is_on_screen: + btn.press() + + def action_next_page(self) -> None: + self.next_button.press() + + def action_prev_page(self) -> None: + self.prev_button.press() + + def action_scroll_down(self) -> None: + body = self.get_widget_by_id('body') + body.scroll_down() + + def action_scroll_up(self) -> None: + body = self.get_widget_by_id('body') + body.scroll_up() + + @on(Button.Pressed, '#btn-next') + async def next_item(self) -> None: + try: + assert self.state.active_panel is not None + self.activate_panel(self.next_enabled_panel(self.state.active_panel, 1)) + except IndexError: + log.write('No next panel. Exiting\n') + await self.action_quit() + + def next_enabled_panel(self, crt: int, delta: int) -> int: + crt += delta + while crt < len(self.panels) and crt >= 0: + if self.panels[crt].disabled: + crt += delta + else: + return crt + raise IndexError('No next enabled panel') + + @on(Button.Pressed, '#btn-prev') + async def prev_item(self) -> None: + assert self.state.active_panel is not None + self.activate_panel(self.next_enabled_panel(self.state.active_panel, -1)) + + def _focus_next(self) -> None: + btn = self.get_widget_by_id('btn-next') + btn.focus() + + def action_confirm_quit(self) -> None: + asyncio.create_task(self._confirm_quit()) + + async def _confirm_quit(self) -> None: + result = await ExitConfirmDialog().run(self) + if result == 'cancel': + return + await self.action_quit() + + async def action_quit(self) -> None: + install_method = self.state.install_method + if install_method is not None and install_method.name == 'custom': + self.exit(message=f'Tests can be run with the following command:\n' + f'\t{install_method.preview}') + else: + self.exit() + + def set_default_executor(self) -> None: + label, name = self.state.get_executor() + + batch_warner = self.get_widget_by_id('warn-no-batch') + if name == 'none': + batch_warner.visible = True + else: + batch_warner.visible = False + + self.scheduler_panel.set_scheduler(name) + + +def run(conftest: types.ModuleType, ci_runner: types.ModuleType) -> None: + PSIJCIInstallWizard(conftest, ci_runner).run() diff --git a/tests/installer/panels/basic_info_panel.py b/tests/installer/panels/basic_info_panel.py new file mode 100644 index 00000000..0aac38b2 --- /dev/null +++ b/tests/installer/panels/basic_info_panel.py @@ -0,0 +1,100 @@ +import random +import socket +import string + +from .panel import Panel +from ..log import log +from ..widgets import ShortcutButton + +from textual import on +from textual.containers import Vertical, Horizontal +from textual.widget import Widget +from textual.widgets import Input, Label, Button + + +FQDN = socket.getfqdn() + + +class BasicInfoPanel(Panel): + def _build_widgets(self) -> Widget: + self.name_input = Input(placeholder='Enter machine name', id='name-input') + self.email_input = Input(placeholder='Enter email', id='email-input') + return Vertical( + Label('Some basic information', classes='header'), + Label('The name should be something descriptive, such as ' + '"aurora.alcf.anl.gov". This name will be displayed on the online testing ' + 'dashboard.', classes='help-text media-large', shrink=True, expand=True), + Vertical( + Label('Machine name (e.g. echo.example.net):', classes='form-label'), + Horizontal( + self.name_input, + ShortcutButton('Use &FQDN', id='btn-use-fqdn'), + id='name-input-group' + ), + classes='form-row h-auto' + ), + Label('Your email allows us to contact you if we need more information about failing ' + 'tests. It does not appear publicly on the dashboard.', + classes='help-text', shrink=True, expand=True), + Vertical( + Label('Maintainer email:', classes='form-label'), + self.email_input, + classes='form-row h-auto' + ), + classes='panel' + ) + + @property + def label(self) -> str: + return 'Basic info' + + @property + def name(self) -> str: + return 'basic-info' + + async def activate(self) -> None: + log.write('Activate basic info panel\n') + if self.name_input.value == '': + conf_name = self.state.conf['id'] + log.write(f'Conf name: {conf_name}\n') + val = None + if conf_name == 'hostname': + val = FQDN + elif conf_name == 'random': + val = ''.join(random.choices(string.ascii_uppercase + string.digits, k=8)) + elif len(conf_name) > 0 and conf_name[0] == '"' and conf_name[-1] == '"': + val = conf_name[1:-1] + else: + val = '' + assert val is not None + self.name_input.value = val + if self.email_input.value == '': + self.email_input.value = self.state.conf['maintainer_email'] + self.name_input.disabled = False + self.name_input.focus(False) + + async def validate(self) -> bool: + log.write(f'name value: {self.name_input.value}\n') + name_valid = self.name_input.value != '' + self.name_input.set_class(not name_valid, 'invalid', update=True) + + if name_valid: + self.state.update_conf('id', self.name_input.value) + if self.email_input.value: + self.state.update_conf('maintainer_email', self.email_input.value) + return name_valid + + @on(Input.Submitted, '#name-input') + def name_submitted(self) -> None: + email = self.get_widget_by_id('email-input') + email.focus(False) + + @on(Input.Submitted, '#email-input') + def email_submitted(self) -> None: + self.app._focus_next() # type: ignore + + @on(Button.Pressed, '#btn-use-fqdn') + def use_fqdn_pressed(self) -> None: + name_input = self.get_widget_by_id('name-input') + assert isinstance(name_input, Input) + name_input.value = FQDN diff --git a/tests/installer/panels/batch_scheduler_panel.py b/tests/installer/panels/batch_scheduler_panel.py new file mode 100644 index 00000000..265c37cd --- /dev/null +++ b/tests/installer/panels/batch_scheduler_panel.py @@ -0,0 +1,418 @@ +import asyncio +import re +from typing import Tuple, Optional, cast + +from psij import JobExecutor, Job, JobSpec, ResourceSpecV1, JobAttributes, JobState +from .panel import Panel +from ..dialogs import TestJobsDialog +from ..log import log +from ..state import Attr +from ..widgets import MSelect, ShortcutButton + +from textual import on +from textual.app import ComposeResult +from textual.containers import Horizontal, Vertical +from textual.screen import ModalScreen +from textual.widget import Widget +from textual.widgets import Label, Input, Button, TextArea, Checkbox, Select + + +class EditAttrsScreen(ModalScreen[None]): + BINDINGS = [ + ('F1', 'context_help', 'Help'), + ('down', 'input_down'), + ('up', 'input_up'), + ('c', 'cancel'), + ('alt+c', 'cancel'), + ('a', 'apply'), + ('alt+a', 'apply'), + ('ctrl+q', 'quit') + ] + + def __init__(self, panel: 'BatchSchedulerPanel') -> None: + super().__init__() + self.panel = panel + self.state = panel.state + + def compose(self) -> ComposeResult: + yield Vertical( + Label('Edit custom attributes.', + classes='header', shrink=True, expand=True), + Label('If your scheduler requires non-standard parameters (e.g., "constraint=knl"), ' + 'enter them here.', + classes='help-text', shrink=True, expand=True), + Vertical( + Horizontal( + Label('Name', classes='th'), + Label('Value', classes='th'), + Label('Filter', classes='th'), + classes='attr-header h-auto' + ), + self._make_row(0)[0], + id='attr-rows', classes='attrs' + ), + Horizontal( + Button('[b bright_yellow]C[/b bright_yellow]lose', variant='error', + id='btn-attrs-cancel', classes='m-r-1'), + Button('[b bright_yellow]A[/b bright_yellow]pply', variant='primary', + id='btn-attrs-apply', classes='m-r'), + classes='action-area' + ), + id='edit-attrs-dialog', classes='dialog' + ) + + def _make_row(self, index: int) -> Tuple[Widget, Input, Input, Input]: + name = Input('', id=f'attr-name-{index}', classes='attr-name td') + value = Input('', id=f'attr-value-{index}', classes='attr-value td') + filter = Input('', id=f'attr-filter-{index}', classes='attr-filter td') + return Horizontal(name, value, filter, classes='attr-row h-auto'), name, value, filter + + def _get_row(self, input: Input) -> int: + id = input.id + assert id is not None + return int(id.split('-')[-1]) + + def action_input_down(self) -> None: + self._move_input_focus(1) + + def action_input_up(self) -> None: + self._move_input_focus(-1) + + def _move_input_focus(self, d: int) -> None: + q = self.query('*:focus') + if len(q) > 0: + focused = q.first() + if focused.has_class('td'): + id = focused.id + assert id is not None + id_parts = id.split('-') + index = int(id_parts[-1]) + q2 = self.query('#' + '-'.join(id_parts[:-1]) + '-' + str(index + d)) + if len(q2) > 0: + q2.first().focus() + else: + self.get_widget_by_id('btn-attrs-apply').focus() + else: + if focused.id == 'btn-attrs-apply': + if d == 1: + self.get_widget_by_id('attr-name-0').focus() + else: + self.get_widget_by_id('attr-rows').children[-1].children[0].focus() + + @on(Input.Submitted, '.attr-name') + def name_submitted(self, event: Input.Submitted) -> None: + log.write(f'input submitted {event.input.id}\n') + event.input.remove_class('invalid') + assert event.input.parent is not None + if event.input.value != '': + self._ensure_row(self._get_row(event.input) + 1) + event.input.parent.children[1].focus() + + @on(Input.Submitted, '.attr-value') + def value_submitted(self, event: Input.Submitted) -> None: + event.input.remove_class('invalid') + assert event.input.parent is not None + if event.input.value != '': + self._ensure_row(self._get_row(event.input) + 1) + event.input.parent.children[2].focus() + + @on(Input.Submitted, '.attr-filter') + def filter_submitted(self, event: Input.Submitted) -> None: + event.input.remove_class('invalid') + row = self._get_row(event.input) + if event.input.value != '': + self._ensure_row(row + 1) + next_name = self.query(f'#attr-name-{row + 1}') + if next_name: + next_name.focus() + else: + self.get_widget_by_id('btn-attrs-apply').focus() + + def _ensure_row(self, row: int) -> Tuple[Input, Input, Input]: + next_name = self.query(f'#attr-name-{row}') + if not next_name: + rows = self.get_widget_by_id('attr-rows') + row_widget, name, value, filter = self._make_row(row) + rows.mount(row_widget) + else: + row_widget = self.get_widget_by_id('attr-rows').children[row + 1] + name = cast(Input, row_widget.children[0]) + value = cast(Input, row_widget.children[1]) + filter = cast(Input, row_widget.children[2]) + return name, value, filter + + def on_mount(self) -> None: + ix = 0 + for attr in self.state.attrs: + name_input, value_input, filter_input = self._ensure_row(ix) + name_input.value = attr.name + value_input.value = attr.value + if attr.filter != '.*': + filter_input.value = attr.filter + ix += 1 + + def action_quit(self) -> None: + self.app.exit() + + def action_apply(self) -> None: + if self._validate_and_commit(): + self.app.pop_screen() + + def action_cancel(self) -> None: + self.app.pop_screen() + + @on(Button.Pressed, '#btn-attrs-apply') + def apply_pressed(self) -> None: + self.action_apply() + + @on(Button.Pressed, '#btn-attrs-cancel') + def cancel_pressed(self) -> None: + self.action_cancel() + + def _tag_input(self, is_valid: bool, input: Input, + first_invalid: Optional[Input]) -> Optional[Input]: + if not is_valid: + input.add_class('invalid') + if first_invalid is None: + return input + return first_invalid + + def _re_valid(self, restr: str) -> bool: + try: + re.compile(restr) + return True + except Exception: + return False + + def _validate_and_commit(self) -> bool: + rows = self.get_widget_by_id('attr-rows') + attrs = [] + first_invalid = None + for row_index in range(1, len(rows.children)): + row = rows.children[row_index] + name = row.children[0] + value = row.children[1] + filter = row.children[2] + assert (isinstance(name, Input) and isinstance(value, Input) + and isinstance(filter, Input)) + if name.value == '' and value.value == '' and filter.value == '': + continue + # at least one set + name_valid = name.value != '' + value_valid = value.value != '' + filter_valid = filter.value == '' or self._re_valid(filter.value) + first_invalid = self._tag_input(name_valid, name, first_invalid) + first_invalid = self._tag_input(value_valid, value, first_invalid) + first_invalid = self._tag_input(filter_valid, filter, first_invalid) + if name_valid and value_valid and filter_valid: + attrs.append(Attr(filter.value, name.value, value.value)) + if first_invalid is None: + self.state.set_custom_attrs(attrs) + self.panel.update_attrs() + else: + first_invalid.focus() + return first_invalid is None + + +class BatchSchedulerPanel(Panel): + BINDINGS = [ + ('t', 'toggle_test_job'), + ('alt+t', 'toggle_test_job') + ] + + def _build_widgets(self) -> Widget: + return Vertical( + Label('Select and configure a batch system.', classes='header'), + Vertical( + Label('Your system does not appear to have a batch scheduler. If you are certain ' + 'that this is wrong, you can select one below. If not, tests will be run ' + 'using non-batch executors.', classes='help-text media-large', + shrink=True, expand=True), + id='warn-no-batch' + ), + Horizontal( + Vertical( + Label('Batch system:', classes='form-label'), + MSelect( + [('Select...', 'none'), ('Local only', 'local'), ('Slurm', 'slurm'), + ('PBS', 'pbs'), ('LSF', 'lsf'), ('Cobalt', 'cobalt')], + id='batch-selector', + allow_blank=False), + classes='form-col-3 form-row' + ), + Vertical( + Label('Queue:', classes='form-label'), + Input(id='queue-input'), + classes='form-col-3 form-row batch-valid' + ), + Vertical( + Label('Account/project:', classes='form-label'), + Input(id='account-input'), + classes='form-col-3 form-row batch-valid' + ), + classes='w-100 form-row', id='batch-system-group' + ), + Horizontal( + Vertical( + Label('Custom attributes:', classes='form-label'), + TextArea('', id='custom-attrs', read_only=True, soft_wrap=False), + id='attr-cell', + classes='form-col-2 h-auto' + ), + Vertical( + Label('', classes='form-label'), + ShortcutButton('&Edit attributes', id='btn-edit-attrs'), + Checkbox('Run [b bright_yellow]t[/b bright_yellow]est job', + id='cb-run-test-job', classes='m-t-1'), + classes='form-col-2 h-auto' + ), + classes='w-100 h-auto batch-valid' + ), + classes='panel' + ) + + async def validate(self) -> bool: + if self.state.run_test_job and self.state.scheduler is not None: + return await self.run_test_jobs() + else: + return True + + @property + def label(self) -> str: + return 'Scheduler' + + @property + def name(self) -> str: + return 'scheduler' + + async def activate(self) -> None: + self.update_attrs() + selector = self.get_widget_by_id('batch-selector') + selector.focus(False) + + def update_attrs(self) -> None: + s = '' + for attr in self.state.attrs: + if attr.filter == '.*': + s += f'{attr.name}: {attr.value}\n' + else: + s += f'{attr.name}: {attr.value} ({attr.filter})\n' + control = self.get_widget_by_id('custom-attrs') + assert isinstance(control, TextArea) + control.text = s + + @on(Select.Changed, '#batch-selector') + def batch_system_selected(self) -> None: + selector = cast(Select[str], self.get_widget_by_id('batch-selector')) + sched = selector.selection + disabled = (sched is None or sched == 'none' or sched == 'local') + if disabled: + self.state.scheduler = None + else: + self.state.scheduler = sched + for widget in self.query('.batch-valid'): + widget.disabled = disabled + if sched == 'local': + self.app._focus_next() # type: ignore + else: + self.get_widget_by_id('queue-input').focus() + + def set_scheduler(self, name: str) -> None: + selector = self.get_widget_by_id('batch-selector') + assert isinstance(selector, Select) + selector.value = name + + @on(Input.Submitted, '#queue-input') + def queue_submitted(self) -> None: + bottom = self.get_widget_by_id('account-input') + bottom.focus() + + @on(Input.Submitted, '#account-input') + def account_submitted(self) -> None: + self.app._focus_next() # type: ignore + + @on(Button.Pressed, '#btn-edit-attrs') + def action_edit_attrs(self) -> None: + if not self.active or self.state.scheduler is None: + return + self.app.push_screen(EditAttrsScreen(self)) + + def action_toggle_test_job(self) -> None: + cb = self.get_widget_by_id('cb-run-test-job') + if not self.active or self.state.scheduler is None: + return + assert isinstance(cb, Checkbox) + cb.value = not cb.value + + @on(Checkbox.Changed, '#cb-run-test-job') + def action_cb_test_job_changed(self, event: Checkbox.Changed) -> None: + self.run_test_job = event.checkbox.value + + async def run_test_jobs(self) -> bool: + jd = TestJobsDialog() + self.app.push_screen(jd) + # without this, the widgets in TestJobsDialog do not seem to be accessible + await asyncio.sleep(0.1) + j1 = await self._run_test_job(jd, 1, 'Single node job', None, '') + j2 = await self._run_test_job(jd, 2, 'Multi node job ', ResourceSpecV1(node_count=4), + f'test_nodefile[{self.state.scheduler}:single') + + if j1 and j2: + jd.focus_continue_button() + else: + jd.focus_back_button() + + return await jd.wait() + + async def _run_test_job(self, jd: TestJobsDialog, job_no: int, label: str, + rspec: Optional[ResourceSpecV1], test_name: str) -> bool: + try: + jd.set_running(job_no) + await asyncio.sleep(2) + + job = self._launch_job(test_name, rspec) + + await self._wait_for_queued_state(job) + try: + job.cancel() + except Exception as ex: + log.write(f'Failed to cancel job: {ex}') + jd.set_status(job_no, 'OK', 'status-succeeded') + return True + except Exception as ex: + jd.log_error(label, ex) + jd.set_status(job_no, 'Failed', 'status-failed') + return False + + def _launch_job(self, test_name: str, rspec: Optional[ResourceSpecV1]) -> Job: + s = self.state.scheduler + assert s is not None + + ex = JobExecutor.get_instance(s) + + attrs = JobAttributes() + account = self.state.conf.get('account', '') + queue = self.state.conf.get('queue', '') + if account != '': + attrs.account = account + if queue != '': + attrs.queue_name = queue + for attr in self.state.attrs: + if re.match(attr.filter, test_name): + attrs.set_custom_attribute(attr.name, attr.value) + + job = Job(JobSpec('/bin/date', attributes=attrs, resources=rspec)) + ex.submit(job) + log.write('Job submitted\n') + return job + + async def _wait_for_queued_state(self, job: Job) -> None: + while True: + status = job.status + log.write(f'Job status: {status}\n') + state = status.state + if state == JobState.QUEUED or state.is_greater_than(JobState.QUEUED): + if state == JobState.FAILED: + raise Exception(status.message) + return + await asyncio.sleep(0.2) diff --git a/tests/installer/panels/complete_panel.py b/tests/installer/panels/complete_panel.py new file mode 100644 index 00000000..a7e605b5 --- /dev/null +++ b/tests/installer/panels/complete_panel.py @@ -0,0 +1,30 @@ +from .panel import Panel +from textual.containers import Vertical +from textual.widget import Widget +from textual.widgets import Label + + +class CompletePanel(Panel): + def _build_widgets(self) -> Widget: + return Vertical( + Label('Installation complete.', + classes='header', shrink=True, expand=True), + Label('The PSI/J CI tests have been installed. You can press the Exit button below ' + 'to exit this installer.', + classes='main-text', shrink=True, expand=True), + classes='panel' + ) + + @property + def label(self) -> str: + return 'Complete' + + @property + def name(self) -> str: + return 'complete' + + async def validate(self) -> bool: + return True + + async def activate(self) -> None: + pass diff --git a/tests/installer/panels/intro_panel.py b/tests/installer/panels/intro_panel.py new file mode 100644 index 00000000..afbb5d62 --- /dev/null +++ b/tests/installer/panels/intro_panel.py @@ -0,0 +1,44 @@ +from .panel import Panel +from textual.containers import Vertical +from textual.widget import Widget +from textual.widgets import Label + +from ..dialogs import ExistingInstallConfirmDialog +from ..log import log +from ..install_methods import existing + + +class IntroPanel(Panel): + def _build_widgets(self) -> Widget: + return Vertical( + Label('This tool will guide you in setting up the PSI/J nightly tests.', + classes='header', shrink=True, expand=True), + Label('Hint: you can likely use your terminal\'s application mode (typically the' + ' Shift or Option keys) to copy and paste text. For example, Shift+Drag to' + ' select text or Shift+Ctrl+V to paste from the clipboard.', + classes='help-text', shrink=True, expand=True), + classes='panel' + ) + + @property + def label(self) -> str: + return 'Introduction' + + @property + def name(self) -> str: + return 'intro' + + async def validate(self) -> bool: + return True + + async def activate(self) -> None: + log.write('Intro activate\n') + if self.state.disable_install: + return + m = existing() + if m is not None: + result = await ExistingInstallConfirmDialog(m.name).run(self.app) + if result == 'quit': + self.app.exit() + if result == 'update': + self.app.disable_install() # type: ignore diff --git a/tests/installer/panels/key_panel.py b/tests/installer/panels/key_panel.py new file mode 100644 index 00000000..593654cf --- /dev/null +++ b/tests/installer/panels/key_panel.py @@ -0,0 +1,228 @@ +import asyncio +from typing import Optional + +from ..dialogs import KeyRequestDialog, ProgressDialog, ErrorDialog +from ..log import log +from .panel import Panel +from ..state import KEY_PATH +from ..widgets import ShortcutButton + +from textual import on +from textual.containers import Vertical, Horizontal +from textual.widget import Widget +from textual.widgets import Input, Label, MaskedInput, Button + + +class KeyPanel(Panel): + def _build_widgets(self) -> Widget: + self.email_input = Input(placeholder='Enter email', id='key-email-input') + self.request_button = ShortcutButton('&Request key', id='btn-request-key') + return Vertical( + Label('Dashboard authentication key', classes='header'), + Label('A key associates your test result uploads with a verified email and allows us ' + 'to prevent unauthorized uploads.', + classes='help-text', shrink=True, expand=True), + Label(f'A valid key was found in {KEY_PATH}', classes='p-l m-b -success hidden', + id='msg-auth-key-valid', shrink=True, expand=True), + Label(f'No key was found in {KEY_PATH}', classes='p-l m-b -error hidden', + id='msg-auth-key-not-found', shrink=True, expand=True), + Label(f'Invalid key found in {KEY_PATH}', classes='p-l m-b -error hidden', + id='msg-auth-key-invalid', shrink=True, expand=True), + Vertical( + Label('Your email address:', classes='form-label'), + Horizontal( + self.email_input, + self.request_button + ), + classes='form-row h-auto', id='key-email-input-group' + ), + Vertical( + Label('Or enter a key below:', classes='form-label'), + MaskedInput(template='NNNNNNNNNNNNNNNN:NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN', # noqa: E501 + id='key-input'), + Label('', classes='form-label -error', id='key-check-error'), + classes='form-row h-auto', id='key-input-group' + ), + classes='panel' + ) + + @property + def label(self) -> str: + return 'Key' + + @property + def name(self) -> str: + return 'key' + + def _show_message(self, id: str) -> None: + for i in ['msg-auth-key-valid', 'msg-auth-key-invalid', 'msg-auth-key-not-found']: + if i == id: + self.get_widget_by_id(i).set_class(False, 'hidden') + else: + self.get_widget_by_id(i).set_class(True, 'hidden') + + async def activate(self) -> None: + if self.state.has_key and self.state.key_is_valid is None: + log.write('Verifying key...\n') + self.state.key_is_valid = await self.verify_key() + input_active = self.update() + if self.email_input.value == '': + maintainer_email = self.state.conf['maintainer_email'] + log.write(f'No email, setting to maintainer email: {maintainer_email}\n') + self.email_input.value = maintainer_email + if input_active: + self.email_input.focus(False) + + @on(Button.Pressed, '#btn-request-key') + def action_request_key(self) -> None: + email_input = self.get_widget_by_id('key-email-input') + assert isinstance(email_input, Input) + email = email_input.value + asyncio.create_task(self.request_key(email)) + + @on(Input.Submitted, '#key-email-input') + def key_email_submitted(self) -> None: + btn = self.get_widget_by_id('btn-request-key') + btn.focus() + + @on(Input.Changed, '#key-input') + def key_input_changed(self) -> None: + # remove error message + self._set_key_check_error('') + self.get_widget_by_id('key-input').remove_class('invalid') + + @on(Input.Submitted, '#key-input') + async def key_submitted(self, event: Input.Changed) -> None: + key = event.value + try: + valid = await self.verify_key(key) + self.get_widget_by_id('key-input').set_class(not valid, 'invalid') + if not valid: + self._set_key_check_error('Invalid key.') + else: + self._set_key_check_error('') + self._key_received(key) + self.update() + except Exception as ex: + log.write(f'Ex: {ex}\n') + + def _set_key_check_error(self, text: str) -> None: + key_check_error = self.get_widget_by_id('key-check-error') + assert isinstance(key_check_error, Label) + key_check_error.update(text) + + def _key_received(self, key: str) -> None: + with open(KEY_PATH, 'w') as f: + f.write(key) + self.state.has_key = True + self.state.key_is_valid = True + self.app._focus_next() # type: ignore + + def update(self) -> bool: + input_active = True + if self.state.has_key: + if self.state.key_is_valid: + self._show_message('msg-auth-key-valid') + input_active = False + else: + self._show_message('msg-auth-key-invalid') + else: + self._show_message('msg-auth-key-not-found') + + self.get_widget_by_id('key-email-input-group').disabled = not input_active + self.get_widget_by_id('key-input-group').disabled = not input_active + return input_active + + async def verify_key(self, key: Optional[str] = None) -> bool: + if not key: + key = self.state.key + if key is None: + return False + # some offline quick validation + if len(key) != 65: + return False + cix = key.find(':') + if cix != 16: + return False + + self.app.push_screen(ProgressDialog('Verifying key...')) + try: + result = await self.state.request('/authVerifyKey', {'key': key}, 'Key verification', + self.display_error_dialog) + if result: + success = result['success'] + assert isinstance(success, bool) + return success + finally: + self.app.pop_screen() + return False + + async def request_key(self, email: str) -> None: + key = await self._key_request(email) + if key: + self._key_received(key) + self.update() + + async def _key_request(self, email: str) -> Optional[str]: + self.app.push_screen(ProgressDialog('Initializing request...')) + try: + result = await self.state.request('/keyRequestInit', {'email': email}, + 'Key request initialization', + self.display_error_dialog) + finally: + self.app.pop_screen() + if not result: + return None + request_id = result['id'] + assert isinstance(request_id, str) + base_url = self.state.conf['server_url'] + d = KeyRequestDialog(f'{base_url}/auth/{request_id}') + self.app.push_screen(d) + try: + return await self._run_key_request_loop(request_id, d) + finally: + log.write('Key request loop done\n') + self.app.pop_screen() + + async def _run_key_request_loop(self, request_id: str, d: KeyRequestDialog) -> Optional[str]: + while d.is_active: + result = await self.state.request('/keyRequestStatus', {'id': request_id}, + 'Key request status check', + self.display_error_dialog) + log.write(f'Result: {result}\n') + if not result: + return None + + success = result['success'] + + if not success: + error = result['error'] + assert isinstance(error, str) + ed = ErrorDialog('Error requesting key', error) + await ed.run(self.app) + return None + else: + status = result['status'] + assert isinstance(status, str) + d.update_status(status) + if status == 'verified': + return result['key'] # type: ignore + await asyncio.sleep(5) + return None + + async def validate(self) -> bool: + if not self.state.key_is_valid: + email_valid = self.email_input.value != '' + self.email_input.set_class(not email_valid, 'invalid', update=True) + else: + self.email_input.set_class(False, 'invalid', update=True) + self.request_button.set_class(not self.state.key_is_valid, 'invalid', update=True) + assert self.state.key_is_valid is not None + return self.state.key_is_valid + + async def display_error_dialog(self, title: str, message: str) -> None: + try: + d = ErrorDialog(title, message) + await d.run(self.app) + except Exception as ex: + log.write(f'caught {ex}\n') diff --git a/tests/installer/panels/panel.py b/tests/installer/panels/panel.py new file mode 100644 index 00000000..67287015 --- /dev/null +++ b/tests/installer/panels/panel.py @@ -0,0 +1,39 @@ +from abc import abstractmethod + +from textual.containers import Vertical +from textual.widget import Widget + +from ..state import State + + +class Panel(Vertical): + def __init__(self, state: State) -> None: + self.state = state + self._widgets = self._build_widgets() + Vertical.__init__(self, self._widgets, classes='panel-wrapper') + self.active = False + + @property + def widgets(self) -> Widget: + return self._widgets + + @abstractmethod + def _build_widgets(self) -> Widget: + pass + + @property + @abstractmethod + def label(self) -> str: + pass + + @property + @abstractmethod + def name(self) -> str: + pass + + @abstractmethod + async def validate(self) -> bool: + pass + + async def activate(self) -> None: + pass diff --git a/tests/installer/panels/schedule_panel.py b/tests/installer/panels/schedule_panel.py new file mode 100644 index 00000000..0048c579 --- /dev/null +++ b/tests/installer/panels/schedule_panel.py @@ -0,0 +1,127 @@ +from typing import cast + +from .panel import Panel + +from textual.containers import Vertical, Horizontal +from textual.widget import Widget +from textual.widgets import Label, RadioSet, RadioButton, TextArea + +from ..dialogs import ErrorDialog +from ..log import log +from ..install_methods import METHODS, InstallMethod + + +class MRadioSet(RadioSet): + BINDINGS = [ + ('enter,space', 'select') + ] + + def __init__(self, panel: 'SchedulePanel', *radios: RadioButton, id: str | None = None) -> None: + super().__init__(*radios, id=id) + self.panel = panel + + def watch__selected(self) -> None: + super().watch__selected() + if self._selected is not None: + self.panel.radio_focused(cast(RadioButton, self.children[self._selected])) + + def watch_has_focus(self, value: bool) -> None: + if not value: + self.panel.radio_focused(None) + super().watch_has_focus(value) + + def action_select(self) -> None: + if self._selected is not None: + selected = self.children[self._selected] + assert isinstance(selected, RadioButton) + if selected.value: + self.app._focus_next() # type: ignore + + self.action_toggle_button() + + def get_selected_index(self) -> int | None: + return self._selected + + +class SchedulePanel(Panel): + def _build_widgets(self) -> Widget: + radios = [] + labels = [] + + for m in METHODS: + available, msg = m.is_available() + if msg is None: + msg = '' + radios.append(RadioButton(m.label, id=f'btn-{m.name}', disabled=not available)) + labels.append(Label(msg, id=f'label-{m.name}')) + labels_container = Vertical(*labels, id='method-status', classes='h-auto') + + return Vertical( + Label('Install', classes='header'), + Label('This step schedules the tests to be run daily.', + classes='help-text', shrink=True, expand=True), + Vertical( + Label('Installation method:', classes='form-label'), + Horizontal( + MRadioSet(self, *radios, id='rs-method'), + labels_container, + classes='h-auto' + ), + classes='h-auto m-b-1' + ), + Vertical( + Label('Preview', classes='form-label'), + TextArea('-', id='method-preview', classes='', language='bash', + read_only=True, disabled=True), + classes='form-row h-auto' + ), + classes='panel' + ) + + @property + def label(self) -> str: + return 'Install' + + @property + def name(self) -> str: + return 'install' + + def _get_selected_method(self) -> InstallMethod | None: + radio_set = self.get_widget_by_id('rs-method') + assert isinstance(radio_set, MRadioSet) + selected_index = radio_set.get_selected_index() + if selected_index is None: + return None + return METHODS[selected_index] + + async def validate(self) -> bool: + try: + m = self._get_selected_method() + assert m is not None + self.state.install_method = m + m.install() + return True + except Exception as ex: + await ErrorDialog('Error scheduling tests', str(ex)).run(self.app) + return False + + async def activate(self) -> None: + self.get_widget_by_id('rs-method').focus(scroll_visible=False) + for btn in self.query('RadioButton'): + if not btn.disabled: + assert isinstance(btn, RadioButton) + btn.value = True + break + + def radio_focused(self, btn: RadioButton | None) -> None: + log.write(f'focused {btn}\n') + preview = self.get_widget_by_id('method-preview') + assert isinstance(preview, TextArea) + + if btn is not None: + assert btn.id is not None + name = btn.id.split('-')[-1] + for m in METHODS: + if m.name == name: + preview.text = m.preview + return diff --git a/tests/installer/py.typed b/tests/installer/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/tests/installer/state.py b/tests/installer/state.py new file mode 100644 index 00000000..fdcc787f --- /dev/null +++ b/tests/installer/state.py @@ -0,0 +1,172 @@ +import asyncio +import json +import os +import pathlib +import types + +import requests +from collections import namedtuple +from typing import List, Dict, Optional, Callable, Awaitable, Tuple, cast + +from .install_methods import InstallMethod +from .log import log + + +KEY_PATH = pathlib.Path('~/.psij/key').expanduser() + + +Attr = namedtuple('Attr', ['filter', 'name', 'value']) + + +class _Options: + def __init__(self) -> None: + self.custom_attributes = None + + +class ConfWrapper: + def __init__(self, dict: Dict[str, str | int | bool | None]): + self.dict = dict + self.option = _Options() + dict['run_id'] = 'x' + dict['save_results'] = False + dict['upload_results'] = True + dict['branch_name_override'] = None + + def getoption(self, name: str) -> str | int | bool | None: + return self.dict[name] + + +class State: + def __init__(self, conftest: types.ModuleType, ci_runner: types.ModuleType): + self.conf = ci_runner.read_conf('testing.conf') + self.env = conftest._discover_environment(ConfWrapper(self.conf)) + log.write('Conf: ' + str(self.conf) + '\n') + log.write(str(self.env) + '\n') + self.disable_install = False + self.install_method: InstallMethod | None = None + self.active_panel: int | None = None + self.scheduler: str | None = None + self.attrs = self._parse_attributes() + self.run_test_job = True + self.has_key = KEY_PATH.exists() + if self.has_key: + with open(KEY_PATH, 'r') as f: + self.key = f.read().strip() + self.key_is_valid: bool | None = None + log.write(f'has key: {self.has_key}\n') + + def _parse_attributes(self) -> List[Attr]: + attrspec = json.loads('[' + self.conf['custom_attributes'] + ']') + attrs = [] + for filter_entry in attrspec: + filter = filter_entry['filter'] + for name, value in filter_entry['value'].items(): + ns = name.split('.', 2) + if len(ns) != 2: + continue + attrs.append(Attr(filter, ns[1], value)) + return attrs + + def _write_conf_value(self, name: str, value: str) -> None: + self.conf[name] = value + nlen = len(name) + with open('testing.conf', 'r') as old: + with open('testing.conf.new', 'w') as new: + for line in old: + sline = line.strip() + if sline == '' or sline.startswith('#'): + new.write(line) + elif sline.startswith(name) and sline[nlen] in [' ', '\t', '=']: + new.write(f'{name} = {value}\n') + else: + new.write(line) + os.rename('testing.conf.new', 'testing.conf') + + def update_conf(self, name: str, value: str) -> None: + if name == 'id': + self._write_conf_value('id', f'"{value}"') + else: + self._write_conf_value(name, value) + + async def request(self, query: str, data: Dict[str, object], title: str, + error_cb: Callable[[str, str], Awaitable[Dict[str, object] | None]]) \ + -> Dict[str, object] | None: + baseUrl = self.conf['server_url'] + response = await asyncio.to_thread(requests.post, baseUrl + query, data=data) + return await self._check_error(response, title, error_cb) + + async def _check_error(self, response: requests.Response, title: str, + error_cb: Callable[[str, str], Awaitable[Dict[str, object] | None]]) \ + -> Dict[str, object] | None: + log.write(f'Response: {response.text}\n') + if response.status_code != 200: + msg = self._extract_response_message(response.text) + if msg: + error = f'Server responded with an error: {msg}' + else: + error = response.text + else: + data = response.json() + if 'success' in data: + return cast(Dict[str, object], data) + else: + error = 'Unknown error' + log.write(f'Error: {error}\n') + if error is not None: + await error_cb(title + ' failed', error) + return None + + def _extract_response_message(self, html: str) -> Optional[str]: + # standard cherrypy error page + ix = html.find('

') + if ix != -1: + return html[ix + 3:html.find('

', ix)] + else: + return None + + def set_custom_attrs(self, attrs: List[Attr]) -> None: + self.attrs = attrs + self._write_custom_attrs(attrs) + + def _write_custom_attrs(self, attrs: List[Attr]) -> None: + # this one is weird to set, so we need a custom solution + sched = self.scheduler + name = 'custom_attributes' + nlen = len(name) + continued = False + purge = False + with open('testing.conf', 'r') as old, open('testing.conf.new', 'w') as new: + for line in old: + sline = line.strip() + if sline == '' or sline.startswith('#'): + new.write(line) + elif sline.startswith(name) and sline[nlen] in [' ', '\t', '=', '[']: + if not purge: + for attr in attrs: + if attr.filter == '.*' or attr.filter == '': + new.write(f'custom_attributes = ' + f'"{sched}.{attr.name}": "{attr.value}"\n') + else: + new.write(f'custom_attributes[{attr.filter}] = ' + f'"{sched}.{attr.name}": "{attr.value}"\n') + purge = True + if sline.endswith('\\'): + continued = True + elif continued: + # just consume the line + continued = sline.endswith('\\') + else: + new.write(line) + os.rename('testing.conf.new', 'testing.conf') + + def get_executor(self) -> Tuple[str, str]: + if self.env['has_slurm']: + return ('Slurm', 'slurm') + if self.env['has_pbs']: + return ('PBS', 'pbs') + if self.env['has_lsf']: + return ('LSF', 'lsf') + if self.env['has_cobalt']: + return ('Cobalt', 'cobalt') + + return ('None', 'none') diff --git a/tests/installer/style.tcss b/tests/installer/style.tcss new file mode 100644 index 00000000..a7e75183 --- /dev/null +++ b/tests/installer/style.tcss @@ -0,0 +1,738 @@ +Screen { + align: center middle; +} + +Header, Footer { + background: #008080; +} + +Button { + text-style: none; +} + +.hidden { + display: none; +} + +App.large .media-small { + display: none; +} + +App.small .media-large { + display: none; +} + +App.no-unicode RadioSet { + border: tall #202020; + padding: 1; + background: #202020; +} + +App.small RadioSet, App.small.no-unicode RadioSet { + padding-top: 0; + padding-bottom: 0; + padding-left: 1; + padding-right: 1; +} + +* { + scrollbar-color: #006060; +} + +*:disabled { + scrollbar-color: #002080; + scrollbar-background: #000040; +} + + +App.no-unicode RadioSet:focus { + border: tall #0080ff; + background: #282828; +} + +App.no-unicode RadioSet > RadioButton { + color: #202020; +} + +App.no-unicode RadioSet:focus > RadioButton.-selected { + color: #0080ff; + background: #0080ff; +} + +RadioSet:focus { + background: #282828; + background-tint: 0%; +} + +RadioSet > RadioButton { + padding: 0; + background: #202020; + color: #404040; +} + +RadioSet > RadioButton .toggle--button { + background: #000000; + color: #000000; +} + +App.no-unicode RadioSet > RadioButton .toggle--button { + background: #404040; +} + +RadioSet > RadioButton .toggle--label { + color: #d0d0d0; + background: #202020; +} + +RadioSet > RadioButton .toggle--button { + background: #404040; +} + +RadioSet:focus > RadioButton .toggle--button { + background: #404040; +} + +RadioSet > RadioButton.-on .toggle--button { + color: #00ff80; + background: #404040; +} + +RadioSet:focus > RadioButton.-on .toggle--button { + background: #404040; +} + +RadioSet:focus > RadioButton, RadioSet:focus > RadioButton .toggle--label { + background: #282828; + background-tint: 0%; +} + +RadioSet:focus > RadioButton.-selected { + background: #0080ff; + text-style: none; +} + +RadioSet:focus > RadioButton.-selected .toggle--label { + color: #ffffff; + background: #0080ff; +} + +RadioSet > RadioButton:disabled .toggle--label { + color: #808080; +} + +Input.invalid { + background: #400000; +} + +Button.invalid { + background: #400000; +} + +ToggleButton { + background: #000060; + border: none; + padding: 1; + color: #606060; +} + +ToggleButton:disabled { + color: #404040; +} + +App.no-unicode ToggleButton { + color: #ffffff; +} + +ToggleButton:focus { + background: #0040a0; +} + +ToggleButton .toggle--label { + color: #ffffff; +} + +ToggleButton .toggle--button { + color: #00ff00; + background: #606060; +} + +App.no-unicode ToggleButton .toggle--button { + background: #000060; +} + +ToggleButton.-on .toggle--button { + color: #000060; + background: #000060; +} + +App.no-unicode ToggleButton.-on .toggle--button { + color: #00ff00; +} + + +ToggleButton:focus .toggle--button { + background: #0040a0; +} + +ToggleButton.-on:focus .toggle--button { + background: #0040a0; + color: #0040a0; +} + +ToggleButton:focus .toggle--label { + text-style: none; + background: #0040a0; +} + +LoadingIndicator { + margin-left: 1; + width: 8; + height: 1; + color: #0000ff; +} + +*:disabled .form-label { + color: #808080; +} + +.dialog { + width: 90%; + height: 90%; + border: round #000000; + background: #d0d0d0; +} + +.dialog .header { + color: #000000; +} + +.dialog .main-text { + color: #404040; + display: block; +} + +.dialog Button { + border-top: tall #80a0a0; + background: #408080; + border-bottom: tall #204040; +} + +.dialog Button:focus { + border-top: tall #a0ffff; + background: #60a0a0; + border-top: tall #408080; +} + +.dialog Button.-error { + background: #400000; +} + +.dialog Button.-error:focus { + background: #a00000; +} + +.dialog .action-area Button { + margin-right: 1; +} + +.dialog .action-area Button:last-of-type { + margin-right: 2; +} + +.dialog Label { + color: #606060; +} + +.dialog .attrs .th { + width: 50%; +} + +.dialog .attrs .td { + width: 50%; +} + +.dialog #btn-add-attr { + margin-top: 1; +} + +.dialog .action-area { + padding-top: 1; + padding-right: 1; + padding-left: 1; + align: right middle; + dock: bottom; + height: 4; +} + +#sidebar { + dock: left; + color: #e0e0e0; + background: #002080; + height: 100%; + width: 10%; + min-width: 18; + padding-left: 2; + padding-top: 2; +} + +App.small #sidebar { + min-width: 16; + padding-left: 1; + padding-top: 1; +} + +#sidebar Label { + margin-right: 2; +} + +#sidebar Label:disabled { + color: #808080; +} + +App.small #sidebar Label { + margin-right: 1; +} + +#body { + background: #0f2080; + height: 100%; + border-left: round #008080; +} + +.v-scrollable { + overflow-y: scroll; +} + +#main { + height: 100%; + width: 100%; +} + +#bottom { + width: 100%; + height: 4; + background: #0f2080; + border-left: round #008080; + padding-left: 0; + dock: bottom; +} + +#buttons { + align: center top; + width: 70%; + height: 4; + background: #0f2080; +} + +#left-padding, #right-padding { + width: 15%; + background: #0f2080; + height: 4; +} + +Button { + padding-left: 2; + padding-right: 2; +} + +#btn-prev { + width: 40%; + margin-left: 2; + margin-right: 2; + margin-bottom: 1; +} + +#btn-next { + width: 40%; + margin-left: 2; + margin-right: 2; + margin-bottom: 1; +} + +.panel { + width: 100%; + padding: 2; + padding-top: 2; + min-height: 100%; + height: auto; +} + +.panel-wrapper { + width: 100%; + height: auto; + min-height: 100%; + align-horizontal: center; +} + +App.small .panel { + padding: 1; + padding-top: 1; +} + +App.large .panel { + width: 80%; + min-width: 70; +} + +.p { + padding: 2; +} + +.p-l { + padding-left: 2; +} + +.p-r { + padding-right: 2; +} + +.m-b { + margin-bottom: 1; +} + +.m-t { + margin-top: 2; +} + +.m-b-1 { + margin-bottom: 1; +} + +.m-t-1 { + margin-top: 1; +} + +.m-l { + margin-left: 2; +} + +.m-r { + margin-right: 2; +} + +.m-r-1 { + margin-right: 1; +} + +Button { + border-top: tall #a0a0a0; + background: #808080; + border-bottom: tall #606060; +} + +Button:focus { + border-top: tall #e0e0e0; + background: #a0a0a0; + border-bottom: tall #808080; +} + +Button.-primary { + border-top: tall #0060a0; + background: #004080; + border-bottom: tall #004040; +} + +Button.-primary:focus { + border-top: tall #80a0ff; + background: #0080ff; + border-bottom: tall #004080; +} + +.header { + color: white; + margin-bottom: 2; +} + +App.small .header { + margin-bottom: 1; +} + +.main-text, .main-text-nh { + color: #b0b0b0; +} + +.help-text { + margin-top: 2; + margin-bottom: 1; + margin-left: 1; + max-width: 80; + color: #b0b0b0; +} + +App.small .help-text { + margin-top: 0; + margin-bottom: 1; + margin-left: 1; +} + +.warning-text { + color: yellow; + padding-top: 1; + margin-left: 2; +} + +.h-auto { + height: auto; +} + +.w-auto { + width: auto; +} + +.w-100 { + width: 100%; +} + +.c { + background: #ff2020; +} + +.d { + background: #20ffff; +} + + +#name-input-group { + width: 100%; + height: auto; +} + +#key-email-input-group { + width: 100%; + height: 1fr; +} + +#key-input { + padding: 0; + margin: 0; + padding-left: 1; +} + +#btn-use-fqdn, #btn-request-key { + width: auto; + dock: right; +} + +.form-label { + margin-bottom: 1; + color: #ffffff; +} + +App.small .form-label { + margin-bottom: 0; +} + +#warn-no-batch { + height: auto; +} + +#batch-warn { + padding-top: 2; +} + +.form-col-3 { + width: 33%; + margin-right: 1; + margin-bottom: 1; +} + +.form-col-2 { + width: 50%; + margin-right: 2; + margin-bottom: 1; +} + +.form-col-2 Button { + width: 27; +} + +#cb-run-test-job { + min-width: 16; + width: 27; + max-width: 32; + margin-top: 1; + margin-right: 2; +} + +#custom-attrs { + background: #002040; + height: 7; + overflow-x: auto; + overflow-y: auto; + padding: 0; + border: none; + color: #c0c0c0; +} + +.form-row { + min-height: 6; + max-height: 6; + padding-bottom: 1; +} + +App.large .form-row { + margin-bottom: 1; +} + +App.small .form-row { + min-height: 5; +} + +#edit-attrs-dialog { + padding-top: 0; + width: 96%; + height: 94%; +} + +#attr-rows { + overflow-y: scroll; + height: 80%; +} + +#attr-rows .td, #attr-rows .th { + width: 30%; +} + +#attr-rows .td:last-of-type, #attr-rows .th:last-of-type { + width: 40%; +} + +#edit-attrs-dialog .attr-row { + margin-bottom: 1; +} + +.dialog { + padding-left: 1; + padding-right: 1; + padding-top: 1; +} + +.error-dialog { + border: #ff0000; + height: 14; + width: 60; +} + +.error-dialog .header { + color: #800000; +} + +#progress-dialog { + width: 50; + height: 6; +} + +#progress-dialog .header { + margin-left: 1; +} + +#progress-indicator-indeterminate { + color: #008000; + margin-top: 1; +} + +#progress-dialog #progress-indicator-indeterminate { + width: 32; + margin-left: 8; +} + +#progress-indicator-indeterminate .bar--indeterminate { + color: #ff0000; +} + +Label.-success { + color: #00ff00; +} + +Label.-error { + color: #ff0000; +} + +#key-request-dialog { + width: 70; + max-width: 80; + height: 20; +} + +#key-request-url { + margin-top: 1; + margin-left: 2; + margin-bottom: 1; + background: #000000; + color: #808080; + padding-top: 1; + padding-bottom: 1; + padding-left: 2; + padding-right: 2; +} + +#key-request-dialog #status-label { + margin-top: 1; + margin-left: 2; +} + +.form-col { + width: auto; + margin-right: 1; + padding-bottom: 1; +} + +.test-job-indicator { + width: 4; + margin-left: 1; + margin-right: 1; +} + +.job-progress-row { + height: 1; +} + +#test-job-errors { + margin-top: 1; + margin-bottom: 1; +} + +.test-job-label { + color: #ffffff; + margin-right: 2; +} + +.test-job-status { + width: 6; +} + +#test-jobs-dialog .status-succeeded { + text-style: bold; + color: #00a040; + padding-left: 2; +} + +#test-jobs-dialog .status-failed { + color: #ff0000; + margin-left: 0; + margin-right: 0; +} + +#test-jobs-dialog Button { + margin-left: 1; +} + +#method-status { + margin-top: 1; + color: #b04040; +} + +#method-preview { + border: none; + padding: 0; + background: #0f2040; + padding-left: 1; + padding-right: 1; + height: 3; +} + +#method-preview:disabled { + background-tint: 0%; + opacity: 100%; + text-opacity: 100%; +} \ No newline at end of file diff --git a/tests/installer/terminal.py b/tests/installer/terminal.py new file mode 100644 index 00000000..a9d79bda --- /dev/null +++ b/tests/installer/terminal.py @@ -0,0 +1,329 @@ +import string +import sys + +from enum import Enum +from typing import Callable + +from rich.color import Color +from rich.style import StyleType + + +def add_alt_key_processing() -> None: + from textual._ansi_sequences import ANSI_SEQUENCES_KEYS + + class MoreKeys(str, Enum): + AltA = 'alt+a' + AltB = 'alt+b' + AltC = 'alt+c' + AltD = 'alt+d' + AltE = 'alt+e' + AltF = 'alt+f' + AltG = 'alt+g' + AltH = 'alt+h' + AltI = 'alt+i' + AltJ = 'alt+j' + AltK = 'alt+k' + AltL = 'alt+l' + AltM = 'alt+m' + AltN = 'alt+n' + AltO = 'alt+o' + AltP = 'alt+p' + AltQ = 'alt+q' + AltR = 'alt+r' + AltS = 'alt+s' + AltT = 'alt+t' + AltU = 'alt+u' + AltV = 'alt+v' + AltW = 'alt+w' + AltX = 'alt+x' + AltY = 'alt+y' + AltZ = 'alt+z' + + AltShiftA = 'alt+shift+a' + AltShiftB = 'alt+shift+b' + AltShiftC = 'alt+shift+c' + AltShiftD = 'alt+shift+d' + AltShiftE = 'alt+shift+e' + AltShiftF = 'alt+shift+f' + AltShiftG = 'alt+shift+g' + AltShiftH = 'alt+shift+h' + AltShiftI = 'alt+shift+i' + AltShiftJ = 'alt+shift+j' + AltShiftK = 'alt+shift+k' + AltShiftL = 'alt+shift+l' + AltShiftM = 'alt+shift+m' + AltShiftN = 'alt+shift+n' + AltShiftO = 'alt+shift+o' + AltShiftP = 'alt+shift+p' + AltShiftQ = 'alt+shift+q' + AltShiftR = 'alt+shift+r' + AltShiftS = 'alt+shift+s' + AltShiftT = 'alt+shift+t' + AltShiftU = 'alt+shift+u' + AltShiftV = 'alt+shift+v' + AltShiftW = 'alt+shift+w' + AltShiftX = 'alt+shift+x' + AltShiftY = 'alt+shift+y' + AltShiftZ = 'alt+shift+z' + + for c in string.ascii_lowercase: + C = c.upper() + ANSI_SEQUENCES_KEYS[f'\x1b{c}'] = (MoreKeys[f'Alt{C}'],) # type: ignore + ANSI_SEQUENCES_KEYS[f'\x1b{C}'] = (MoreKeys[f'AltShift{C}'],) # type: ignore + + +def patch_borders() -> None: + EXTRA_CELLS = frozenset(['\033(0' + c + '\033(1' for c in 'lqkxmja']) + from rich import cells + prev_cell_len = cells.cell_len + + def cell_len(text: str, _cell_len: Callable[[str], int] = cells.cached_cell_len) -> int: + if text in EXTRA_CELLS: + return 1 + elif '\033' in text: + return len(text.replace('\033(0', '').replace('\033(1', '')) + else: + return prev_cell_len(text, _cell_len) + + cells.cell_len = cell_len + cells.cached_cell_len = cell_len # type: ignore + + BORDER = ( + ("\033(0l\033(1", "\033(0q\033(1", "\033(0k\033(1"), + ("\033(0x\033(1", " ", "\033(0x\033(1"), # noqa: E241 + ("\033(0m\033(1", "\033(0q\033(1", "\033(0j\033(1") + ) + + from textual import _border + + for t in ['round', 'solid', 'double', 'dashed', 'heavy', 'inner', 'outer', 'thick', 'tall', + 'panel', 'tab', 'wide']: + _border.BORDER_CHARS[t] = BORDER # type: ignore + + _border.BORDER_CHARS['tall'] = ((" ", " ", " "), (" ", " ", " "), (" ", " ", " ")) + _border.BORDER_CHARS['outer'] = ((" ", " ", " "), (" ", " ", " "), (" ", ".", ".")) + + _border.BORDER_CHARS['vkey'] = ( + ("\033(0l\033(1", " ", "\033(0k\033(1"), + ("\033(0x\033(1", " ", "\033(0x\033(1"), + ("\033(0m\033(1", " ", "\033(0j\033(1") + ) + + _border.BORDER_CHARS['hkey'] = ( + ("\033(0l\033(1", "\033(0q\033(1", "\033(0k\033(1"), + (" ", " ", " "), + ("\033(0m\033(1", "\033(0q\033(1", "\033(0j\033(1") + ) + + +def patch_toggle_button(unicode: bool) -> None: + from textual.widgets._toggle_button import ToggleButton + from textual.widgets._radio_button import RadioButton + from rich.text import Text + from rich.style import Style + + class PatchedToggleButton(ToggleButton): + @property + def _button(self) -> Text: + button_style = self.get_component_rich_style('toggle--button') + side_style = Style.from_color(self.colors[3].rich_color, + self.background_colors[1].rich_color) + return Text.assemble( + (self.BUTTON_LEFT, side_style), + (self.BUTTON_INNER, button_style), + (self.BUTTON_RIGHT, side_style) + ) + ToggleButton._button = PatchedToggleButton._button # type: ignore + RadioButton._button = PatchedToggleButton._button # type: ignore + + if not unicode: + ToggleButton.BUTTON_LEFT = '[' + ToggleButton.BUTTON_RIGHT = ']' + ToggleButton.BUTTON_INNER = 'x' + RadioButton.BUTTON_LEFT = '(' + RadioButton.BUTTON_INNER = '\033(0`\033(1' + RadioButton.BUTTON_RIGHT = ')' + + from textual.widgets import _rule + + _rule._HORIZONTAL_LINE_CHARS['solid'] = '\033(0q\033(1' + + +def patch_scrollbar(unicode: bool) -> None: + from textual.scrollbar import ScrollBarRender + from rich.segment import Segment, Segments + from rich.style import Style + + if unicode: + VERTICAL_BAR = ' ' + HORIZONTAL_BAR = ' ' + else: + VERTICAL_BAR = ' ' + HORIZONTAL_BAR = ' ' + + class SBRenderer(ScrollBarRender): + + def __init__(self, virtual_size: int = 100, window_size: int = 0, position: float = 0, + thickness: int = 1, vertical: bool = True, + style: StyleType = "bright_magenta on #555555") -> None: + super().__init__(virtual_size, window_size, position, thickness, vertical, style) + + @classmethod + def render_bar(cls, size: int = 25, virtual_size: float = 50, window_size: float = 20, + position: float = 0, thickness: int = 1, vertical: bool = True, + back_color: Color = Color.parse("#555555"), + bar_color: Color = Color.parse("bright_magenta")) -> Segments: + if vertical: + bar = VERTICAL_BAR + else: + bar = HORIZONTAL_BAR + if window_size and size and virtual_size and size != virtual_size: + thumb_size = int(window_size / virtual_size * size) + if thumb_size < 1: + thumb_size = 1 + position = (size - thumb_size) * position / (virtual_size - window_size) + top_size = int(position) + bottom_size = size - top_size - thumb_size + + upper_segment = Segment(bar, Style(bgcolor=back_color, color=bar_color, + meta={'@mouse.up': 'scroll_up'})) + lower_segment = Segment(bar, Style(bgcolor=back_color, color=bar_color, + meta={'@mouse.up': 'scroll_down'})) + thumb_segment = Segment(' ', Style(color=bar_color, reverse=True)) + segments = ([upper_segment] * top_size + [thumb_segment] * thumb_size + + [lower_segment] * bottom_size) + else: + segments = [Segment(bar, Style(bgcolor=back_color))] * size + + if vertical: + return Segments(segments, new_lines=True) + else: + return Segments(segments, new_lines=False) + + ScrollBarRender.render_bar = SBRenderer.render_bar # type: ignore + + +def patch_markdown() -> None: + from textual.widgets import Markdown + + Markdown.BULLETS = ['\033(0`\033(1', '*', '-', 'o'] + + +def patch_select() -> None: + from textual.widgets import _select + from textual.widgets import Static + + def compose(self): # type: ignore + yield Static(self.placeholder, id='label') + yield Static("_", classes='arrow down-arrow') + yield Static("^", classes='arrow up-arrow') + + _select.SelectCurrent.compose = compose # type: ignore + + +def patch_progress_bar() -> None: + from textual.app import ComposeResult, RenderResult + from textual.widgets import ProgressBar + from textual.widgets._progress_bar import Bar + + class BR: + def __init__(self, br: RenderResult) -> None: + self.br = br + + def __rich_console__(self, console, options): # type: ignore + r = next(self.br.__rich_console__(console, options)) # type: ignore + for seg in r.render(console, options): + yield seg._replace(text=''.join(['\033(0q\033(1'] * len(str(seg[0]))), style=seg[1]) + + class PatchedBar(Bar): + def render(self) -> RenderResult: + br = super().render() + return BR(br) + + def compose(self: Bar) -> ComposeResult: + yield PatchedBar(id='bar', clock=self._clock).data_bind( + ProgressBar.percentage).data_bind(ProgressBar.gradient) + + ProgressBar.compose = compose # type: ignore + + +_LASTC = '' + + +def _expect(c: str) -> None: + global _LASTC + if _LASTC != '': + r = _LASTC + _LASTC = '' + else: + r = sys.stdin.read(1) + if r != c: + raise Exception('Unexpected terminal response: %s' % r) + + +def _readnum() -> int: + global _LASTC + s = '' + while True: + c = sys.stdin.read(1) + if c.isdigit(): + s += c + else: + _LASTC = c + return int(s) + + +_TERMINAL_SUPPORTS_UNICODE = None + + +def terminal_supports_unicode() -> bool: + global _TERMINAL_SUPPORTS_UNICODE + if _TERMINAL_SUPPORTS_UNICODE is not None: + return _TERMINAL_SUPPORTS_UNICODE + try: + import termios + initial_mode = termios.tcgetattr(sys.stdin) + mode = termios.tcgetattr(sys.stdin) + mode[3] &= ~termios.ICANON & ~termios.ECHO + termios.tcsetattr(sys.stdin, termios.TCSAFLUSH, mode) + try: + # save cursor, move to col 0, print unicode char, report position, restore cursor + print('\0337\033[0G─\033[6n\0338', end='', flush=True) + _expect('\033') + _expect('[') + _ = _readnum() + _expect(';') + x = _readnum() + _expect('R') + _TERMINAL_SUPPORTS_UNICODE = x == 2 + print(' ') + return _TERMINAL_SUPPORTS_UNICODE + finally: + termios.tcsetattr(sys.stdin, termios.TCSAFLUSH, initial_mode) + except ImportError: + print('Cannot import termios') + raise + + +def patch_rich_terminals() -> None: + from rich import console + from rich.color import ColorSystem + console._TERM_COLORS['rxvt'] = ColorSystem.EIGHT_BIT + + +def run_patches() -> None: + unicode = terminal_supports_unicode() + if not unicode: + patch_borders() + patch_select() + patch_progress_bar() + patch_markdown() + + patch_scrollbar(unicode) + patch_toggle_button(unicode) + + add_alt_key_processing() + patch_rich_terminals() + + +run_patches() diff --git a/tests/installer/widgets.py b/tests/installer/widgets.py new file mode 100644 index 00000000..6bceb9e6 --- /dev/null +++ b/tests/installer/widgets.py @@ -0,0 +1,53 @@ +from textual.app import RenderResult +from typing import Optional, Any +from textual.binding import Binding +from textual.widgets import Select, Button, LoadingIndicator + + +class MSelect(Select[str]): + BINDINGS = [ + Binding('enter, space', 'show_overlay2', 'Show menu', show=False), + Binding('down', 'down', 'Bypass', show=False), + Binding('up', 'up', 'Bypass', show=False), + ] + + def action_show_overlay(self) -> None: + pass + + def action_up(self) -> None: + self.app.action_scroll_up() # type: ignore + + def action_down(self) -> None: + self.app.action_scroll_down() # type: ignore + + def action_show_overlay2(self) -> None: + super().action_show_overlay() + + +class ShortcutButton(Button): + def __init__(self, label: str, *args: Any, **kwargs: Any) -> None: + self.key: Optional[str] = None + label = self._handle_shortcut(label) + super().__init__(label, *args, **kwargs) + + def on_mount(self) -> None: + if self.key: + self.app.register_button_shortcut(self.key, self) # type: ignore + + def _handle_shortcut(self, label: str) -> str: + ix = label.index('&') + if ix == -1: + return label + self.key = label[ix + 1] + return (label[0:ix] + '[b bright_yellow]' + label[ix + 1] + + '[/b bright_yellow]' + label[ix + 2:]) + + def set_label(self, label: str) -> None: + self.label = self._handle_shortcut(label) + + +class DottedLoadingIndicator(LoadingIndicator): + def render(self) -> RenderResult: + text = super().render() + text.plain = '......' # type: ignore + return text diff --git a/tests/run_installer.py b/tests/run_installer.py new file mode 100644 index 00000000..4fc18a13 --- /dev/null +++ b/tests/run_installer.py @@ -0,0 +1,5 @@ +import conftest +import ci_runner +from installer import main + +main.run(conftest, ci_runner) From 571dbf85157c599e0834ce6f3fd072681e49ce63 Mon Sep 17 00:00:00 2001 From: hategan Date: Thu, 13 Feb 2025 22:15:20 -0800 Subject: [PATCH 02/33] An undocumented feature to push manual run strings to the clipboard for terminals that might support it. --- tests/installer/main.py | 1 - tests/installer/panels/schedule_panel.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/installer/main.py b/tests/installer/main.py index 5b938b78..7501022a 100644 --- a/tests/installer/main.py +++ b/tests/installer/main.py @@ -102,7 +102,6 @@ async def on_mount(self) -> None: self.watch(body, 'scroll_y', self._on_y_scroll) self.next_button.focus() self.activate_panel(0) - self.copy_to_clipboard('Testing, 1, 2, 3') async def _on_y_scroll(self, y: int) -> None: log.write(f'on_y_scroll({y})\n') diff --git a/tests/installer/panels/schedule_panel.py b/tests/installer/panels/schedule_panel.py index 0048c579..924eff66 100644 --- a/tests/installer/panels/schedule_panel.py +++ b/tests/installer/panels/schedule_panel.py @@ -100,6 +100,8 @@ async def validate(self) -> bool: assert m is not None self.state.install_method = m m.install() + if m.name == 'custom': + self.app.copy_to_clipboard(m.preview) return True except Exception as ex: await ErrorDialog('Error scheduling tests', str(ex)).run(self.app) From 7325764fb24c71ea28ed11e3fb869da344d39f91 Mon Sep 17 00:00:00 2001 From: hategan Date: Fri, 14 Feb 2025 10:41:54 -0800 Subject: [PATCH 03/33] Also track python version when caching packages. --- psij-ci-setup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/psij-ci-setup b/psij-ci-setup index 3355544c..b800eadc 100755 --- a/psij-ci-setup +++ b/psij-ci-setup @@ -289,7 +289,8 @@ not_cached() { return 0 fi REQ_CSUM=`cat .packages/req_csum.txt` - CRT_CSUM=`cat requirements*.txt | md5sum` + PYTHON=`which $PYTHON` + CRT_CSUM=`echo $PYTHON | cat - requirements*.txt | md5sum` if [ "$REQ_CSUM" != "$CRT_CSUM" ]; then return 0 From 193b1d93534a4f15d071028d20e8819e3dcd4f9e Mon Sep 17 00:00:00 2001 From: hategan Date: Fri, 14 Feb 2025 10:42:21 -0800 Subject: [PATCH 04/33] 3.8 compatibility --- tests/installer/dialogs.py | 7 ++++-- tests/installer/install_methods.py | 12 ++++----- tests/installer/panels/schedule_panel.py | 11 +++++---- tests/installer/state.py | 31 +++++++++++++++--------- 4 files changed, 36 insertions(+), 25 deletions(-) diff --git a/tests/installer/dialogs.py b/tests/installer/dialogs.py index d079bbe5..c5498555 100644 --- a/tests/installer/dialogs.py +++ b/tests/installer/dialogs.py @@ -1,5 +1,5 @@ import asyncio -from typing import cast +from typing import cast, TypeVar, Generic from textual import on from textual.app import App, ComposeResult @@ -11,7 +11,10 @@ from installer.widgets import DottedLoadingIndicator, ShortcutButton -class RunnableDialog[T](ModalScreen[None]): +T = TypeVar('T') + + +class RunnableDialog(ModalScreen[None], Generic[T]): def __init__(self) -> None: super().__init__() self.done = asyncio.get_running_loop().create_future() diff --git a/tests/installer/install_methods.py b/tests/installer/install_methods.py index 2da9adcb..b6cd6c61 100644 --- a/tests/installer/install_methods.py +++ b/tests/installer/install_methods.py @@ -34,7 +34,7 @@ def _save_env() -> None: class InstallMethod(ABC): @abstractmethod - def is_available(self) -> Tuple[bool, str | None]: + def is_available(self) -> Tuple[bool, Optional[str]]: pass @abstractmethod @@ -73,7 +73,7 @@ def __init__(self) -> None: minute = random.randint(0, 59) self.line = f'{minute} {hour} * * * "{cwd}/psij-ci-run" --log' - def is_available(self) -> Tuple[bool, str | None]: + def is_available(self) -> Tuple[bool, Optional[str]]: if not _succeeds("ps -eo command | awk '{print $1}' | grep cron"): return False, 'not found' if _succeeds('crontab -l 2>&1 | grep "not allowed"'): @@ -126,7 +126,7 @@ def __init__(self) -> None: self.minute = random.randint(0, 59) self.cmd = f'psij-ci-run --reschedule {self.hour}:{self.minute} --log' - def is_available(self) -> Tuple[bool, str | None]: + def is_available(self) -> Tuple[bool, Optional[str]]: fd, path = mkstemp() os.close(fd) @@ -187,7 +187,7 @@ class Screen(InstallMethod): def __init__(self) -> None: self.cmd = 'screen -d -m bash -c "./psij-ci-run --repeat --log"' - def is_available(self) -> Tuple[bool, str | None]: + def is_available(self) -> Tuple[bool, Optional[str]]: if _succeeds('which screen'): return True, None else: @@ -219,7 +219,7 @@ def help_message(self) -> str: class Custom(InstallMethod): - def is_available(self) -> Tuple[bool, str | None]: + def is_available(self) -> Tuple[bool, Optional[str]]: return True, None def already_installed(self) -> bool: @@ -255,7 +255,7 @@ def help_message(self) -> str: ] -def existing() -> InstallMethod | None: +def existing() -> Optional[InstallMethod]: for method in METHODS: if method.already_installed(): return method diff --git a/tests/installer/panels/schedule_panel.py b/tests/installer/panels/schedule_panel.py index 924eff66..61f26800 100644 --- a/tests/installer/panels/schedule_panel.py +++ b/tests/installer/panels/schedule_panel.py @@ -1,4 +1,4 @@ -from typing import cast +from typing import cast, Optional from .panel import Panel @@ -16,7 +16,8 @@ class MRadioSet(RadioSet): ('enter,space', 'select') ] - def __init__(self, panel: 'SchedulePanel', *radios: RadioButton, id: str | None = None) -> None: + def __init__(self, panel: 'SchedulePanel', *radios: RadioButton, + id: Optional[str] = None) -> None: super().__init__(*radios, id=id) self.panel = panel @@ -39,7 +40,7 @@ def action_select(self) -> None: self.action_toggle_button() - def get_selected_index(self) -> int | None: + def get_selected_index(self) -> Optional[int]: return self._selected @@ -86,7 +87,7 @@ def label(self) -> str: def name(self) -> str: return 'install' - def _get_selected_method(self) -> InstallMethod | None: + def _get_selected_method(self) -> Optional[InstallMethod]: radio_set = self.get_widget_by_id('rs-method') assert isinstance(radio_set, MRadioSet) selected_index = radio_set.get_selected_index() @@ -115,7 +116,7 @@ async def activate(self) -> None: btn.value = True break - def radio_focused(self, btn: RadioButton | None) -> None: + def radio_focused(self, btn: Optional[RadioButton]) -> None: log.write(f'focused {btn}\n') preview = self.get_widget_by_id('method-preview') assert isinstance(preview, TextArea) diff --git a/tests/installer/state.py b/tests/installer/state.py index fdcc787f..f4cf2ef5 100644 --- a/tests/installer/state.py +++ b/tests/installer/state.py @@ -1,4 +1,5 @@ import asyncio +import functools import json import os import pathlib @@ -6,7 +7,7 @@ import requests from collections import namedtuple -from typing import List, Dict, Optional, Callable, Awaitable, Tuple, cast +from typing import List, Dict, Optional, Callable, Awaitable, Tuple, cast, Union from .install_methods import InstallMethod from .log import log @@ -18,13 +19,19 @@ Attr = namedtuple('Attr', ['filter', 'name', 'value']) +async def _to_thread(func, /, *args, **kwargs): # type: ignore + loop = asyncio.get_running_loop() + func_call = functools.partial(func, *args, **kwargs) + return await loop.run_in_executor(None, func_call) + + class _Options: def __init__(self) -> None: self.custom_attributes = None class ConfWrapper: - def __init__(self, dict: Dict[str, str | int | bool | None]): + def __init__(self, dict: Dict[str, Union[str, int, bool, None]]): self.dict = dict self.option = _Options() dict['run_id'] = 'x' @@ -32,7 +39,7 @@ def __init__(self, dict: Dict[str, str | int | bool | None]): dict['upload_results'] = True dict['branch_name_override'] = None - def getoption(self, name: str) -> str | int | bool | None: + def getoption(self, name: str) -> Union[str, int, bool, None]: return self.dict[name] @@ -43,16 +50,16 @@ def __init__(self, conftest: types.ModuleType, ci_runner: types.ModuleType): log.write('Conf: ' + str(self.conf) + '\n') log.write(str(self.env) + '\n') self.disable_install = False - self.install_method: InstallMethod | None = None - self.active_panel: int | None = None - self.scheduler: str | None = None + self.install_method: Optional[InstallMethod] = None + self.active_panel: Optional[int] = None + self.scheduler: Optional[str] = None self.attrs = self._parse_attributes() self.run_test_job = True self.has_key = KEY_PATH.exists() if self.has_key: with open(KEY_PATH, 'r') as f: self.key = f.read().strip() - self.key_is_valid: bool | None = None + self.key_is_valid: Optional[bool] = None log.write(f'has key: {self.has_key}\n') def _parse_attributes(self) -> List[Attr]: @@ -89,15 +96,15 @@ def update_conf(self, name: str, value: str) -> None: self._write_conf_value(name, value) async def request(self, query: str, data: Dict[str, object], title: str, - error_cb: Callable[[str, str], Awaitable[Dict[str, object] | None]]) \ - -> Dict[str, object] | None: + error_cb: Callable[[str, str], Awaitable[Optional[Dict[str, object]]]]) \ + -> Optional[Dict[str, object]]: baseUrl = self.conf['server_url'] - response = await asyncio.to_thread(requests.post, baseUrl + query, data=data) + response = await _to_thread(requests.post, baseUrl + query, data=data) # type: ignore return await self._check_error(response, title, error_cb) async def _check_error(self, response: requests.Response, title: str, - error_cb: Callable[[str, str], Awaitable[Dict[str, object] | None]]) \ - -> Dict[str, object] | None: + error_cb: Callable[[str, str], Awaitable[Optional[Dict[str, object]]]]) \ + -> Optional[Dict[str, object]]: log.write(f'Response: {response.text}\n') if response.status_code != 200: msg = self._extract_response_message(response.text) From d2dc9e604fbb4e793fbc267f2014d7fbd8860b8b Mon Sep 17 00:00:00 2001 From: hategan Date: Fri, 14 Feb 2025 14:25:14 -0800 Subject: [PATCH 05/33] Proper checksum generation --- psij-ci-setup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/psij-ci-setup b/psij-ci-setup index b800eadc..bcec6a38 100755 --- a/psij-ci-setup +++ b/psij-ci-setup @@ -318,7 +318,8 @@ install_deps() { exit 2 else echo "Done" - cat requirements*.txt | md5sum >.packages/req_csum.txt + CSUM=`echo $PYTHON | cat - requirements*.txt | md5sum` + echo "$CSUM" >.packages/req_csum.txt fi fi From cc59314fb7a7e15cbd8c1357b33a7efc0275715f Mon Sep 17 00:00:00 2001 From: hategan Date: Thu, 13 Feb 2025 22:10:03 -0800 Subject: [PATCH 06/33] Added a TUI installer for the CI tests. --- .flake8 | 20 +- .github/dependabot.yml | 1 + psij-ci-run | 33 +- psij-ci-setup | 100 ++- requirements-tests.txt | 3 + tests/installer/__init__.py | 0 tests/installer/dialogs.py | 305 ++++++++ tests/installer/install_methods.py | 264 +++++++ tests/installer/log.py | 1 + tests/installer/main.py | 258 ++++++ tests/installer/panels/basic_info_panel.py | 100 +++ .../installer/panels/batch_scheduler_panel.py | 418 ++++++++++ tests/installer/panels/complete_panel.py | 30 + tests/installer/panels/intro_panel.py | 44 ++ tests/installer/panels/key_panel.py | 228 ++++++ tests/installer/panels/panel.py | 39 + tests/installer/panels/schedule_panel.py | 127 +++ tests/installer/py.typed | 0 tests/installer/state.py | 172 ++++ tests/installer/style.tcss | 738 ++++++++++++++++++ tests/installer/terminal.py | 329 ++++++++ tests/installer/widgets.py | 53 ++ tests/run_installer.py | 5 + 23 files changed, 3240 insertions(+), 28 deletions(-) create mode 100644 tests/installer/__init__.py create mode 100644 tests/installer/dialogs.py create mode 100644 tests/installer/install_methods.py create mode 100644 tests/installer/log.py create mode 100644 tests/installer/main.py create mode 100644 tests/installer/panels/basic_info_panel.py create mode 100644 tests/installer/panels/batch_scheduler_panel.py create mode 100644 tests/installer/panels/complete_panel.py create mode 100644 tests/installer/panels/intro_panel.py create mode 100644 tests/installer/panels/key_panel.py create mode 100644 tests/installer/panels/panel.py create mode 100644 tests/installer/panels/schedule_panel.py create mode 100644 tests/installer/py.typed create mode 100644 tests/installer/state.py create mode 100644 tests/installer/style.tcss create mode 100644 tests/installer/terminal.py create mode 100644 tests/installer/widgets.py create mode 100644 tests/run_installer.py diff --git a/.flake8 b/.flake8 index fe2f46ac..c24af0b3 100644 --- a/.flake8 +++ b/.flake8 @@ -55,7 +55,7 @@ max-line-length = 100 # One can omit the description on the __init__ docstring and simply # document the parameters, but then flake8 complains about the # structure of the __init__ doctsring. The D205 and D400 errors are -# precisely those complains. We disable these in order to have a +# precisely those complaints. We disable these in order to have a # sane way of documenting classes with Sphinx' autoclass. @@ -63,6 +63,20 @@ ignore = B902, D205, D400, D401, D100, W503 # D103 - Missing docstring in public function # -# Ignore docstrings requirement in tests +# Ignore docstrings requirement in tests. +# +# D101, D102, D103, D107 +# +# We don't quite document the installer. +# +# E402 - module level import not at top of file +# +# In the installer, we need to patch Textual for non-unicode terminal +# support, and that needs to be the first thing done before we import +# other stuff. +# -per-file-ignores = tests/*:D103 +per-file-ignores = + tests/*: D103 + tests/installer/*: D101, D102, D103, D104, D107 + tests/installer/main.py: E402, D101, D102, D103, D107 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index ed04926b..cf382e3e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -16,4 +16,5 @@ updates: - dependency-name: "pystache" - dependency-name: "typeguard" - dependency-name: "packaging" + - dependency-name: "requests" diff --git a/psij-ci-run b/psij-ci-run index 3783bfba..f9b9b4af 100755 --- a/psij-ci-run +++ b/psij-ci-run @@ -2,9 +2,17 @@ usage() { cat </dev/null 2>&1 ; pwd -P )" +cd "$MYPATH" + if [ -f psij-ci-env ]; then source psij-ci-env fi REPEAT=0 RESCHEDULE=0 +NO_OUT=0 while [ "$1" != "" ]; do case "$1" in @@ -38,13 +50,26 @@ while [ "$1" != "" ]; do TIME="$2" shift 2 ;; + --log) + NO_OUT=1 + shift + ;; *) echo "Unrecognized command line option: $1." usage exit 1 + ;; esac done +if [ "$NO_OUT" == "1" ]; then + exec 1<&- + exec 2<&- + + exec 1>>./testing.log + exec 2>&1 +fi + if python --version 2>&1 | egrep -q 'Python 3\..*' >/dev/null 2>&1 ; then PYTHON="python" else @@ -68,7 +93,7 @@ if [ "$REPEAT" == "1" ]; then sleep $TO_SLEEP done elif [ "$RESCHEDULE" == "1" ]; then - CMD="./psij-ci-run --reschedule $TIME >> testing.log 2>&1" + CMD="./psij-ci-run --reschedule $TIME --log" echo "$CMD" | at $TIME $PYTHON tests/ci_runner.py $MODE else diff --git a/psij-ci-setup b/psij-ci-setup index c080c70a..3355544c 100755 --- a/psij-ci-setup +++ b/psij-ci-setup @@ -1,11 +1,20 @@ #!/bin/bash +set -o pipefail + if pip --version 2>&1 | egrep -q 'python 3\..*' >/dev/null 2>&1 ; then PIP="pip" else PIP="pip3" fi +if python --version 2>&1 | egrep -q 'Python 3\..*' >/dev/null 2>&1 ; then + PYTHON="python" +else + PYTHON="python3" +fi + + existing_error_trailer() { echo "If you are certain that you want to install multiple entries, " echo "you can re-run this script with the \"-f\" flag. " @@ -50,7 +59,7 @@ cron_existing_error() { } cron_install() { - CMD="$CMD >> testing.log 2>&1" + CMD="$CMD --log" LINE="$MINUTE $HOUR * * * cd \"$MYPATH\" && $CMD" echo echo "================================================================" @@ -109,7 +118,7 @@ at_existing_error() { } at_install() { - CMD="$CMD --reschedule $HOUR:$MINUTE >> testing.log 2>&1" + CMD="$CMD --reschedule $HOUR:$MINUTE --log" echo echo "================================================================" echo "The following will be executed:" @@ -141,7 +150,7 @@ screen_existing_error() { } screen_install() { - CMD="$CMD --repeat >> testing.log 2>&1" + CMD="$CMD --repeat --log" echo echo "================================================================" echo "WARNING: Screen sessions do not persist across reboots. Please " @@ -166,7 +175,7 @@ manual_existing_error() { } manual_install() { - CMD="$CMD --repeat >> testing.log 2>&1" + CMD="$CMD --repeat --log" echo echo "================================================================" echo "Please run the following command in the background: " @@ -271,14 +280,76 @@ check_key() { fi } + +not_cached() { + if [ ! -d .packages ]; then + return 0 + fi + if [ ! -f .packages/req_csum.txt ]; then + return 0 + fi + REQ_CSUM=`cat .packages/req_csum.txt` + CRT_CSUM=`cat requirements*.txt | md5sum` + + if [ "$REQ_CSUM" != "$CRT_CSUM" ]; then + return 0 + else + return 1 + fi +} + +dots_for_lines() { + while read LINE; do + echo -n . + done +} + +install_deps() { + if not_cached; then + echo -n "Installing dependencies..." + + exec 3> >(dots_for_lines >> /dev/stdout) + OUT=`$PIP install --target .packages --upgrade -r requirements-tests.txt --no-compile 2>&1 1>&3` + + if [ "$?" != "0" ]; then + echo "FAILED" + echo "$OUT" + exit 2 + else + echo "Done" + cat requirements*.txt | md5sum >.packages/req_csum.txt + fi + fi + + export PYTHONPATH=`pwd`/.packages:$PYTHONPATH +} + + FORCE=0 +PLAIN=0 -if [ "$1" == "-f" ]; then - FORCE=1 +while [ "$1" != "" ]; do + if [ "$1" == "-f" ]; then + FORCE=1 + elif [ "$1" == "--plain" ]; then + PLAIN=1 + else + echo "Unknown option \"$1\"" + exit 1 + fi shift -fi +done MYPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +cd "$MYPATH" + +if [ "$PLAIN" != "1" ]; then + install_deps + + echo "Running TUI installer. To run the normal installer, use \"psij-ci-setup --plain\"" + + exec $PYTHON tests/run_installer.py +fi echo echo "================================================================" @@ -297,21 +368,8 @@ check_email check_key -cd "$MYPATH" - -echo -n "Installing dependencies..." - -OUT=`$PIP install --target .packages --upgrade -r requirements-tests.txt 2>&1` - -if [ "$?" != "0" ]; then - echo "FAILED" - echo $OUT - exit 2 -else - echo "Done" -fi +install_deps -export PYTHONPATH=`pwd`/.packages:$PYTHONPATH HOUR=`echo $(($RANDOM % 24))` MINUTE=`echo $(($RANDOM % 60))` diff --git a/requirements-tests.txt b/requirements-tests.txt index cf9ed31a..ef9572d5 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,3 +8,6 @@ requests >= 2.25.1 pytest-cov pytest-timeout filelock >= 3.4, < 3.18 +textual +tree-sitter +tree-sitter-bash diff --git a/tests/installer/__init__.py b/tests/installer/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/installer/dialogs.py b/tests/installer/dialogs.py new file mode 100644 index 00000000..d079bbe5 --- /dev/null +++ b/tests/installer/dialogs.py @@ -0,0 +1,305 @@ +import asyncio +from typing import cast + +from textual import on +from textual.app import App, ComposeResult +from textual.containers import Horizontal, Vertical +from textual.screen import ModalScreen +from textual.widget import Widget +from textual.widgets import Button, Label, RichLog, ProgressBar + +from installer.widgets import DottedLoadingIndicator, ShortcutButton + + +class RunnableDialog[T](ModalScreen[None]): + def __init__(self) -> None: + super().__init__() + self.done = asyncio.get_running_loop().create_future() + + def set_result(self, result: T) -> None: + self.done.set_result(result) + + async def wait(self) -> bool: + return cast(bool, await self.done) + + async def run(self, app: App[object]) -> T: + app.push_screen(self) + try: + await self.done + return cast(T, self.done.result()) + finally: + app.pop_screen() + + +class HelpDialog(RunnableDialog[None]): + BINDINGS = [ + ('escape', 'close', 'Close'), + ('c', 'close', 'Close'), + ('alt+c', 'close', 'Close') + ] + + def __init__(self, title: str, widget: Widget) -> None: + super().__init__() + self.title = title + self.widget = widget + + def compose(self) -> ComposeResult: + assert self.title is not None + yield Vertical( + Label(self.title, classes='header'), + Vertical(self.widget, classes='v-scrollable'), + Horizontal( + ShortcutButton('&Close', variant='error', id='btn-close'), + classes='action-area' + ), + id='help-dialog', classes='dialog help-dialog' + ) + + def on_button_pressed(self, event: Button.Pressed) -> None: + self.action_close() + + def action_close(self) -> None: + self.set_result(None) + + +class ExitConfirmDialog(RunnableDialog[str]): + BINDINGS = [ + ('escape', 'close'), + ('c', 'close', 'Close'), + ('alt+c', 'close', 'Close') + ] + + def __init__(self) -> None: + super().__init__() + + def compose(self) -> ComposeResult: + self.btn_quit = ShortcutButton('&Quit', variant='error', id='btn-quit') + self.btn_cancel = ShortcutButton('&Cancel', variant='warning', id='btn-cancel') + yield Vertical( + Label('Confirm exit', classes='header'), + Label('Are you sure you want to exit the installer?', + classes='main-text', + shrink=True, expand=True), + Horizontal(self.btn_quit, self.btn_cancel, classes='action-area'), + id='replace-dialog', classes='dialog error-dialog' + ) + + def action_close(self) -> None: + self.set_result('cancel') + + def on_button_pressed(self, event: Button.Pressed) -> None: + if event.button == self.btn_quit: + self.set_result('quit') + else: + self.set_result('cancel') + + +class ProgressDialog(ModalScreen[None]): + def __init__(self, message: str) -> None: + super().__init__() + self.message = message + + def compose(self) -> ComposeResult: + yield Vertical( + Label(self.message, classes='header', shrink=True, expand=True), + ProgressBar(id='progress-indicator-indeterminate', + show_percentage=False, show_bar=True, show_eta=False), + id='progress-dialog', classes='dialog' + ) + + +class TestJobsDialog(RunnableDialog[bool]): + BINDINGS = [ + ('b', 'back', 'Back'), + ('alt+b', 'back', 'Back'), + ('c', 'continue', 'Continue'), + ('alt+c', 'continue', 'Continue'), + ] + + def __init__(self) -> None: + super().__init__() + self.result = None + + def compose(self) -> ComposeResult: + yield Vertical( + Label('Running test jobs', classes='header', shrink=True, expand=True), + Horizontal( + Label('Single node job', id='label-job-1', classes='test-job-label'), + Label('[ ', classes='test-job-marker'), + DottedLoadingIndicator(id='indicator-job-1', classes='test-job-indicator hidden'), + Label('', id='status-job-1', classes='test-job-status'), + Label(' ]', classes='test-job-marker'), + classes='job-progress-row' + ), + Horizontal( + Label('Multi node job ', id='label-job-2', classes='test-job-label'), + Label('[ ', classes='test-job-marker'), + DottedLoadingIndicator(id='indicator-job-2', classes='test-job-indicator hidden'), + Label('', id='status-job-2', classes='test-job-status'), + Label(' ]', classes='test-job-marker'), + classes='job-progress-row' + ), + RichLog(id='test-job-errors', wrap=True), + Horizontal( + ShortcutButton('Go &back', variant='error', id='btn-back'), + ShortcutButton('&Continue', id='btn-continue'), + classes='action-area' + ), + id='test-jobs-dialog', classes='dialog' + ) + + def on_button_pressed(self, event: Button.Pressed) -> None: + if event.button.id == 'btn-back': + self.action_back() + else: + self.action_continue() + + def action_back(self) -> None: + self.app.pop_screen() + self.set_result(False) + + def action_continue(self) -> None: + self.app.pop_screen() + self.set_result(True) + + def set_running(self, job_no: int) -> None: + indicator = self.get_widget_by_id(f'indicator-job-{job_no}') + indicator.remove_class('hidden') + + label = self.get_widget_by_id(f'status-job-{job_no}') + label.add_class('hidden') + + def set_status(self, job_no: int, status: str, cls: str) -> None: + indicator = self.get_widget_by_id(f'indicator-job-{job_no}') + indicator.add_class('hidden') + + label = self.get_widget_by_id(f'status-job-{job_no}') + assert isinstance(label, Label) + label.update(status) + label.remove_class('hidden') + label.add_class(cls) + + def log_error(self, label: str, ex: Exception) -> None: + log = self.get_widget_by_id('test-job-errors') + assert isinstance(log, RichLog) + + log.write(f'----------- {label} -----------') + log.write(str(ex)) + log.write('') + + def focus_back_button(self) -> None: + self.get_widget_by_id('btn-back').focus() + + def focus_continue_button(self) -> None: + self.get_widget_by_id('btn-continue').focus() + + +class ErrorDialog(RunnableDialog[bool]): + BINDINGS = [ + ('c', 'close', 'Close'), + ('alt+c', 'close', 'Close') + ] + + def __init__(self, title: str, message: str) -> None: + super().__init__() + self.title = title + self.message = message + + def compose(self) -> ComposeResult: + assert self.title is not None + yield Vertical( + Label(self.title, classes='header'), + Label(self.message, classes='main-text', shrink=True, expand=True), + Horizontal( + ShortcutButton('&Close', variant='error', id='btn-close'), + classes='action-area' + ), + id='error-dialog', classes='dialog error-dialog' + ) + + def on_button_pressed(self, event: Button.Pressed) -> None: + self.action_close() + + def action_close(self) -> None: + self.set_result(True) + + +class KeyRequestDialog(ModalScreen[None]): + BINDINGS = [ + ('c', 'cancel', 'Cancel'), + ('alt+c', 'cancel', 'Cancel') + ] + + def __init__(self, url: str) -> None: + super().__init__() + self.url = url + + def compose(self) -> ComposeResult: + yield Vertical( + Label('Key request', classes='header'), + Label('Please enter the following URL in a web browser:', classes='main-text'), + Label(self.url, classes='main-text', id='key-request-url'), + Label('Then follow the instructions the browser. The URL is valid for 10 minutes.', + classes='main-text', expand=True, shrink=True), + Horizontal( + ProgressBar(id='progress-indicator-indeterminate'), + Label('Waiting', classes='main-text', id='status-label'), + ), + Horizontal( + ShortcutButton('&Cancel', variant='error', id='btn-cancel'), + classes='action-area' + ), + id='key-request-dialog', classes='dialog' + ) + + def update_status(self, status_str: str) -> None: + status_label = self.get_widget_by_id('status-label') + assert isinstance(status_label, Label) + if status_str == 'initialized': + pass + elif status_str == 'seen': + status_label.update('Page opened') + elif status_str == 'email_sent': + status_label.update('Email sent') + elif status_str == 'verified': + status_label.update('Email verified') + + @on(Button.Pressed, selector='#btn-cancel') + def action_cancel(self) -> None: + self.app.pop_screen() + + +class ExistingInstallConfirmDialog(RunnableDialog[str]): + BINDINGS = [ + ('c', 'close', 'Close'), + ('alt+c', 'close', 'Close') + ] + + def __init__(self, method: str) -> None: + super().__init__() + self.method = method + + def compose(self) -> ComposeResult: + assert self.method is not None + yield Vertical( + Label('Existing installation detected', classes='header'), + Label(f'An existing {self.method} installation of the tests was detected. Continuing ' + 'will update the settings used by the existing installation.', + classes='main-text', + shrink=True, expand=True), + Horizontal( + ShortcutButton('&Quit', variant='error', id='btn-quit'), + ShortcutButton('&Update', variant='warning', id='btn-update'), + ShortcutButton('&Reinstall', variant='warning', id='btn-reinstall'), + classes='action-area' + ), + id='replace-dialog', classes='dialog error-dialog' + ) + + def on_button_pressed(self, event: Button.Pressed) -> None: + if event.button.id == 'btn-quit': + self.set_result('quit') + elif event.button.id == 'btn-update': + self.set_result('update') + else: + self.set_result('reinstall') diff --git a/tests/installer/install_methods.py b/tests/installer/install_methods.py new file mode 100644 index 00000000..2da9adcb --- /dev/null +++ b/tests/installer/install_methods.py @@ -0,0 +1,264 @@ +import os +import subprocess +import time +from abc import ABC, abstractmethod +import random +from tempfile import mkstemp +from typing import Optional, Tuple + +from installer.log import log + + +def _run(cmd: str, input: Optional[str] = None) -> Tuple[int, str]: + p = subprocess.run(['/bin/bash', '-c', cmd], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + input=input, text=True) + return p.returncode, p.stdout + + +def _must_succeed(cmd: str, input: Optional[str] = None) -> str: + ec, out = _run(cmd, input) + if ec != 0: + raise Exception(f'Command {cmd} failed: {out}') + return out + + +def _succeeds(cmd: str, input: Optional[str] = None) -> bool: + ec, _ = _run(cmd, input) + return ec == 0 + + +def _save_env() -> None: + _must_succeed('declare -x | egrep -v "^declare -x ' + '(BASH_VERSINFO|DISPLAY|EUID|GROUPS|SHELLOPTS|TERM|UID|_)=" >psij-ci-env') + + +class InstallMethod(ABC): + @abstractmethod + def is_available(self) -> Tuple[bool, str | None]: + pass + + @abstractmethod + def install(self) -> None: + pass + + @abstractmethod + def already_installed(self) -> bool: + pass + + @property + @abstractmethod + def preview(self) -> str: + pass + + @property + @abstractmethod + def name(self) -> str: + pass + + @property + @abstractmethod + def label(self) -> str: + pass + + @property + @abstractmethod + def help_message(self) -> str: + pass + + +class Crontab(InstallMethod): + def __init__(self) -> None: + cwd = os.getcwd() + hour = random.randint(0, 23) + minute = random.randint(0, 59) + self.line = f'{minute} {hour} * * * "{cwd}/psij-ci-run" --log' + + def is_available(self) -> Tuple[bool, str | None]: + if not _succeeds("ps -eo command | awk '{print $1}' | grep cron"): + return False, 'not found' + if _succeeds('crontab -l 2>&1 | grep "not allowed"'): + return False, 'not allowed' + return True, None + + def _crt_crontab(self) -> str: + ec, out = _run('crontab -l') + if ec != 0: + if 'no crontab for' in out: + return '' + else: + raise Exception(f'Error getting crontab: {out}') + else: + return out + + def already_installed(self) -> bool: + cwd = os.getcwd() + out = self._crt_crontab() + if f'cd "{cwd}" && ./psij-ci-run' in out or '"{cwd}/psij-ci-run"' in out: + return True + else: + return False + + def install(self) -> None: + crt = self._crt_crontab() + _save_env() + _must_succeed('crontab -', input=f'{crt}\n{self.line}\n') + + @property + def preview(self) -> str: + return self.line + + @property + def name(self) -> str: + return 'cron' + + @property + def label(self) -> str: + return 'Cron - the standard UNIX job scheduler' + + @property + def help_message(self) -> str: + return 'Uses the Cron scheduler to schedule daily runs of the tests.' + + +class At(InstallMethod): + def __init__(self) -> None: + self.hour = random.randint(0, 23) + self.minute = random.randint(0, 59) + self.cmd = f'psij-ci-run --reschedule {self.hour}:{self.minute} --log' + + def is_available(self) -> Tuple[bool, str | None]: + fd, path = mkstemp() + os.close(fd) + + try: + ec, out = _run('at now', input=f'rm {path}') + if ec != 0: + return False, 'not found' + time.sleep(0.2) + if out.startswith('job'): + job_no = out.split()[1] + _run(f'atrm {job_no}') + if 'No atd' in out: + return False, 'not running' + if os.path.exists(path): + os.remove(path) + return False, 'unknown error' + return True, None + finally: + try: + os.remove(path) + except FileNotFoundError: + pass + + def already_installed(self) -> bool: + out = _must_succeed('atq') + out = out.strip() + if len(out) == 0: + return False + for line in out.split('\n'): + job_no = line.split()[0] + job = _must_succeed(f'at -c {job_no}') + if 'psij-ci-run' in job: + return False + return True + + def install(self) -> None: + _must_succeed(f'at {self.hour}:{self.minute}', input=self.cmd) + + @property + def preview(self) -> str: + return f'echo "{self.cmd}" | at {self.hour}:{self.minute}' + + @property + def name(self) -> str: + return 'at' + + @property + def label(self) -> str: + return 'at - the standard UNIX "at" command' + + @property + def help_message(self) -> str: + return ('Uses the "at" job scheduler to schedule the tests which then re-schedule ' + 'themselves.') + + +class Screen(InstallMethod): + def __init__(self) -> None: + self.cmd = 'screen -d -m bash -c "./psij-ci-run --repeat --log"' + + def is_available(self) -> Tuple[bool, str | None]: + if _succeeds('which screen'): + return True, None + else: + return False, 'not found' + + def already_installed(self) -> bool: + ec, out = _run('screen -list | grep psij-ci-run') + return ec == 0 + + def install(self) -> None: + _must_succeed(self.cmd) + + @property + def preview(self) -> str: + return self.cmd + + @property + def name(self) -> str: + return 'screen' + + @property + def label(self) -> str: + return 'Screen - the Screen terminal multiplexer' + + @property + def help_message(self) -> str: + return ('Uses GNU Screen to run the tests in a Screen session. Does not persist across ' + 'reboots.') + + +class Custom(InstallMethod): + def is_available(self) -> Tuple[bool, str | None]: + return True, None + + def already_installed(self) -> bool: + return False + + def install(self) -> None: + _save_env() + + @property + def preview(self) -> str: + cwd = os.getcwd() + return f'"{cwd}/psij-ci-run" --log' + + @property + def name(self) -> str: + return 'custom' + + @property + def label(self) -> str: + return 'Custom - run with custom tool' + + @property + def help_message(self) -> str: + return ('Prints a command that can be used to run the tests, which can then be used with ' + 'a custom scheduler.') + + +METHODS = [ + Crontab(), + At(), + Screen(), + Custom() +] + + +def existing() -> InstallMethod | None: + for method in METHODS: + if method.already_installed(): + return method + else: + log.write(f'Not installed {method.name}\n') + return None diff --git a/tests/installer/log.py b/tests/installer/log.py new file mode 100644 index 00000000..bb5bc75c --- /dev/null +++ b/tests/installer/log.py @@ -0,0 +1 @@ +log = open('psij-ci-setup.log', 'w', buffering=1) diff --git a/tests/installer/main.py b/tests/installer/main.py new file mode 100644 index 00000000..5b938b78 --- /dev/null +++ b/tests/installer/main.py @@ -0,0 +1,258 @@ +from .terminal import run_patches, terminal_supports_unicode +run_patches() + +import asyncio +import types + +from .dialogs import ExitConfirmDialog +from .state import State +from .log import log +from .panels.basic_info_panel import BasicInfoPanel +from .panels.batch_scheduler_panel import BatchSchedulerPanel +from .panels.intro_panel import IntroPanel +from .panels.key_panel import KeyPanel +from .panels.schedule_panel import SchedulePanel +from .panels.complete_panel import CompletePanel +from .widgets import ShortcutButton + +from textual import on, events +from textual.app import App, ComposeResult +from textual.containers import Horizontal, Vertical +from textual.widgets import Footer, Header, Static, Label, Button +from typing import Optional, Dict, cast, List + + +class PSIJCIInstallWizard(App[object]): + CSS_PATH = 'style.tcss' + + ENABLE_COMMAND_PALETTE = False + + BINDINGS = [ + ('down', 'scroll_down'), + ('up', 'scroll_up'), + ('pageup', 'prev_page'), + ('pagedown', 'next_page'), + ('ctrl+q', 'confirm_quit', 'Quit'), + ('ctrl+c', 'quit', 'Quit'), + ('x', 'confirm_quit'), + ('alt+x', 'confirm_quit'), + ('ctrl+x', 'confirm_quit'), + ('escape', 'confirm_quit') + ] + + def __init__(self, conftest: types.ModuleType, ci_runner: types.ModuleType) -> None: + super().__init__() + self.state = State(conftest, ci_runner) + self.body = None + self.shortcuts: Dict[str, List[ShortcutButton]] = {} + self.key_panel = KeyPanel(self.state) + self.scheduler_panel = BatchSchedulerPanel(self.state) + self.install_panel = SchedulePanel(self.state) + + self.panels = [ + IntroPanel(self.state), + BasicInfoPanel(self.state), + self.key_panel, + self.scheduler_panel, + self.install_panel, + CompletePanel(self.state) + ] + + def disable_install(self) -> None: + self.state.disable_install = True + self.install_panel.disabled = True + self.get_widget_by_id('panel-label-install').disabled = True + + def compose(self) -> ComposeResult: + self.prev_button = ShortcutButton('&Previous', disabled=True, id='btn-prev') + self.next_button = ShortcutButton('&Next', variant='primary', id='btn-next') + + yield Header(icon='=') + with Horizontal(id='root'): + with Vertical(id='sidebar', classes='sidebar') as sidebar: + self.sidebar = sidebar + for panel in self.panels: + yield Label(' ' + panel.label, id=f'panel-label-{panel.name}', + classes='panel-label') + with Vertical(id='main'): + with Vertical(id='body', classes='v-scrollable'): + for panel in self.panels: + yield panel + with Horizontal(id='bottom'): + yield Static('', id='left-padding') + with Horizontal(id='buttons'): + yield self.prev_button + yield self.next_button + yield Static('', id='right-padding') + yield Footer() + + async def on_mount(self) -> None: + for panel in self.panels: + panel.widgets.disabled = True + if self.size.height < 35: + self.add_class('small') + else: + self.add_class('large') + if not terminal_supports_unicode(): + self.add_class('no-unicode') + self.set_default_executor() + + body = self.get_widget_by_id('body') + + self.watch(body, 'scroll_y', self._on_y_scroll) + self.next_button.focus() + self.activate_panel(0) + self.copy_to_clipboard('Testing, 1, 2, 3') + + async def _on_y_scroll(self, y: int) -> None: + log.write(f'on_y_scroll({y})\n') + cy = y + self.size.height / 2 + for i in range(len(self.panels)): + # parent because all panels are wrapped + panel = self.panels[i] + region = panel.widgets.virtual_region + if cy >= region.y and cy <= region.bottom: + self.activate_panel(i, False) + break + + def done(self) -> None: + super().exit() + + def activate_panel(self, n: int, scroll: Optional[bool] = True) -> None: + if n == len(self.panels): + self.done() + asyncio.create_task(self.a_activate_panel(n, scroll)) + + async def a_activate_panel(self, n: int, scroll: Optional[bool] = True) -> None: + try: + if n == self.state.active_panel: + return + new_panel = self.panels[n] + log.write(f'activate_panel({n}: {new_panel}, scroll={scroll})\n') + labels = cast(List[Label], self.sidebar.children) + if self.state.active_panel is not None: + old_panel = self.panels[self.state.active_panel] + if scroll and n > self.state.active_panel: + if not await old_panel.validate(): + return + label = labels[self.state.active_panel] + label.update(' ' + old_panel.label) + old_panel.widgets.disabled = True + old_panel.active = False + for i in range(n): + labels[i].styles.text_style = 'bold' + for i in range(n, len(labels)): + labels[i].styles.text_style = 'none' + label = labels[n] + label.update('> ' + new_panel.label) + self.prev_button.disabled = n == 0 + self.set_next_button(n == len(self.panels) - 1) + if scroll: + new_panel.anchor() + + self.state.active_panel = n + new_panel.active = True + await new_panel.activate() + new_panel.widgets.disabled = False + except Exception as ex: + import traceback + log.write(f'Ex: {ex}\n') + traceback.print_exc(file=log) + + def set_next_button(self, exit: bool) -> None: + btn_next = self.get_widget_by_id('btn-next') + assert isinstance(btn_next, ShortcutButton) + if exit: + btn_next.set_label('E&xit') + else: + btn_next.set_label('&Next') + + def register_button_shortcut(self, char: str, btn: ShortcutButton) -> None: + char = char.lower() + if char not in self.shortcuts: + self.shortcuts[char] = [] + self.shortcuts[char].append(btn) + self.bind(f'{char}, alt+{char}', 'on_key', show=False) + + def on_key(self, event: events.Key) -> None: + k = event.key + if k.startswith('alt+'): + k = k[4:] + if k in self.shortcuts: + btns = self.shortcuts[k] + for btn in btns: + if not btn.disabled and btn.is_on_screen: + btn.press() + + def action_next_page(self) -> None: + self.next_button.press() + + def action_prev_page(self) -> None: + self.prev_button.press() + + def action_scroll_down(self) -> None: + body = self.get_widget_by_id('body') + body.scroll_down() + + def action_scroll_up(self) -> None: + body = self.get_widget_by_id('body') + body.scroll_up() + + @on(Button.Pressed, '#btn-next') + async def next_item(self) -> None: + try: + assert self.state.active_panel is not None + self.activate_panel(self.next_enabled_panel(self.state.active_panel, 1)) + except IndexError: + log.write('No next panel. Exiting\n') + await self.action_quit() + + def next_enabled_panel(self, crt: int, delta: int) -> int: + crt += delta + while crt < len(self.panels) and crt >= 0: + if self.panels[crt].disabled: + crt += delta + else: + return crt + raise IndexError('No next enabled panel') + + @on(Button.Pressed, '#btn-prev') + async def prev_item(self) -> None: + assert self.state.active_panel is not None + self.activate_panel(self.next_enabled_panel(self.state.active_panel, -1)) + + def _focus_next(self) -> None: + btn = self.get_widget_by_id('btn-next') + btn.focus() + + def action_confirm_quit(self) -> None: + asyncio.create_task(self._confirm_quit()) + + async def _confirm_quit(self) -> None: + result = await ExitConfirmDialog().run(self) + if result == 'cancel': + return + await self.action_quit() + + async def action_quit(self) -> None: + install_method = self.state.install_method + if install_method is not None and install_method.name == 'custom': + self.exit(message=f'Tests can be run with the following command:\n' + f'\t{install_method.preview}') + else: + self.exit() + + def set_default_executor(self) -> None: + label, name = self.state.get_executor() + + batch_warner = self.get_widget_by_id('warn-no-batch') + if name == 'none': + batch_warner.visible = True + else: + batch_warner.visible = False + + self.scheduler_panel.set_scheduler(name) + + +def run(conftest: types.ModuleType, ci_runner: types.ModuleType) -> None: + PSIJCIInstallWizard(conftest, ci_runner).run() diff --git a/tests/installer/panels/basic_info_panel.py b/tests/installer/panels/basic_info_panel.py new file mode 100644 index 00000000..0aac38b2 --- /dev/null +++ b/tests/installer/panels/basic_info_panel.py @@ -0,0 +1,100 @@ +import random +import socket +import string + +from .panel import Panel +from ..log import log +from ..widgets import ShortcutButton + +from textual import on +from textual.containers import Vertical, Horizontal +from textual.widget import Widget +from textual.widgets import Input, Label, Button + + +FQDN = socket.getfqdn() + + +class BasicInfoPanel(Panel): + def _build_widgets(self) -> Widget: + self.name_input = Input(placeholder='Enter machine name', id='name-input') + self.email_input = Input(placeholder='Enter email', id='email-input') + return Vertical( + Label('Some basic information', classes='header'), + Label('The name should be something descriptive, such as ' + '"aurora.alcf.anl.gov". This name will be displayed on the online testing ' + 'dashboard.', classes='help-text media-large', shrink=True, expand=True), + Vertical( + Label('Machine name (e.g. echo.example.net):', classes='form-label'), + Horizontal( + self.name_input, + ShortcutButton('Use &FQDN', id='btn-use-fqdn'), + id='name-input-group' + ), + classes='form-row h-auto' + ), + Label('Your email allows us to contact you if we need more information about failing ' + 'tests. It does not appear publicly on the dashboard.', + classes='help-text', shrink=True, expand=True), + Vertical( + Label('Maintainer email:', classes='form-label'), + self.email_input, + classes='form-row h-auto' + ), + classes='panel' + ) + + @property + def label(self) -> str: + return 'Basic info' + + @property + def name(self) -> str: + return 'basic-info' + + async def activate(self) -> None: + log.write('Activate basic info panel\n') + if self.name_input.value == '': + conf_name = self.state.conf['id'] + log.write(f'Conf name: {conf_name}\n') + val = None + if conf_name == 'hostname': + val = FQDN + elif conf_name == 'random': + val = ''.join(random.choices(string.ascii_uppercase + string.digits, k=8)) + elif len(conf_name) > 0 and conf_name[0] == '"' and conf_name[-1] == '"': + val = conf_name[1:-1] + else: + val = '' + assert val is not None + self.name_input.value = val + if self.email_input.value == '': + self.email_input.value = self.state.conf['maintainer_email'] + self.name_input.disabled = False + self.name_input.focus(False) + + async def validate(self) -> bool: + log.write(f'name value: {self.name_input.value}\n') + name_valid = self.name_input.value != '' + self.name_input.set_class(not name_valid, 'invalid', update=True) + + if name_valid: + self.state.update_conf('id', self.name_input.value) + if self.email_input.value: + self.state.update_conf('maintainer_email', self.email_input.value) + return name_valid + + @on(Input.Submitted, '#name-input') + def name_submitted(self) -> None: + email = self.get_widget_by_id('email-input') + email.focus(False) + + @on(Input.Submitted, '#email-input') + def email_submitted(self) -> None: + self.app._focus_next() # type: ignore + + @on(Button.Pressed, '#btn-use-fqdn') + def use_fqdn_pressed(self) -> None: + name_input = self.get_widget_by_id('name-input') + assert isinstance(name_input, Input) + name_input.value = FQDN diff --git a/tests/installer/panels/batch_scheduler_panel.py b/tests/installer/panels/batch_scheduler_panel.py new file mode 100644 index 00000000..265c37cd --- /dev/null +++ b/tests/installer/panels/batch_scheduler_panel.py @@ -0,0 +1,418 @@ +import asyncio +import re +from typing import Tuple, Optional, cast + +from psij import JobExecutor, Job, JobSpec, ResourceSpecV1, JobAttributes, JobState +from .panel import Panel +from ..dialogs import TestJobsDialog +from ..log import log +from ..state import Attr +from ..widgets import MSelect, ShortcutButton + +from textual import on +from textual.app import ComposeResult +from textual.containers import Horizontal, Vertical +from textual.screen import ModalScreen +from textual.widget import Widget +from textual.widgets import Label, Input, Button, TextArea, Checkbox, Select + + +class EditAttrsScreen(ModalScreen[None]): + BINDINGS = [ + ('F1', 'context_help', 'Help'), + ('down', 'input_down'), + ('up', 'input_up'), + ('c', 'cancel'), + ('alt+c', 'cancel'), + ('a', 'apply'), + ('alt+a', 'apply'), + ('ctrl+q', 'quit') + ] + + def __init__(self, panel: 'BatchSchedulerPanel') -> None: + super().__init__() + self.panel = panel + self.state = panel.state + + def compose(self) -> ComposeResult: + yield Vertical( + Label('Edit custom attributes.', + classes='header', shrink=True, expand=True), + Label('If your scheduler requires non-standard parameters (e.g., "constraint=knl"), ' + 'enter them here.', + classes='help-text', shrink=True, expand=True), + Vertical( + Horizontal( + Label('Name', classes='th'), + Label('Value', classes='th'), + Label('Filter', classes='th'), + classes='attr-header h-auto' + ), + self._make_row(0)[0], + id='attr-rows', classes='attrs' + ), + Horizontal( + Button('[b bright_yellow]C[/b bright_yellow]lose', variant='error', + id='btn-attrs-cancel', classes='m-r-1'), + Button('[b bright_yellow]A[/b bright_yellow]pply', variant='primary', + id='btn-attrs-apply', classes='m-r'), + classes='action-area' + ), + id='edit-attrs-dialog', classes='dialog' + ) + + def _make_row(self, index: int) -> Tuple[Widget, Input, Input, Input]: + name = Input('', id=f'attr-name-{index}', classes='attr-name td') + value = Input('', id=f'attr-value-{index}', classes='attr-value td') + filter = Input('', id=f'attr-filter-{index}', classes='attr-filter td') + return Horizontal(name, value, filter, classes='attr-row h-auto'), name, value, filter + + def _get_row(self, input: Input) -> int: + id = input.id + assert id is not None + return int(id.split('-')[-1]) + + def action_input_down(self) -> None: + self._move_input_focus(1) + + def action_input_up(self) -> None: + self._move_input_focus(-1) + + def _move_input_focus(self, d: int) -> None: + q = self.query('*:focus') + if len(q) > 0: + focused = q.first() + if focused.has_class('td'): + id = focused.id + assert id is not None + id_parts = id.split('-') + index = int(id_parts[-1]) + q2 = self.query('#' + '-'.join(id_parts[:-1]) + '-' + str(index + d)) + if len(q2) > 0: + q2.first().focus() + else: + self.get_widget_by_id('btn-attrs-apply').focus() + else: + if focused.id == 'btn-attrs-apply': + if d == 1: + self.get_widget_by_id('attr-name-0').focus() + else: + self.get_widget_by_id('attr-rows').children[-1].children[0].focus() + + @on(Input.Submitted, '.attr-name') + def name_submitted(self, event: Input.Submitted) -> None: + log.write(f'input submitted {event.input.id}\n') + event.input.remove_class('invalid') + assert event.input.parent is not None + if event.input.value != '': + self._ensure_row(self._get_row(event.input) + 1) + event.input.parent.children[1].focus() + + @on(Input.Submitted, '.attr-value') + def value_submitted(self, event: Input.Submitted) -> None: + event.input.remove_class('invalid') + assert event.input.parent is not None + if event.input.value != '': + self._ensure_row(self._get_row(event.input) + 1) + event.input.parent.children[2].focus() + + @on(Input.Submitted, '.attr-filter') + def filter_submitted(self, event: Input.Submitted) -> None: + event.input.remove_class('invalid') + row = self._get_row(event.input) + if event.input.value != '': + self._ensure_row(row + 1) + next_name = self.query(f'#attr-name-{row + 1}') + if next_name: + next_name.focus() + else: + self.get_widget_by_id('btn-attrs-apply').focus() + + def _ensure_row(self, row: int) -> Tuple[Input, Input, Input]: + next_name = self.query(f'#attr-name-{row}') + if not next_name: + rows = self.get_widget_by_id('attr-rows') + row_widget, name, value, filter = self._make_row(row) + rows.mount(row_widget) + else: + row_widget = self.get_widget_by_id('attr-rows').children[row + 1] + name = cast(Input, row_widget.children[0]) + value = cast(Input, row_widget.children[1]) + filter = cast(Input, row_widget.children[2]) + return name, value, filter + + def on_mount(self) -> None: + ix = 0 + for attr in self.state.attrs: + name_input, value_input, filter_input = self._ensure_row(ix) + name_input.value = attr.name + value_input.value = attr.value + if attr.filter != '.*': + filter_input.value = attr.filter + ix += 1 + + def action_quit(self) -> None: + self.app.exit() + + def action_apply(self) -> None: + if self._validate_and_commit(): + self.app.pop_screen() + + def action_cancel(self) -> None: + self.app.pop_screen() + + @on(Button.Pressed, '#btn-attrs-apply') + def apply_pressed(self) -> None: + self.action_apply() + + @on(Button.Pressed, '#btn-attrs-cancel') + def cancel_pressed(self) -> None: + self.action_cancel() + + def _tag_input(self, is_valid: bool, input: Input, + first_invalid: Optional[Input]) -> Optional[Input]: + if not is_valid: + input.add_class('invalid') + if first_invalid is None: + return input + return first_invalid + + def _re_valid(self, restr: str) -> bool: + try: + re.compile(restr) + return True + except Exception: + return False + + def _validate_and_commit(self) -> bool: + rows = self.get_widget_by_id('attr-rows') + attrs = [] + first_invalid = None + for row_index in range(1, len(rows.children)): + row = rows.children[row_index] + name = row.children[0] + value = row.children[1] + filter = row.children[2] + assert (isinstance(name, Input) and isinstance(value, Input) + and isinstance(filter, Input)) + if name.value == '' and value.value == '' and filter.value == '': + continue + # at least one set + name_valid = name.value != '' + value_valid = value.value != '' + filter_valid = filter.value == '' or self._re_valid(filter.value) + first_invalid = self._tag_input(name_valid, name, first_invalid) + first_invalid = self._tag_input(value_valid, value, first_invalid) + first_invalid = self._tag_input(filter_valid, filter, first_invalid) + if name_valid and value_valid and filter_valid: + attrs.append(Attr(filter.value, name.value, value.value)) + if first_invalid is None: + self.state.set_custom_attrs(attrs) + self.panel.update_attrs() + else: + first_invalid.focus() + return first_invalid is None + + +class BatchSchedulerPanel(Panel): + BINDINGS = [ + ('t', 'toggle_test_job'), + ('alt+t', 'toggle_test_job') + ] + + def _build_widgets(self) -> Widget: + return Vertical( + Label('Select and configure a batch system.', classes='header'), + Vertical( + Label('Your system does not appear to have a batch scheduler. If you are certain ' + 'that this is wrong, you can select one below. If not, tests will be run ' + 'using non-batch executors.', classes='help-text media-large', + shrink=True, expand=True), + id='warn-no-batch' + ), + Horizontal( + Vertical( + Label('Batch system:', classes='form-label'), + MSelect( + [('Select...', 'none'), ('Local only', 'local'), ('Slurm', 'slurm'), + ('PBS', 'pbs'), ('LSF', 'lsf'), ('Cobalt', 'cobalt')], + id='batch-selector', + allow_blank=False), + classes='form-col-3 form-row' + ), + Vertical( + Label('Queue:', classes='form-label'), + Input(id='queue-input'), + classes='form-col-3 form-row batch-valid' + ), + Vertical( + Label('Account/project:', classes='form-label'), + Input(id='account-input'), + classes='form-col-3 form-row batch-valid' + ), + classes='w-100 form-row', id='batch-system-group' + ), + Horizontal( + Vertical( + Label('Custom attributes:', classes='form-label'), + TextArea('', id='custom-attrs', read_only=True, soft_wrap=False), + id='attr-cell', + classes='form-col-2 h-auto' + ), + Vertical( + Label('', classes='form-label'), + ShortcutButton('&Edit attributes', id='btn-edit-attrs'), + Checkbox('Run [b bright_yellow]t[/b bright_yellow]est job', + id='cb-run-test-job', classes='m-t-1'), + classes='form-col-2 h-auto' + ), + classes='w-100 h-auto batch-valid' + ), + classes='panel' + ) + + async def validate(self) -> bool: + if self.state.run_test_job and self.state.scheduler is not None: + return await self.run_test_jobs() + else: + return True + + @property + def label(self) -> str: + return 'Scheduler' + + @property + def name(self) -> str: + return 'scheduler' + + async def activate(self) -> None: + self.update_attrs() + selector = self.get_widget_by_id('batch-selector') + selector.focus(False) + + def update_attrs(self) -> None: + s = '' + for attr in self.state.attrs: + if attr.filter == '.*': + s += f'{attr.name}: {attr.value}\n' + else: + s += f'{attr.name}: {attr.value} ({attr.filter})\n' + control = self.get_widget_by_id('custom-attrs') + assert isinstance(control, TextArea) + control.text = s + + @on(Select.Changed, '#batch-selector') + def batch_system_selected(self) -> None: + selector = cast(Select[str], self.get_widget_by_id('batch-selector')) + sched = selector.selection + disabled = (sched is None or sched == 'none' or sched == 'local') + if disabled: + self.state.scheduler = None + else: + self.state.scheduler = sched + for widget in self.query('.batch-valid'): + widget.disabled = disabled + if sched == 'local': + self.app._focus_next() # type: ignore + else: + self.get_widget_by_id('queue-input').focus() + + def set_scheduler(self, name: str) -> None: + selector = self.get_widget_by_id('batch-selector') + assert isinstance(selector, Select) + selector.value = name + + @on(Input.Submitted, '#queue-input') + def queue_submitted(self) -> None: + bottom = self.get_widget_by_id('account-input') + bottom.focus() + + @on(Input.Submitted, '#account-input') + def account_submitted(self) -> None: + self.app._focus_next() # type: ignore + + @on(Button.Pressed, '#btn-edit-attrs') + def action_edit_attrs(self) -> None: + if not self.active or self.state.scheduler is None: + return + self.app.push_screen(EditAttrsScreen(self)) + + def action_toggle_test_job(self) -> None: + cb = self.get_widget_by_id('cb-run-test-job') + if not self.active or self.state.scheduler is None: + return + assert isinstance(cb, Checkbox) + cb.value = not cb.value + + @on(Checkbox.Changed, '#cb-run-test-job') + def action_cb_test_job_changed(self, event: Checkbox.Changed) -> None: + self.run_test_job = event.checkbox.value + + async def run_test_jobs(self) -> bool: + jd = TestJobsDialog() + self.app.push_screen(jd) + # without this, the widgets in TestJobsDialog do not seem to be accessible + await asyncio.sleep(0.1) + j1 = await self._run_test_job(jd, 1, 'Single node job', None, '') + j2 = await self._run_test_job(jd, 2, 'Multi node job ', ResourceSpecV1(node_count=4), + f'test_nodefile[{self.state.scheduler}:single') + + if j1 and j2: + jd.focus_continue_button() + else: + jd.focus_back_button() + + return await jd.wait() + + async def _run_test_job(self, jd: TestJobsDialog, job_no: int, label: str, + rspec: Optional[ResourceSpecV1], test_name: str) -> bool: + try: + jd.set_running(job_no) + await asyncio.sleep(2) + + job = self._launch_job(test_name, rspec) + + await self._wait_for_queued_state(job) + try: + job.cancel() + except Exception as ex: + log.write(f'Failed to cancel job: {ex}') + jd.set_status(job_no, 'OK', 'status-succeeded') + return True + except Exception as ex: + jd.log_error(label, ex) + jd.set_status(job_no, 'Failed', 'status-failed') + return False + + def _launch_job(self, test_name: str, rspec: Optional[ResourceSpecV1]) -> Job: + s = self.state.scheduler + assert s is not None + + ex = JobExecutor.get_instance(s) + + attrs = JobAttributes() + account = self.state.conf.get('account', '') + queue = self.state.conf.get('queue', '') + if account != '': + attrs.account = account + if queue != '': + attrs.queue_name = queue + for attr in self.state.attrs: + if re.match(attr.filter, test_name): + attrs.set_custom_attribute(attr.name, attr.value) + + job = Job(JobSpec('/bin/date', attributes=attrs, resources=rspec)) + ex.submit(job) + log.write('Job submitted\n') + return job + + async def _wait_for_queued_state(self, job: Job) -> None: + while True: + status = job.status + log.write(f'Job status: {status}\n') + state = status.state + if state == JobState.QUEUED or state.is_greater_than(JobState.QUEUED): + if state == JobState.FAILED: + raise Exception(status.message) + return + await asyncio.sleep(0.2) diff --git a/tests/installer/panels/complete_panel.py b/tests/installer/panels/complete_panel.py new file mode 100644 index 00000000..a7e605b5 --- /dev/null +++ b/tests/installer/panels/complete_panel.py @@ -0,0 +1,30 @@ +from .panel import Panel +from textual.containers import Vertical +from textual.widget import Widget +from textual.widgets import Label + + +class CompletePanel(Panel): + def _build_widgets(self) -> Widget: + return Vertical( + Label('Installation complete.', + classes='header', shrink=True, expand=True), + Label('The PSI/J CI tests have been installed. You can press the Exit button below ' + 'to exit this installer.', + classes='main-text', shrink=True, expand=True), + classes='panel' + ) + + @property + def label(self) -> str: + return 'Complete' + + @property + def name(self) -> str: + return 'complete' + + async def validate(self) -> bool: + return True + + async def activate(self) -> None: + pass diff --git a/tests/installer/panels/intro_panel.py b/tests/installer/panels/intro_panel.py new file mode 100644 index 00000000..afbb5d62 --- /dev/null +++ b/tests/installer/panels/intro_panel.py @@ -0,0 +1,44 @@ +from .panel import Panel +from textual.containers import Vertical +from textual.widget import Widget +from textual.widgets import Label + +from ..dialogs import ExistingInstallConfirmDialog +from ..log import log +from ..install_methods import existing + + +class IntroPanel(Panel): + def _build_widgets(self) -> Widget: + return Vertical( + Label('This tool will guide you in setting up the PSI/J nightly tests.', + classes='header', shrink=True, expand=True), + Label('Hint: you can likely use your terminal\'s application mode (typically the' + ' Shift or Option keys) to copy and paste text. For example, Shift+Drag to' + ' select text or Shift+Ctrl+V to paste from the clipboard.', + classes='help-text', shrink=True, expand=True), + classes='panel' + ) + + @property + def label(self) -> str: + return 'Introduction' + + @property + def name(self) -> str: + return 'intro' + + async def validate(self) -> bool: + return True + + async def activate(self) -> None: + log.write('Intro activate\n') + if self.state.disable_install: + return + m = existing() + if m is not None: + result = await ExistingInstallConfirmDialog(m.name).run(self.app) + if result == 'quit': + self.app.exit() + if result == 'update': + self.app.disable_install() # type: ignore diff --git a/tests/installer/panels/key_panel.py b/tests/installer/panels/key_panel.py new file mode 100644 index 00000000..593654cf --- /dev/null +++ b/tests/installer/panels/key_panel.py @@ -0,0 +1,228 @@ +import asyncio +from typing import Optional + +from ..dialogs import KeyRequestDialog, ProgressDialog, ErrorDialog +from ..log import log +from .panel import Panel +from ..state import KEY_PATH +from ..widgets import ShortcutButton + +from textual import on +from textual.containers import Vertical, Horizontal +from textual.widget import Widget +from textual.widgets import Input, Label, MaskedInput, Button + + +class KeyPanel(Panel): + def _build_widgets(self) -> Widget: + self.email_input = Input(placeholder='Enter email', id='key-email-input') + self.request_button = ShortcutButton('&Request key', id='btn-request-key') + return Vertical( + Label('Dashboard authentication key', classes='header'), + Label('A key associates your test result uploads with a verified email and allows us ' + 'to prevent unauthorized uploads.', + classes='help-text', shrink=True, expand=True), + Label(f'A valid key was found in {KEY_PATH}', classes='p-l m-b -success hidden', + id='msg-auth-key-valid', shrink=True, expand=True), + Label(f'No key was found in {KEY_PATH}', classes='p-l m-b -error hidden', + id='msg-auth-key-not-found', shrink=True, expand=True), + Label(f'Invalid key found in {KEY_PATH}', classes='p-l m-b -error hidden', + id='msg-auth-key-invalid', shrink=True, expand=True), + Vertical( + Label('Your email address:', classes='form-label'), + Horizontal( + self.email_input, + self.request_button + ), + classes='form-row h-auto', id='key-email-input-group' + ), + Vertical( + Label('Or enter a key below:', classes='form-label'), + MaskedInput(template='NNNNNNNNNNNNNNNN:NNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN', # noqa: E501 + id='key-input'), + Label('', classes='form-label -error', id='key-check-error'), + classes='form-row h-auto', id='key-input-group' + ), + classes='panel' + ) + + @property + def label(self) -> str: + return 'Key' + + @property + def name(self) -> str: + return 'key' + + def _show_message(self, id: str) -> None: + for i in ['msg-auth-key-valid', 'msg-auth-key-invalid', 'msg-auth-key-not-found']: + if i == id: + self.get_widget_by_id(i).set_class(False, 'hidden') + else: + self.get_widget_by_id(i).set_class(True, 'hidden') + + async def activate(self) -> None: + if self.state.has_key and self.state.key_is_valid is None: + log.write('Verifying key...\n') + self.state.key_is_valid = await self.verify_key() + input_active = self.update() + if self.email_input.value == '': + maintainer_email = self.state.conf['maintainer_email'] + log.write(f'No email, setting to maintainer email: {maintainer_email}\n') + self.email_input.value = maintainer_email + if input_active: + self.email_input.focus(False) + + @on(Button.Pressed, '#btn-request-key') + def action_request_key(self) -> None: + email_input = self.get_widget_by_id('key-email-input') + assert isinstance(email_input, Input) + email = email_input.value + asyncio.create_task(self.request_key(email)) + + @on(Input.Submitted, '#key-email-input') + def key_email_submitted(self) -> None: + btn = self.get_widget_by_id('btn-request-key') + btn.focus() + + @on(Input.Changed, '#key-input') + def key_input_changed(self) -> None: + # remove error message + self._set_key_check_error('') + self.get_widget_by_id('key-input').remove_class('invalid') + + @on(Input.Submitted, '#key-input') + async def key_submitted(self, event: Input.Changed) -> None: + key = event.value + try: + valid = await self.verify_key(key) + self.get_widget_by_id('key-input').set_class(not valid, 'invalid') + if not valid: + self._set_key_check_error('Invalid key.') + else: + self._set_key_check_error('') + self._key_received(key) + self.update() + except Exception as ex: + log.write(f'Ex: {ex}\n') + + def _set_key_check_error(self, text: str) -> None: + key_check_error = self.get_widget_by_id('key-check-error') + assert isinstance(key_check_error, Label) + key_check_error.update(text) + + def _key_received(self, key: str) -> None: + with open(KEY_PATH, 'w') as f: + f.write(key) + self.state.has_key = True + self.state.key_is_valid = True + self.app._focus_next() # type: ignore + + def update(self) -> bool: + input_active = True + if self.state.has_key: + if self.state.key_is_valid: + self._show_message('msg-auth-key-valid') + input_active = False + else: + self._show_message('msg-auth-key-invalid') + else: + self._show_message('msg-auth-key-not-found') + + self.get_widget_by_id('key-email-input-group').disabled = not input_active + self.get_widget_by_id('key-input-group').disabled = not input_active + return input_active + + async def verify_key(self, key: Optional[str] = None) -> bool: + if not key: + key = self.state.key + if key is None: + return False + # some offline quick validation + if len(key) != 65: + return False + cix = key.find(':') + if cix != 16: + return False + + self.app.push_screen(ProgressDialog('Verifying key...')) + try: + result = await self.state.request('/authVerifyKey', {'key': key}, 'Key verification', + self.display_error_dialog) + if result: + success = result['success'] + assert isinstance(success, bool) + return success + finally: + self.app.pop_screen() + return False + + async def request_key(self, email: str) -> None: + key = await self._key_request(email) + if key: + self._key_received(key) + self.update() + + async def _key_request(self, email: str) -> Optional[str]: + self.app.push_screen(ProgressDialog('Initializing request...')) + try: + result = await self.state.request('/keyRequestInit', {'email': email}, + 'Key request initialization', + self.display_error_dialog) + finally: + self.app.pop_screen() + if not result: + return None + request_id = result['id'] + assert isinstance(request_id, str) + base_url = self.state.conf['server_url'] + d = KeyRequestDialog(f'{base_url}/auth/{request_id}') + self.app.push_screen(d) + try: + return await self._run_key_request_loop(request_id, d) + finally: + log.write('Key request loop done\n') + self.app.pop_screen() + + async def _run_key_request_loop(self, request_id: str, d: KeyRequestDialog) -> Optional[str]: + while d.is_active: + result = await self.state.request('/keyRequestStatus', {'id': request_id}, + 'Key request status check', + self.display_error_dialog) + log.write(f'Result: {result}\n') + if not result: + return None + + success = result['success'] + + if not success: + error = result['error'] + assert isinstance(error, str) + ed = ErrorDialog('Error requesting key', error) + await ed.run(self.app) + return None + else: + status = result['status'] + assert isinstance(status, str) + d.update_status(status) + if status == 'verified': + return result['key'] # type: ignore + await asyncio.sleep(5) + return None + + async def validate(self) -> bool: + if not self.state.key_is_valid: + email_valid = self.email_input.value != '' + self.email_input.set_class(not email_valid, 'invalid', update=True) + else: + self.email_input.set_class(False, 'invalid', update=True) + self.request_button.set_class(not self.state.key_is_valid, 'invalid', update=True) + assert self.state.key_is_valid is not None + return self.state.key_is_valid + + async def display_error_dialog(self, title: str, message: str) -> None: + try: + d = ErrorDialog(title, message) + await d.run(self.app) + except Exception as ex: + log.write(f'caught {ex}\n') diff --git a/tests/installer/panels/panel.py b/tests/installer/panels/panel.py new file mode 100644 index 00000000..67287015 --- /dev/null +++ b/tests/installer/panels/panel.py @@ -0,0 +1,39 @@ +from abc import abstractmethod + +from textual.containers import Vertical +from textual.widget import Widget + +from ..state import State + + +class Panel(Vertical): + def __init__(self, state: State) -> None: + self.state = state + self._widgets = self._build_widgets() + Vertical.__init__(self, self._widgets, classes='panel-wrapper') + self.active = False + + @property + def widgets(self) -> Widget: + return self._widgets + + @abstractmethod + def _build_widgets(self) -> Widget: + pass + + @property + @abstractmethod + def label(self) -> str: + pass + + @property + @abstractmethod + def name(self) -> str: + pass + + @abstractmethod + async def validate(self) -> bool: + pass + + async def activate(self) -> None: + pass diff --git a/tests/installer/panels/schedule_panel.py b/tests/installer/panels/schedule_panel.py new file mode 100644 index 00000000..0048c579 --- /dev/null +++ b/tests/installer/panels/schedule_panel.py @@ -0,0 +1,127 @@ +from typing import cast + +from .panel import Panel + +from textual.containers import Vertical, Horizontal +from textual.widget import Widget +from textual.widgets import Label, RadioSet, RadioButton, TextArea + +from ..dialogs import ErrorDialog +from ..log import log +from ..install_methods import METHODS, InstallMethod + + +class MRadioSet(RadioSet): + BINDINGS = [ + ('enter,space', 'select') + ] + + def __init__(self, panel: 'SchedulePanel', *radios: RadioButton, id: str | None = None) -> None: + super().__init__(*radios, id=id) + self.panel = panel + + def watch__selected(self) -> None: + super().watch__selected() + if self._selected is not None: + self.panel.radio_focused(cast(RadioButton, self.children[self._selected])) + + def watch_has_focus(self, value: bool) -> None: + if not value: + self.panel.radio_focused(None) + super().watch_has_focus(value) + + def action_select(self) -> None: + if self._selected is not None: + selected = self.children[self._selected] + assert isinstance(selected, RadioButton) + if selected.value: + self.app._focus_next() # type: ignore + + self.action_toggle_button() + + def get_selected_index(self) -> int | None: + return self._selected + + +class SchedulePanel(Panel): + def _build_widgets(self) -> Widget: + radios = [] + labels = [] + + for m in METHODS: + available, msg = m.is_available() + if msg is None: + msg = '' + radios.append(RadioButton(m.label, id=f'btn-{m.name}', disabled=not available)) + labels.append(Label(msg, id=f'label-{m.name}')) + labels_container = Vertical(*labels, id='method-status', classes='h-auto') + + return Vertical( + Label('Install', classes='header'), + Label('This step schedules the tests to be run daily.', + classes='help-text', shrink=True, expand=True), + Vertical( + Label('Installation method:', classes='form-label'), + Horizontal( + MRadioSet(self, *radios, id='rs-method'), + labels_container, + classes='h-auto' + ), + classes='h-auto m-b-1' + ), + Vertical( + Label('Preview', classes='form-label'), + TextArea('-', id='method-preview', classes='', language='bash', + read_only=True, disabled=True), + classes='form-row h-auto' + ), + classes='panel' + ) + + @property + def label(self) -> str: + return 'Install' + + @property + def name(self) -> str: + return 'install' + + def _get_selected_method(self) -> InstallMethod | None: + radio_set = self.get_widget_by_id('rs-method') + assert isinstance(radio_set, MRadioSet) + selected_index = radio_set.get_selected_index() + if selected_index is None: + return None + return METHODS[selected_index] + + async def validate(self) -> bool: + try: + m = self._get_selected_method() + assert m is not None + self.state.install_method = m + m.install() + return True + except Exception as ex: + await ErrorDialog('Error scheduling tests', str(ex)).run(self.app) + return False + + async def activate(self) -> None: + self.get_widget_by_id('rs-method').focus(scroll_visible=False) + for btn in self.query('RadioButton'): + if not btn.disabled: + assert isinstance(btn, RadioButton) + btn.value = True + break + + def radio_focused(self, btn: RadioButton | None) -> None: + log.write(f'focused {btn}\n') + preview = self.get_widget_by_id('method-preview') + assert isinstance(preview, TextArea) + + if btn is not None: + assert btn.id is not None + name = btn.id.split('-')[-1] + for m in METHODS: + if m.name == name: + preview.text = m.preview + return diff --git a/tests/installer/py.typed b/tests/installer/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/tests/installer/state.py b/tests/installer/state.py new file mode 100644 index 00000000..fdcc787f --- /dev/null +++ b/tests/installer/state.py @@ -0,0 +1,172 @@ +import asyncio +import json +import os +import pathlib +import types + +import requests +from collections import namedtuple +from typing import List, Dict, Optional, Callable, Awaitable, Tuple, cast + +from .install_methods import InstallMethod +from .log import log + + +KEY_PATH = pathlib.Path('~/.psij/key').expanduser() + + +Attr = namedtuple('Attr', ['filter', 'name', 'value']) + + +class _Options: + def __init__(self) -> None: + self.custom_attributes = None + + +class ConfWrapper: + def __init__(self, dict: Dict[str, str | int | bool | None]): + self.dict = dict + self.option = _Options() + dict['run_id'] = 'x' + dict['save_results'] = False + dict['upload_results'] = True + dict['branch_name_override'] = None + + def getoption(self, name: str) -> str | int | bool | None: + return self.dict[name] + + +class State: + def __init__(self, conftest: types.ModuleType, ci_runner: types.ModuleType): + self.conf = ci_runner.read_conf('testing.conf') + self.env = conftest._discover_environment(ConfWrapper(self.conf)) + log.write('Conf: ' + str(self.conf) + '\n') + log.write(str(self.env) + '\n') + self.disable_install = False + self.install_method: InstallMethod | None = None + self.active_panel: int | None = None + self.scheduler: str | None = None + self.attrs = self._parse_attributes() + self.run_test_job = True + self.has_key = KEY_PATH.exists() + if self.has_key: + with open(KEY_PATH, 'r') as f: + self.key = f.read().strip() + self.key_is_valid: bool | None = None + log.write(f'has key: {self.has_key}\n') + + def _parse_attributes(self) -> List[Attr]: + attrspec = json.loads('[' + self.conf['custom_attributes'] + ']') + attrs = [] + for filter_entry in attrspec: + filter = filter_entry['filter'] + for name, value in filter_entry['value'].items(): + ns = name.split('.', 2) + if len(ns) != 2: + continue + attrs.append(Attr(filter, ns[1], value)) + return attrs + + def _write_conf_value(self, name: str, value: str) -> None: + self.conf[name] = value + nlen = len(name) + with open('testing.conf', 'r') as old: + with open('testing.conf.new', 'w') as new: + for line in old: + sline = line.strip() + if sline == '' or sline.startswith('#'): + new.write(line) + elif sline.startswith(name) and sline[nlen] in [' ', '\t', '=']: + new.write(f'{name} = {value}\n') + else: + new.write(line) + os.rename('testing.conf.new', 'testing.conf') + + def update_conf(self, name: str, value: str) -> None: + if name == 'id': + self._write_conf_value('id', f'"{value}"') + else: + self._write_conf_value(name, value) + + async def request(self, query: str, data: Dict[str, object], title: str, + error_cb: Callable[[str, str], Awaitable[Dict[str, object] | None]]) \ + -> Dict[str, object] | None: + baseUrl = self.conf['server_url'] + response = await asyncio.to_thread(requests.post, baseUrl + query, data=data) + return await self._check_error(response, title, error_cb) + + async def _check_error(self, response: requests.Response, title: str, + error_cb: Callable[[str, str], Awaitable[Dict[str, object] | None]]) \ + -> Dict[str, object] | None: + log.write(f'Response: {response.text}\n') + if response.status_code != 200: + msg = self._extract_response_message(response.text) + if msg: + error = f'Server responded with an error: {msg}' + else: + error = response.text + else: + data = response.json() + if 'success' in data: + return cast(Dict[str, object], data) + else: + error = 'Unknown error' + log.write(f'Error: {error}\n') + if error is not None: + await error_cb(title + ' failed', error) + return None + + def _extract_response_message(self, html: str) -> Optional[str]: + # standard cherrypy error page + ix = html.find('

') + if ix != -1: + return html[ix + 3:html.find('

', ix)] + else: + return None + + def set_custom_attrs(self, attrs: List[Attr]) -> None: + self.attrs = attrs + self._write_custom_attrs(attrs) + + def _write_custom_attrs(self, attrs: List[Attr]) -> None: + # this one is weird to set, so we need a custom solution + sched = self.scheduler + name = 'custom_attributes' + nlen = len(name) + continued = False + purge = False + with open('testing.conf', 'r') as old, open('testing.conf.new', 'w') as new: + for line in old: + sline = line.strip() + if sline == '' or sline.startswith('#'): + new.write(line) + elif sline.startswith(name) and sline[nlen] in [' ', '\t', '=', '[']: + if not purge: + for attr in attrs: + if attr.filter == '.*' or attr.filter == '': + new.write(f'custom_attributes = ' + f'"{sched}.{attr.name}": "{attr.value}"\n') + else: + new.write(f'custom_attributes[{attr.filter}] = ' + f'"{sched}.{attr.name}": "{attr.value}"\n') + purge = True + if sline.endswith('\\'): + continued = True + elif continued: + # just consume the line + continued = sline.endswith('\\') + else: + new.write(line) + os.rename('testing.conf.new', 'testing.conf') + + def get_executor(self) -> Tuple[str, str]: + if self.env['has_slurm']: + return ('Slurm', 'slurm') + if self.env['has_pbs']: + return ('PBS', 'pbs') + if self.env['has_lsf']: + return ('LSF', 'lsf') + if self.env['has_cobalt']: + return ('Cobalt', 'cobalt') + + return ('None', 'none') diff --git a/tests/installer/style.tcss b/tests/installer/style.tcss new file mode 100644 index 00000000..a7e75183 --- /dev/null +++ b/tests/installer/style.tcss @@ -0,0 +1,738 @@ +Screen { + align: center middle; +} + +Header, Footer { + background: #008080; +} + +Button { + text-style: none; +} + +.hidden { + display: none; +} + +App.large .media-small { + display: none; +} + +App.small .media-large { + display: none; +} + +App.no-unicode RadioSet { + border: tall #202020; + padding: 1; + background: #202020; +} + +App.small RadioSet, App.small.no-unicode RadioSet { + padding-top: 0; + padding-bottom: 0; + padding-left: 1; + padding-right: 1; +} + +* { + scrollbar-color: #006060; +} + +*:disabled { + scrollbar-color: #002080; + scrollbar-background: #000040; +} + + +App.no-unicode RadioSet:focus { + border: tall #0080ff; + background: #282828; +} + +App.no-unicode RadioSet > RadioButton { + color: #202020; +} + +App.no-unicode RadioSet:focus > RadioButton.-selected { + color: #0080ff; + background: #0080ff; +} + +RadioSet:focus { + background: #282828; + background-tint: 0%; +} + +RadioSet > RadioButton { + padding: 0; + background: #202020; + color: #404040; +} + +RadioSet > RadioButton .toggle--button { + background: #000000; + color: #000000; +} + +App.no-unicode RadioSet > RadioButton .toggle--button { + background: #404040; +} + +RadioSet > RadioButton .toggle--label { + color: #d0d0d0; + background: #202020; +} + +RadioSet > RadioButton .toggle--button { + background: #404040; +} + +RadioSet:focus > RadioButton .toggle--button { + background: #404040; +} + +RadioSet > RadioButton.-on .toggle--button { + color: #00ff80; + background: #404040; +} + +RadioSet:focus > RadioButton.-on .toggle--button { + background: #404040; +} + +RadioSet:focus > RadioButton, RadioSet:focus > RadioButton .toggle--label { + background: #282828; + background-tint: 0%; +} + +RadioSet:focus > RadioButton.-selected { + background: #0080ff; + text-style: none; +} + +RadioSet:focus > RadioButton.-selected .toggle--label { + color: #ffffff; + background: #0080ff; +} + +RadioSet > RadioButton:disabled .toggle--label { + color: #808080; +} + +Input.invalid { + background: #400000; +} + +Button.invalid { + background: #400000; +} + +ToggleButton { + background: #000060; + border: none; + padding: 1; + color: #606060; +} + +ToggleButton:disabled { + color: #404040; +} + +App.no-unicode ToggleButton { + color: #ffffff; +} + +ToggleButton:focus { + background: #0040a0; +} + +ToggleButton .toggle--label { + color: #ffffff; +} + +ToggleButton .toggle--button { + color: #00ff00; + background: #606060; +} + +App.no-unicode ToggleButton .toggle--button { + background: #000060; +} + +ToggleButton.-on .toggle--button { + color: #000060; + background: #000060; +} + +App.no-unicode ToggleButton.-on .toggle--button { + color: #00ff00; +} + + +ToggleButton:focus .toggle--button { + background: #0040a0; +} + +ToggleButton.-on:focus .toggle--button { + background: #0040a0; + color: #0040a0; +} + +ToggleButton:focus .toggle--label { + text-style: none; + background: #0040a0; +} + +LoadingIndicator { + margin-left: 1; + width: 8; + height: 1; + color: #0000ff; +} + +*:disabled .form-label { + color: #808080; +} + +.dialog { + width: 90%; + height: 90%; + border: round #000000; + background: #d0d0d0; +} + +.dialog .header { + color: #000000; +} + +.dialog .main-text { + color: #404040; + display: block; +} + +.dialog Button { + border-top: tall #80a0a0; + background: #408080; + border-bottom: tall #204040; +} + +.dialog Button:focus { + border-top: tall #a0ffff; + background: #60a0a0; + border-top: tall #408080; +} + +.dialog Button.-error { + background: #400000; +} + +.dialog Button.-error:focus { + background: #a00000; +} + +.dialog .action-area Button { + margin-right: 1; +} + +.dialog .action-area Button:last-of-type { + margin-right: 2; +} + +.dialog Label { + color: #606060; +} + +.dialog .attrs .th { + width: 50%; +} + +.dialog .attrs .td { + width: 50%; +} + +.dialog #btn-add-attr { + margin-top: 1; +} + +.dialog .action-area { + padding-top: 1; + padding-right: 1; + padding-left: 1; + align: right middle; + dock: bottom; + height: 4; +} + +#sidebar { + dock: left; + color: #e0e0e0; + background: #002080; + height: 100%; + width: 10%; + min-width: 18; + padding-left: 2; + padding-top: 2; +} + +App.small #sidebar { + min-width: 16; + padding-left: 1; + padding-top: 1; +} + +#sidebar Label { + margin-right: 2; +} + +#sidebar Label:disabled { + color: #808080; +} + +App.small #sidebar Label { + margin-right: 1; +} + +#body { + background: #0f2080; + height: 100%; + border-left: round #008080; +} + +.v-scrollable { + overflow-y: scroll; +} + +#main { + height: 100%; + width: 100%; +} + +#bottom { + width: 100%; + height: 4; + background: #0f2080; + border-left: round #008080; + padding-left: 0; + dock: bottom; +} + +#buttons { + align: center top; + width: 70%; + height: 4; + background: #0f2080; +} + +#left-padding, #right-padding { + width: 15%; + background: #0f2080; + height: 4; +} + +Button { + padding-left: 2; + padding-right: 2; +} + +#btn-prev { + width: 40%; + margin-left: 2; + margin-right: 2; + margin-bottom: 1; +} + +#btn-next { + width: 40%; + margin-left: 2; + margin-right: 2; + margin-bottom: 1; +} + +.panel { + width: 100%; + padding: 2; + padding-top: 2; + min-height: 100%; + height: auto; +} + +.panel-wrapper { + width: 100%; + height: auto; + min-height: 100%; + align-horizontal: center; +} + +App.small .panel { + padding: 1; + padding-top: 1; +} + +App.large .panel { + width: 80%; + min-width: 70; +} + +.p { + padding: 2; +} + +.p-l { + padding-left: 2; +} + +.p-r { + padding-right: 2; +} + +.m-b { + margin-bottom: 1; +} + +.m-t { + margin-top: 2; +} + +.m-b-1 { + margin-bottom: 1; +} + +.m-t-1 { + margin-top: 1; +} + +.m-l { + margin-left: 2; +} + +.m-r { + margin-right: 2; +} + +.m-r-1 { + margin-right: 1; +} + +Button { + border-top: tall #a0a0a0; + background: #808080; + border-bottom: tall #606060; +} + +Button:focus { + border-top: tall #e0e0e0; + background: #a0a0a0; + border-bottom: tall #808080; +} + +Button.-primary { + border-top: tall #0060a0; + background: #004080; + border-bottom: tall #004040; +} + +Button.-primary:focus { + border-top: tall #80a0ff; + background: #0080ff; + border-bottom: tall #004080; +} + +.header { + color: white; + margin-bottom: 2; +} + +App.small .header { + margin-bottom: 1; +} + +.main-text, .main-text-nh { + color: #b0b0b0; +} + +.help-text { + margin-top: 2; + margin-bottom: 1; + margin-left: 1; + max-width: 80; + color: #b0b0b0; +} + +App.small .help-text { + margin-top: 0; + margin-bottom: 1; + margin-left: 1; +} + +.warning-text { + color: yellow; + padding-top: 1; + margin-left: 2; +} + +.h-auto { + height: auto; +} + +.w-auto { + width: auto; +} + +.w-100 { + width: 100%; +} + +.c { + background: #ff2020; +} + +.d { + background: #20ffff; +} + + +#name-input-group { + width: 100%; + height: auto; +} + +#key-email-input-group { + width: 100%; + height: 1fr; +} + +#key-input { + padding: 0; + margin: 0; + padding-left: 1; +} + +#btn-use-fqdn, #btn-request-key { + width: auto; + dock: right; +} + +.form-label { + margin-bottom: 1; + color: #ffffff; +} + +App.small .form-label { + margin-bottom: 0; +} + +#warn-no-batch { + height: auto; +} + +#batch-warn { + padding-top: 2; +} + +.form-col-3 { + width: 33%; + margin-right: 1; + margin-bottom: 1; +} + +.form-col-2 { + width: 50%; + margin-right: 2; + margin-bottom: 1; +} + +.form-col-2 Button { + width: 27; +} + +#cb-run-test-job { + min-width: 16; + width: 27; + max-width: 32; + margin-top: 1; + margin-right: 2; +} + +#custom-attrs { + background: #002040; + height: 7; + overflow-x: auto; + overflow-y: auto; + padding: 0; + border: none; + color: #c0c0c0; +} + +.form-row { + min-height: 6; + max-height: 6; + padding-bottom: 1; +} + +App.large .form-row { + margin-bottom: 1; +} + +App.small .form-row { + min-height: 5; +} + +#edit-attrs-dialog { + padding-top: 0; + width: 96%; + height: 94%; +} + +#attr-rows { + overflow-y: scroll; + height: 80%; +} + +#attr-rows .td, #attr-rows .th { + width: 30%; +} + +#attr-rows .td:last-of-type, #attr-rows .th:last-of-type { + width: 40%; +} + +#edit-attrs-dialog .attr-row { + margin-bottom: 1; +} + +.dialog { + padding-left: 1; + padding-right: 1; + padding-top: 1; +} + +.error-dialog { + border: #ff0000; + height: 14; + width: 60; +} + +.error-dialog .header { + color: #800000; +} + +#progress-dialog { + width: 50; + height: 6; +} + +#progress-dialog .header { + margin-left: 1; +} + +#progress-indicator-indeterminate { + color: #008000; + margin-top: 1; +} + +#progress-dialog #progress-indicator-indeterminate { + width: 32; + margin-left: 8; +} + +#progress-indicator-indeterminate .bar--indeterminate { + color: #ff0000; +} + +Label.-success { + color: #00ff00; +} + +Label.-error { + color: #ff0000; +} + +#key-request-dialog { + width: 70; + max-width: 80; + height: 20; +} + +#key-request-url { + margin-top: 1; + margin-left: 2; + margin-bottom: 1; + background: #000000; + color: #808080; + padding-top: 1; + padding-bottom: 1; + padding-left: 2; + padding-right: 2; +} + +#key-request-dialog #status-label { + margin-top: 1; + margin-left: 2; +} + +.form-col { + width: auto; + margin-right: 1; + padding-bottom: 1; +} + +.test-job-indicator { + width: 4; + margin-left: 1; + margin-right: 1; +} + +.job-progress-row { + height: 1; +} + +#test-job-errors { + margin-top: 1; + margin-bottom: 1; +} + +.test-job-label { + color: #ffffff; + margin-right: 2; +} + +.test-job-status { + width: 6; +} + +#test-jobs-dialog .status-succeeded { + text-style: bold; + color: #00a040; + padding-left: 2; +} + +#test-jobs-dialog .status-failed { + color: #ff0000; + margin-left: 0; + margin-right: 0; +} + +#test-jobs-dialog Button { + margin-left: 1; +} + +#method-status { + margin-top: 1; + color: #b04040; +} + +#method-preview { + border: none; + padding: 0; + background: #0f2040; + padding-left: 1; + padding-right: 1; + height: 3; +} + +#method-preview:disabled { + background-tint: 0%; + opacity: 100%; + text-opacity: 100%; +} \ No newline at end of file diff --git a/tests/installer/terminal.py b/tests/installer/terminal.py new file mode 100644 index 00000000..a9d79bda --- /dev/null +++ b/tests/installer/terminal.py @@ -0,0 +1,329 @@ +import string +import sys + +from enum import Enum +from typing import Callable + +from rich.color import Color +from rich.style import StyleType + + +def add_alt_key_processing() -> None: + from textual._ansi_sequences import ANSI_SEQUENCES_KEYS + + class MoreKeys(str, Enum): + AltA = 'alt+a' + AltB = 'alt+b' + AltC = 'alt+c' + AltD = 'alt+d' + AltE = 'alt+e' + AltF = 'alt+f' + AltG = 'alt+g' + AltH = 'alt+h' + AltI = 'alt+i' + AltJ = 'alt+j' + AltK = 'alt+k' + AltL = 'alt+l' + AltM = 'alt+m' + AltN = 'alt+n' + AltO = 'alt+o' + AltP = 'alt+p' + AltQ = 'alt+q' + AltR = 'alt+r' + AltS = 'alt+s' + AltT = 'alt+t' + AltU = 'alt+u' + AltV = 'alt+v' + AltW = 'alt+w' + AltX = 'alt+x' + AltY = 'alt+y' + AltZ = 'alt+z' + + AltShiftA = 'alt+shift+a' + AltShiftB = 'alt+shift+b' + AltShiftC = 'alt+shift+c' + AltShiftD = 'alt+shift+d' + AltShiftE = 'alt+shift+e' + AltShiftF = 'alt+shift+f' + AltShiftG = 'alt+shift+g' + AltShiftH = 'alt+shift+h' + AltShiftI = 'alt+shift+i' + AltShiftJ = 'alt+shift+j' + AltShiftK = 'alt+shift+k' + AltShiftL = 'alt+shift+l' + AltShiftM = 'alt+shift+m' + AltShiftN = 'alt+shift+n' + AltShiftO = 'alt+shift+o' + AltShiftP = 'alt+shift+p' + AltShiftQ = 'alt+shift+q' + AltShiftR = 'alt+shift+r' + AltShiftS = 'alt+shift+s' + AltShiftT = 'alt+shift+t' + AltShiftU = 'alt+shift+u' + AltShiftV = 'alt+shift+v' + AltShiftW = 'alt+shift+w' + AltShiftX = 'alt+shift+x' + AltShiftY = 'alt+shift+y' + AltShiftZ = 'alt+shift+z' + + for c in string.ascii_lowercase: + C = c.upper() + ANSI_SEQUENCES_KEYS[f'\x1b{c}'] = (MoreKeys[f'Alt{C}'],) # type: ignore + ANSI_SEQUENCES_KEYS[f'\x1b{C}'] = (MoreKeys[f'AltShift{C}'],) # type: ignore + + +def patch_borders() -> None: + EXTRA_CELLS = frozenset(['\033(0' + c + '\033(1' for c in 'lqkxmja']) + from rich import cells + prev_cell_len = cells.cell_len + + def cell_len(text: str, _cell_len: Callable[[str], int] = cells.cached_cell_len) -> int: + if text in EXTRA_CELLS: + return 1 + elif '\033' in text: + return len(text.replace('\033(0', '').replace('\033(1', '')) + else: + return prev_cell_len(text, _cell_len) + + cells.cell_len = cell_len + cells.cached_cell_len = cell_len # type: ignore + + BORDER = ( + ("\033(0l\033(1", "\033(0q\033(1", "\033(0k\033(1"), + ("\033(0x\033(1", " ", "\033(0x\033(1"), # noqa: E241 + ("\033(0m\033(1", "\033(0q\033(1", "\033(0j\033(1") + ) + + from textual import _border + + for t in ['round', 'solid', 'double', 'dashed', 'heavy', 'inner', 'outer', 'thick', 'tall', + 'panel', 'tab', 'wide']: + _border.BORDER_CHARS[t] = BORDER # type: ignore + + _border.BORDER_CHARS['tall'] = ((" ", " ", " "), (" ", " ", " "), (" ", " ", " ")) + _border.BORDER_CHARS['outer'] = ((" ", " ", " "), (" ", " ", " "), (" ", ".", ".")) + + _border.BORDER_CHARS['vkey'] = ( + ("\033(0l\033(1", " ", "\033(0k\033(1"), + ("\033(0x\033(1", " ", "\033(0x\033(1"), + ("\033(0m\033(1", " ", "\033(0j\033(1") + ) + + _border.BORDER_CHARS['hkey'] = ( + ("\033(0l\033(1", "\033(0q\033(1", "\033(0k\033(1"), + (" ", " ", " "), + ("\033(0m\033(1", "\033(0q\033(1", "\033(0j\033(1") + ) + + +def patch_toggle_button(unicode: bool) -> None: + from textual.widgets._toggle_button import ToggleButton + from textual.widgets._radio_button import RadioButton + from rich.text import Text + from rich.style import Style + + class PatchedToggleButton(ToggleButton): + @property + def _button(self) -> Text: + button_style = self.get_component_rich_style('toggle--button') + side_style = Style.from_color(self.colors[3].rich_color, + self.background_colors[1].rich_color) + return Text.assemble( + (self.BUTTON_LEFT, side_style), + (self.BUTTON_INNER, button_style), + (self.BUTTON_RIGHT, side_style) + ) + ToggleButton._button = PatchedToggleButton._button # type: ignore + RadioButton._button = PatchedToggleButton._button # type: ignore + + if not unicode: + ToggleButton.BUTTON_LEFT = '[' + ToggleButton.BUTTON_RIGHT = ']' + ToggleButton.BUTTON_INNER = 'x' + RadioButton.BUTTON_LEFT = '(' + RadioButton.BUTTON_INNER = '\033(0`\033(1' + RadioButton.BUTTON_RIGHT = ')' + + from textual.widgets import _rule + + _rule._HORIZONTAL_LINE_CHARS['solid'] = '\033(0q\033(1' + + +def patch_scrollbar(unicode: bool) -> None: + from textual.scrollbar import ScrollBarRender + from rich.segment import Segment, Segments + from rich.style import Style + + if unicode: + VERTICAL_BAR = ' ' + HORIZONTAL_BAR = ' ' + else: + VERTICAL_BAR = ' ' + HORIZONTAL_BAR = ' ' + + class SBRenderer(ScrollBarRender): + + def __init__(self, virtual_size: int = 100, window_size: int = 0, position: float = 0, + thickness: int = 1, vertical: bool = True, + style: StyleType = "bright_magenta on #555555") -> None: + super().__init__(virtual_size, window_size, position, thickness, vertical, style) + + @classmethod + def render_bar(cls, size: int = 25, virtual_size: float = 50, window_size: float = 20, + position: float = 0, thickness: int = 1, vertical: bool = True, + back_color: Color = Color.parse("#555555"), + bar_color: Color = Color.parse("bright_magenta")) -> Segments: + if vertical: + bar = VERTICAL_BAR + else: + bar = HORIZONTAL_BAR + if window_size and size and virtual_size and size != virtual_size: + thumb_size = int(window_size / virtual_size * size) + if thumb_size < 1: + thumb_size = 1 + position = (size - thumb_size) * position / (virtual_size - window_size) + top_size = int(position) + bottom_size = size - top_size - thumb_size + + upper_segment = Segment(bar, Style(bgcolor=back_color, color=bar_color, + meta={'@mouse.up': 'scroll_up'})) + lower_segment = Segment(bar, Style(bgcolor=back_color, color=bar_color, + meta={'@mouse.up': 'scroll_down'})) + thumb_segment = Segment(' ', Style(color=bar_color, reverse=True)) + segments = ([upper_segment] * top_size + [thumb_segment] * thumb_size + + [lower_segment] * bottom_size) + else: + segments = [Segment(bar, Style(bgcolor=back_color))] * size + + if vertical: + return Segments(segments, new_lines=True) + else: + return Segments(segments, new_lines=False) + + ScrollBarRender.render_bar = SBRenderer.render_bar # type: ignore + + +def patch_markdown() -> None: + from textual.widgets import Markdown + + Markdown.BULLETS = ['\033(0`\033(1', '*', '-', 'o'] + + +def patch_select() -> None: + from textual.widgets import _select + from textual.widgets import Static + + def compose(self): # type: ignore + yield Static(self.placeholder, id='label') + yield Static("_", classes='arrow down-arrow') + yield Static("^", classes='arrow up-arrow') + + _select.SelectCurrent.compose = compose # type: ignore + + +def patch_progress_bar() -> None: + from textual.app import ComposeResult, RenderResult + from textual.widgets import ProgressBar + from textual.widgets._progress_bar import Bar + + class BR: + def __init__(self, br: RenderResult) -> None: + self.br = br + + def __rich_console__(self, console, options): # type: ignore + r = next(self.br.__rich_console__(console, options)) # type: ignore + for seg in r.render(console, options): + yield seg._replace(text=''.join(['\033(0q\033(1'] * len(str(seg[0]))), style=seg[1]) + + class PatchedBar(Bar): + def render(self) -> RenderResult: + br = super().render() + return BR(br) + + def compose(self: Bar) -> ComposeResult: + yield PatchedBar(id='bar', clock=self._clock).data_bind( + ProgressBar.percentage).data_bind(ProgressBar.gradient) + + ProgressBar.compose = compose # type: ignore + + +_LASTC = '' + + +def _expect(c: str) -> None: + global _LASTC + if _LASTC != '': + r = _LASTC + _LASTC = '' + else: + r = sys.stdin.read(1) + if r != c: + raise Exception('Unexpected terminal response: %s' % r) + + +def _readnum() -> int: + global _LASTC + s = '' + while True: + c = sys.stdin.read(1) + if c.isdigit(): + s += c + else: + _LASTC = c + return int(s) + + +_TERMINAL_SUPPORTS_UNICODE = None + + +def terminal_supports_unicode() -> bool: + global _TERMINAL_SUPPORTS_UNICODE + if _TERMINAL_SUPPORTS_UNICODE is not None: + return _TERMINAL_SUPPORTS_UNICODE + try: + import termios + initial_mode = termios.tcgetattr(sys.stdin) + mode = termios.tcgetattr(sys.stdin) + mode[3] &= ~termios.ICANON & ~termios.ECHO + termios.tcsetattr(sys.stdin, termios.TCSAFLUSH, mode) + try: + # save cursor, move to col 0, print unicode char, report position, restore cursor + print('\0337\033[0G─\033[6n\0338', end='', flush=True) + _expect('\033') + _expect('[') + _ = _readnum() + _expect(';') + x = _readnum() + _expect('R') + _TERMINAL_SUPPORTS_UNICODE = x == 2 + print(' ') + return _TERMINAL_SUPPORTS_UNICODE + finally: + termios.tcsetattr(sys.stdin, termios.TCSAFLUSH, initial_mode) + except ImportError: + print('Cannot import termios') + raise + + +def patch_rich_terminals() -> None: + from rich import console + from rich.color import ColorSystem + console._TERM_COLORS['rxvt'] = ColorSystem.EIGHT_BIT + + +def run_patches() -> None: + unicode = terminal_supports_unicode() + if not unicode: + patch_borders() + patch_select() + patch_progress_bar() + patch_markdown() + + patch_scrollbar(unicode) + patch_toggle_button(unicode) + + add_alt_key_processing() + patch_rich_terminals() + + +run_patches() diff --git a/tests/installer/widgets.py b/tests/installer/widgets.py new file mode 100644 index 00000000..6bceb9e6 --- /dev/null +++ b/tests/installer/widgets.py @@ -0,0 +1,53 @@ +from textual.app import RenderResult +from typing import Optional, Any +from textual.binding import Binding +from textual.widgets import Select, Button, LoadingIndicator + + +class MSelect(Select[str]): + BINDINGS = [ + Binding('enter, space', 'show_overlay2', 'Show menu', show=False), + Binding('down', 'down', 'Bypass', show=False), + Binding('up', 'up', 'Bypass', show=False), + ] + + def action_show_overlay(self) -> None: + pass + + def action_up(self) -> None: + self.app.action_scroll_up() # type: ignore + + def action_down(self) -> None: + self.app.action_scroll_down() # type: ignore + + def action_show_overlay2(self) -> None: + super().action_show_overlay() + + +class ShortcutButton(Button): + def __init__(self, label: str, *args: Any, **kwargs: Any) -> None: + self.key: Optional[str] = None + label = self._handle_shortcut(label) + super().__init__(label, *args, **kwargs) + + def on_mount(self) -> None: + if self.key: + self.app.register_button_shortcut(self.key, self) # type: ignore + + def _handle_shortcut(self, label: str) -> str: + ix = label.index('&') + if ix == -1: + return label + self.key = label[ix + 1] + return (label[0:ix] + '[b bright_yellow]' + label[ix + 1] + + '[/b bright_yellow]' + label[ix + 2:]) + + def set_label(self, label: str) -> None: + self.label = self._handle_shortcut(label) + + +class DottedLoadingIndicator(LoadingIndicator): + def render(self) -> RenderResult: + text = super().render() + text.plain = '......' # type: ignore + return text diff --git a/tests/run_installer.py b/tests/run_installer.py new file mode 100644 index 00000000..4fc18a13 --- /dev/null +++ b/tests/run_installer.py @@ -0,0 +1,5 @@ +import conftest +import ci_runner +from installer import main + +main.run(conftest, ci_runner) From 05631d7ddf3a991fa72bba17724eba37d071c980 Mon Sep 17 00:00:00 2001 From: hategan Date: Thu, 13 Feb 2025 22:15:20 -0800 Subject: [PATCH 07/33] An undocumented feature to push manual run strings to the clipboard for terminals that might support it. --- tests/installer/main.py | 1 - tests/installer/panels/schedule_panel.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/installer/main.py b/tests/installer/main.py index 5b938b78..7501022a 100644 --- a/tests/installer/main.py +++ b/tests/installer/main.py @@ -102,7 +102,6 @@ async def on_mount(self) -> None: self.watch(body, 'scroll_y', self._on_y_scroll) self.next_button.focus() self.activate_panel(0) - self.copy_to_clipboard('Testing, 1, 2, 3') async def _on_y_scroll(self, y: int) -> None: log.write(f'on_y_scroll({y})\n') diff --git a/tests/installer/panels/schedule_panel.py b/tests/installer/panels/schedule_panel.py index 0048c579..924eff66 100644 --- a/tests/installer/panels/schedule_panel.py +++ b/tests/installer/panels/schedule_panel.py @@ -100,6 +100,8 @@ async def validate(self) -> bool: assert m is not None self.state.install_method = m m.install() + if m.name == 'custom': + self.app.copy_to_clipboard(m.preview) return True except Exception as ex: await ErrorDialog('Error scheduling tests', str(ex)).run(self.app) From 607ad5686688d04d191a6a36bdcd8e5e3a7ee2e3 Mon Sep 17 00:00:00 2001 From: hategan Date: Fri, 14 Feb 2025 10:41:54 -0800 Subject: [PATCH 08/33] Also track python version when caching packages. --- psij-ci-setup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/psij-ci-setup b/psij-ci-setup index 3355544c..b800eadc 100755 --- a/psij-ci-setup +++ b/psij-ci-setup @@ -289,7 +289,8 @@ not_cached() { return 0 fi REQ_CSUM=`cat .packages/req_csum.txt` - CRT_CSUM=`cat requirements*.txt | md5sum` + PYTHON=`which $PYTHON` + CRT_CSUM=`echo $PYTHON | cat - requirements*.txt | md5sum` if [ "$REQ_CSUM" != "$CRT_CSUM" ]; then return 0 From 839701a6a947c6a6ea7377bd8d728fc910be83f4 Mon Sep 17 00:00:00 2001 From: hategan Date: Fri, 14 Feb 2025 10:42:21 -0800 Subject: [PATCH 09/33] 3.8 compatibility --- tests/installer/dialogs.py | 7 ++++-- tests/installer/install_methods.py | 12 ++++----- tests/installer/panels/schedule_panel.py | 11 +++++---- tests/installer/state.py | 31 +++++++++++++++--------- 4 files changed, 36 insertions(+), 25 deletions(-) diff --git a/tests/installer/dialogs.py b/tests/installer/dialogs.py index d079bbe5..c5498555 100644 --- a/tests/installer/dialogs.py +++ b/tests/installer/dialogs.py @@ -1,5 +1,5 @@ import asyncio -from typing import cast +from typing import cast, TypeVar, Generic from textual import on from textual.app import App, ComposeResult @@ -11,7 +11,10 @@ from installer.widgets import DottedLoadingIndicator, ShortcutButton -class RunnableDialog[T](ModalScreen[None]): +T = TypeVar('T') + + +class RunnableDialog(ModalScreen[None], Generic[T]): def __init__(self) -> None: super().__init__() self.done = asyncio.get_running_loop().create_future() diff --git a/tests/installer/install_methods.py b/tests/installer/install_methods.py index 2da9adcb..b6cd6c61 100644 --- a/tests/installer/install_methods.py +++ b/tests/installer/install_methods.py @@ -34,7 +34,7 @@ def _save_env() -> None: class InstallMethod(ABC): @abstractmethod - def is_available(self) -> Tuple[bool, str | None]: + def is_available(self) -> Tuple[bool, Optional[str]]: pass @abstractmethod @@ -73,7 +73,7 @@ def __init__(self) -> None: minute = random.randint(0, 59) self.line = f'{minute} {hour} * * * "{cwd}/psij-ci-run" --log' - def is_available(self) -> Tuple[bool, str | None]: + def is_available(self) -> Tuple[bool, Optional[str]]: if not _succeeds("ps -eo command | awk '{print $1}' | grep cron"): return False, 'not found' if _succeeds('crontab -l 2>&1 | grep "not allowed"'): @@ -126,7 +126,7 @@ def __init__(self) -> None: self.minute = random.randint(0, 59) self.cmd = f'psij-ci-run --reschedule {self.hour}:{self.minute} --log' - def is_available(self) -> Tuple[bool, str | None]: + def is_available(self) -> Tuple[bool, Optional[str]]: fd, path = mkstemp() os.close(fd) @@ -187,7 +187,7 @@ class Screen(InstallMethod): def __init__(self) -> None: self.cmd = 'screen -d -m bash -c "./psij-ci-run --repeat --log"' - def is_available(self) -> Tuple[bool, str | None]: + def is_available(self) -> Tuple[bool, Optional[str]]: if _succeeds('which screen'): return True, None else: @@ -219,7 +219,7 @@ def help_message(self) -> str: class Custom(InstallMethod): - def is_available(self) -> Tuple[bool, str | None]: + def is_available(self) -> Tuple[bool, Optional[str]]: return True, None def already_installed(self) -> bool: @@ -255,7 +255,7 @@ def help_message(self) -> str: ] -def existing() -> InstallMethod | None: +def existing() -> Optional[InstallMethod]: for method in METHODS: if method.already_installed(): return method diff --git a/tests/installer/panels/schedule_panel.py b/tests/installer/panels/schedule_panel.py index 924eff66..61f26800 100644 --- a/tests/installer/panels/schedule_panel.py +++ b/tests/installer/panels/schedule_panel.py @@ -1,4 +1,4 @@ -from typing import cast +from typing import cast, Optional from .panel import Panel @@ -16,7 +16,8 @@ class MRadioSet(RadioSet): ('enter,space', 'select') ] - def __init__(self, panel: 'SchedulePanel', *radios: RadioButton, id: str | None = None) -> None: + def __init__(self, panel: 'SchedulePanel', *radios: RadioButton, + id: Optional[str] = None) -> None: super().__init__(*radios, id=id) self.panel = panel @@ -39,7 +40,7 @@ def action_select(self) -> None: self.action_toggle_button() - def get_selected_index(self) -> int | None: + def get_selected_index(self) -> Optional[int]: return self._selected @@ -86,7 +87,7 @@ def label(self) -> str: def name(self) -> str: return 'install' - def _get_selected_method(self) -> InstallMethod | None: + def _get_selected_method(self) -> Optional[InstallMethod]: radio_set = self.get_widget_by_id('rs-method') assert isinstance(radio_set, MRadioSet) selected_index = radio_set.get_selected_index() @@ -115,7 +116,7 @@ async def activate(self) -> None: btn.value = True break - def radio_focused(self, btn: RadioButton | None) -> None: + def radio_focused(self, btn: Optional[RadioButton]) -> None: log.write(f'focused {btn}\n') preview = self.get_widget_by_id('method-preview') assert isinstance(preview, TextArea) diff --git a/tests/installer/state.py b/tests/installer/state.py index fdcc787f..f4cf2ef5 100644 --- a/tests/installer/state.py +++ b/tests/installer/state.py @@ -1,4 +1,5 @@ import asyncio +import functools import json import os import pathlib @@ -6,7 +7,7 @@ import requests from collections import namedtuple -from typing import List, Dict, Optional, Callable, Awaitable, Tuple, cast +from typing import List, Dict, Optional, Callable, Awaitable, Tuple, cast, Union from .install_methods import InstallMethod from .log import log @@ -18,13 +19,19 @@ Attr = namedtuple('Attr', ['filter', 'name', 'value']) +async def _to_thread(func, /, *args, **kwargs): # type: ignore + loop = asyncio.get_running_loop() + func_call = functools.partial(func, *args, **kwargs) + return await loop.run_in_executor(None, func_call) + + class _Options: def __init__(self) -> None: self.custom_attributes = None class ConfWrapper: - def __init__(self, dict: Dict[str, str | int | bool | None]): + def __init__(self, dict: Dict[str, Union[str, int, bool, None]]): self.dict = dict self.option = _Options() dict['run_id'] = 'x' @@ -32,7 +39,7 @@ def __init__(self, dict: Dict[str, str | int | bool | None]): dict['upload_results'] = True dict['branch_name_override'] = None - def getoption(self, name: str) -> str | int | bool | None: + def getoption(self, name: str) -> Union[str, int, bool, None]: return self.dict[name] @@ -43,16 +50,16 @@ def __init__(self, conftest: types.ModuleType, ci_runner: types.ModuleType): log.write('Conf: ' + str(self.conf) + '\n') log.write(str(self.env) + '\n') self.disable_install = False - self.install_method: InstallMethod | None = None - self.active_panel: int | None = None - self.scheduler: str | None = None + self.install_method: Optional[InstallMethod] = None + self.active_panel: Optional[int] = None + self.scheduler: Optional[str] = None self.attrs = self._parse_attributes() self.run_test_job = True self.has_key = KEY_PATH.exists() if self.has_key: with open(KEY_PATH, 'r') as f: self.key = f.read().strip() - self.key_is_valid: bool | None = None + self.key_is_valid: Optional[bool] = None log.write(f'has key: {self.has_key}\n') def _parse_attributes(self) -> List[Attr]: @@ -89,15 +96,15 @@ def update_conf(self, name: str, value: str) -> None: self._write_conf_value(name, value) async def request(self, query: str, data: Dict[str, object], title: str, - error_cb: Callable[[str, str], Awaitable[Dict[str, object] | None]]) \ - -> Dict[str, object] | None: + error_cb: Callable[[str, str], Awaitable[Optional[Dict[str, object]]]]) \ + -> Optional[Dict[str, object]]: baseUrl = self.conf['server_url'] - response = await asyncio.to_thread(requests.post, baseUrl + query, data=data) + response = await _to_thread(requests.post, baseUrl + query, data=data) # type: ignore return await self._check_error(response, title, error_cb) async def _check_error(self, response: requests.Response, title: str, - error_cb: Callable[[str, str], Awaitable[Dict[str, object] | None]]) \ - -> Dict[str, object] | None: + error_cb: Callable[[str, str], Awaitable[Optional[Dict[str, object]]]]) \ + -> Optional[Dict[str, object]]: log.write(f'Response: {response.text}\n') if response.status_code != 200: msg = self._extract_response_message(response.text) From da52a37b194bc6fc5891e37ab146cd641dcc6211 Mon Sep 17 00:00:00 2001 From: hategan Date: Fri, 14 Feb 2025 14:25:14 -0800 Subject: [PATCH 10/33] Proper checksum generation --- psij-ci-setup | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/psij-ci-setup b/psij-ci-setup index b800eadc..bcec6a38 100755 --- a/psij-ci-setup +++ b/psij-ci-setup @@ -318,7 +318,8 @@ install_deps() { exit 2 else echo "Done" - cat requirements*.txt | md5sum >.packages/req_csum.txt + CSUM=`echo $PYTHON | cat - requirements*.txt | md5sum` + echo "$CSUM" >.packages/req_csum.txt fi fi From 13626b3e40cb73070e9d89733e6ea6afd676bc98 Mon Sep 17 00:00:00 2001 From: hategan Date: Tue, 18 Feb 2025 14:21:55 -0800 Subject: [PATCH 11/33] Added multi-node queue and fixed some styling issues --- tests/conftest.py | 5 +- .../installer/panels/batch_scheduler_panel.py | 57 ++++++++++++------- tests/installer/panels/intro_panel.py | 2 +- tests/installer/panels/schedule_panel.py | 2 +- tests/installer/style.tcss | 42 ++++++++++++-- 5 files changed, 78 insertions(+), 30 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 3e0ca9a4..417e0bbe 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -55,7 +55,7 @@ def pytest_addoption(parser): help='Maximum age, in hours, of tests to keep saved.') parser.addoption('--id', action='store', default='hostname', help='An identifier for this site to attach to tests when aggregated.') - parser.addoption('--server-url', action='store', default='https://testing.exaworks.org', + parser.addoption('--server-url', action='store', default='https://testing.psij.io', help='The base URL of the test aggregation server.') parser.addoption('--key', action='store', default='random', help='A secret to use when communicating to the aggregation server.') @@ -67,6 +67,8 @@ def pytest_addoption(parser): help='Pretend that the current git branch is this value.') parser.addoption('--queue-name', action='store', default=None, help='A queue to run the batch jobs in.') + parser.addoption('--multi-node-queue-name', action='store', default=None, + help='An optional queue to run multi-node batch jobs in.') parser.addoption('--project-name', action='store', default=None, help='A project/account name to associate the batch jobs with.') parser.addoption('--account', action='store', default=None, @@ -176,6 +178,7 @@ def pytest_generate_tests(metafunc): etps = [] for x in _get_executors((metafunc.config)): etp = ExecutorTestParams(x, queue_name=options.queue_name, + multi_node_queue_name=options.multi_node_queue_name, account=_get_account(options), custom_attributes_raw=options.custom_attributes) etps.append(etp) diff --git a/tests/installer/panels/batch_scheduler_panel.py b/tests/installer/panels/batch_scheduler_panel.py index 265c37cd..754c9a71 100644 --- a/tests/installer/panels/batch_scheduler_panel.py +++ b/tests/installer/panels/batch_scheduler_panel.py @@ -238,33 +238,39 @@ def _build_widgets(self) -> Widget: ('PBS', 'pbs'), ('LSF', 'lsf'), ('Cobalt', 'cobalt')], id='batch-selector', allow_blank=False), - classes='form-col-3 form-row' + classes='bs-col-1 form-row', id='batch-selector-col' ), + Vertical( + Label('Account/project:', classes='form-label'), + Input(id='account-input'), + classes='bs-col-2 form-row batch-valid' + ), + classes='w-100 form-row', id='batch-system-group-1' + ), + Horizontal( Vertical( Label('Queue:', classes='form-label'), Input(id='queue-input'), - classes='form-col-3 form-row batch-valid' + classes='bs-col-1 form-row batch-valid' ), Vertical( - Label('Account/project:', classes='form-label'), - Input(id='account-input'), - classes='form-col-3 form-row batch-valid' + Label('Multi-node queue:', classes='form-label'), + Input(id='mqueue-input'), + classes='bs-col-2 form-row batch-valid' ), - classes='w-100 form-row', id='batch-system-group' + Checkbox('Run [b bright_yellow]t[/b bright_yellow]est job', + id='cb-run-test-job', classes='bs-col-3 m-t-1 batch-valid'), + classes='w-100 form-row', id='batch-system-group-2' ), Horizontal( Vertical( Label('Custom attributes:', classes='form-label'), - TextArea('', id='custom-attrs', read_only=True, soft_wrap=False), - id='attr-cell', - classes='form-col-2 h-auto' + ShortcutButton('&Edit attrs.', id='btn-edit-attrs'), + classes='bs-col-1 h-auto' ), Vertical( - Label('', classes='form-label'), - ShortcutButton('&Edit attributes', id='btn-edit-attrs'), - Checkbox('Run [b bright_yellow]t[/b bright_yellow]est job', - id='cb-run-test-job', classes='m-t-1'), - classes='form-col-2 h-auto' + TextArea('', id='custom-attrs', read_only=True, soft_wrap=False), + classes='bs-col-23 h-auto' ), classes='w-100 h-auto batch-valid' ), @@ -315,20 +321,28 @@ def batch_system_selected(self) -> None: if sched == 'local': self.app._focus_next() # type: ignore else: - self.get_widget_by_id('queue-input').focus() + self.get_widget_by_id('account-input').focus() def set_scheduler(self, name: str) -> None: selector = self.get_widget_by_id('batch-selector') assert isinstance(selector, Select) selector.value = name - @on(Input.Submitted, '#queue-input') - def queue_submitted(self) -> None: - bottom = self.get_widget_by_id('account-input') - bottom.focus() - @on(Input.Submitted, '#account-input') def account_submitted(self) -> None: + next = self.get_widget_by_id('queue-input') + next.focus() + + @on(Input.Submitted, '#queue-input') + def queue_submitted(self, event: Input.Submitted) -> None: + next = self.get_widget_by_id('mqueue-input') + assert isinstance(next, Input) + if next.value == '': + next.value = event.input.value + next.focus() + + @on(Input.Submitted, '#mqueue-input') + def mqueue_submitted(self) -> None: self.app._focus_next() # type: ignore @on(Button.Pressed, '#btn-edit-attrs') @@ -368,8 +382,7 @@ async def _run_test_job(self, jd: TestJobsDialog, job_no: int, label: str, rspec: Optional[ResourceSpecV1], test_name: str) -> bool: try: jd.set_running(job_no) - await asyncio.sleep(2) - + await asyncio.sleep(0.5) job = self._launch_job(test_name, rspec) await self._wait_for_queued_state(job) diff --git a/tests/installer/panels/intro_panel.py b/tests/installer/panels/intro_panel.py index afbb5d62..a5e1f9dc 100644 --- a/tests/installer/panels/intro_panel.py +++ b/tests/installer/panels/intro_panel.py @@ -22,7 +22,7 @@ def _build_widgets(self) -> Widget: @property def label(self) -> str: - return 'Introduction' + return 'Welcome' @property def name(self) -> str: diff --git a/tests/installer/panels/schedule_panel.py b/tests/installer/panels/schedule_panel.py index 61f26800..27f5d332 100644 --- a/tests/installer/panels/schedule_panel.py +++ b/tests/installer/panels/schedule_panel.py @@ -71,7 +71,7 @@ def _build_widgets(self) -> Widget: classes='h-auto m-b-1' ), Vertical( - Label('Preview', classes='form-label'), + Label('Preview:', classes='form-label'), TextArea('-', id='method-preview', classes='', language='bash', read_only=True, disabled=True), classes='form-row h-auto' diff --git a/tests/installer/style.tcss b/tests/installer/style.tcss index a7e75183..8809b367 100644 --- a/tests/installer/style.tcss +++ b/tests/installer/style.tcss @@ -157,7 +157,8 @@ ToggleButton .toggle--button { } App.no-unicode ToggleButton .toggle--button { - background: #000060; + color: black; + background: transparent; } ToggleButton.-on .toggle--button { @@ -169,9 +170,8 @@ App.no-unicode ToggleButton.-on .toggle--button { color: #00ff00; } - ToggleButton:focus .toggle--button { - background: #0040a0; + background: #606060; } ToggleButton.-on:focus .toggle--button { @@ -276,7 +276,7 @@ LoadingIndicator { } App.small #sidebar { - min-width: 16; + min-width: 14; padding-left: 1; padding-top: 1; } @@ -522,6 +522,10 @@ App.small .form-label { margin-bottom: 0; } +#batch-selector-col { + min-width: 18; +} + #warn-no-batch { height: auto; } @@ -530,12 +534,34 @@ App.small .form-label { padding-top: 2; } +.bs-col-1 { + width: 36%; +} + +.bs-col-2 { + width: 30%; +} + +.bs-col-3 { + width: 34%; +} + +.bs-col-23 { + width: 64%; +} + .form-col-3 { width: 33%; margin-right: 1; margin-bottom: 1; } +.form-col-4 { + width: 25%; + margin-right: 0; + margin-bottom: 1; +} + .form-col-2 { width: 50%; margin-right: 2; @@ -554,14 +580,20 @@ App.small .form-label { margin-right: 2; } +#btn-edit-attrs { + margin-right: 1; + width: 100%; +} + #custom-attrs { background: #002040; - height: 7; + height: 5; overflow-x: auto; overflow-y: auto; padding: 0; border: none; color: #c0c0c0; + border-left: tall #002040; } .form-row { From 8e81e0e51fe66d2ac602a364d5e484ce0b0de0e7 Mon Sep 17 00:00:00 2001 From: hategan Date: Tue, 18 Feb 2025 15:10:24 -0800 Subject: [PATCH 12/33] Add check for minimum python version --- psij-ci-setup | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/psij-ci-setup b/psij-ci-setup index bcec6a38..62fe85a3 100755 --- a/psij-ci-setup +++ b/psij-ci-setup @@ -1,5 +1,7 @@ #!/bin/bash +MIN_PYTHON_VERSION="3.8" + set -o pipefail if pip --version 2>&1 | egrep -q 'python 3\..*' >/dev/null 2>&1 ; then @@ -280,6 +282,14 @@ check_key() { fi } +check_python_version() { + VERSION=`$PYTHON --version | awk '{print $2}'` + FIRST=`echo -e "$VERSION\n$MIN_PYTHON_VERSION" | sort -V|head -n 1` + if [ "$FIRST" == "$VERSION" ]; then + echo "Error: PSI/J requires Python $MIN_PYTHON_VERSION or above. Your current version is $VERSION." + exit 2 + fi +} not_cached() { if [ ! -d .packages ]; then @@ -342,6 +352,8 @@ while [ "$1" != "" ]; do shift done +check_python_version + MYPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" cd "$MYPATH" From 9a36e959061a483dc6f34427e36ea984122f13a1 Mon Sep 17 00:00:00 2001 From: hategan Date: Tue, 18 Feb 2025 15:20:27 -0800 Subject: [PATCH 13/33] Use multi-node queue for multi-node test --- tests/installer/panels/batch_scheduler_panel.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/installer/panels/batch_scheduler_panel.py b/tests/installer/panels/batch_scheduler_panel.py index 754c9a71..fed77bec 100644 --- a/tests/installer/panels/batch_scheduler_panel.py +++ b/tests/installer/panels/batch_scheduler_panel.py @@ -369,7 +369,7 @@ async def run_test_jobs(self) -> bool: await asyncio.sleep(0.1) j1 = await self._run_test_job(jd, 1, 'Single node job', None, '') j2 = await self._run_test_job(jd, 2, 'Multi node job ', ResourceSpecV1(node_count=4), - f'test_nodefile[{self.state.scheduler}:single') + f'test_nodefile[{self.state.scheduler}:multiple') if j1 and j2: jd.focus_continue_button() @@ -405,7 +405,10 @@ def _launch_job(self, test_name: str, rspec: Optional[ResourceSpecV1]) -> Job: attrs = JobAttributes() account = self.state.conf.get('account', '') - queue = self.state.conf.get('queue', '') + if rspec is not None and rspec.computed_node_count > 1: + queue = self.state.conf.get('multi_node_queue_name', '') + else: + queue = self.state.conf.get('queue_name', '') if account != '': attrs.account = account if queue != '': From 903ab97c3b71cb21ef7924f23d723f5c78b28efc Mon Sep 17 00:00:00 2001 From: hategan Date: Tue, 18 Feb 2025 15:42:30 -0800 Subject: [PATCH 14/33] Focus account input if a batch scheduler is auto-selected. --- tests/installer/panels/batch_scheduler_panel.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/installer/panels/batch_scheduler_panel.py b/tests/installer/panels/batch_scheduler_panel.py index fed77bec..bb6a6dfe 100644 --- a/tests/installer/panels/batch_scheduler_panel.py +++ b/tests/installer/panels/batch_scheduler_panel.py @@ -294,7 +294,14 @@ def name(self) -> str: async def activate(self) -> None: self.update_attrs() selector = self.get_widget_by_id('batch-selector') - selector.focus(False) + assert isinstance(selector, Select) + sched = selector.selection + if sched is None or sched == 'none': + selector.focus(False) + elif sched == 'local': + self.app._focus_next() # type: ignore + else: + self.get_widget_by_id('account-input').focus() def update_attrs(self) -> None: s = '' From 1a8b1ad87b54a3b3c0bb888c5f3093e04258d7ea Mon Sep 17 00:00:00 2001 From: hategan Date: Tue, 18 Feb 2025 15:44:17 -0800 Subject: [PATCH 15/33] Focus without scrolling --- tests/installer/panels/batch_scheduler_panel.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/installer/panels/batch_scheduler_panel.py b/tests/installer/panels/batch_scheduler_panel.py index bb6a6dfe..9784466a 100644 --- a/tests/installer/panels/batch_scheduler_panel.py +++ b/tests/installer/panels/batch_scheduler_panel.py @@ -301,7 +301,7 @@ async def activate(self) -> None: elif sched == 'local': self.app._focus_next() # type: ignore else: - self.get_widget_by_id('account-input').focus() + self.get_widget_by_id('account-input').focus(False) def update_attrs(self) -> None: s = '' @@ -328,7 +328,7 @@ def batch_system_selected(self) -> None: if sched == 'local': self.app._focus_next() # type: ignore else: - self.get_widget_by_id('account-input').focus() + self.get_widget_by_id('account-input').focus(False) def set_scheduler(self, name: str) -> None: selector = self.get_widget_by_id('batch-selector') @@ -338,7 +338,7 @@ def set_scheduler(self, name: str) -> None: @on(Input.Submitted, '#account-input') def account_submitted(self) -> None: next = self.get_widget_by_id('queue-input') - next.focus() + next.focus(False) @on(Input.Submitted, '#queue-input') def queue_submitted(self, event: Input.Submitted) -> None: @@ -346,7 +346,7 @@ def queue_submitted(self, event: Input.Submitted) -> None: assert isinstance(next, Input) if next.value == '': next.value = event.input.value - next.focus() + next.focus(False) @on(Input.Submitted, '#mqueue-input') def mqueue_submitted(self) -> None: From 71b63dfbafd705c68f9dad670d3439c84c2393c0 Mon Sep 17 00:00:00 2001 From: hategan Date: Tue, 18 Feb 2025 19:37:38 -0800 Subject: [PATCH 16/33] Updated executor handling and back up conf file if updating. --- .flake8 | 2 +- tests/installer/main.py | 13 ++- .../installer/panels/batch_scheduler_panel.py | 68 ++++++++---- tests/installer/panels/intro_panel.py | 1 + tests/installer/state.py | 100 ++++++++++++++++-- 5 files changed, 151 insertions(+), 33 deletions(-) diff --git a/.flake8 b/.flake8 index c24af0b3..f4e6c7ce 100644 --- a/.flake8 +++ b/.flake8 @@ -78,5 +78,5 @@ ignore = B902, D205, D400, D401, D100, W503 per-file-ignores = tests/*: D103 - tests/installer/*: D101, D102, D103, D104, D107 + tests/installer/*: D101, D102, D103, D104, D105, D107 tests/installer/main.py: E402, D101, D102, D103, D107 diff --git a/tests/installer/main.py b/tests/installer/main.py index 7501022a..4c358625 100644 --- a/tests/installer/main.py +++ b/tests/installer/main.py @@ -235,14 +235,21 @@ async def _confirm_quit(self) -> None: async def action_quit(self) -> None: install_method = self.state.install_method + messages = [] + if self.state.conf_backed_up: + messages.append('Your previous configuration was saved in "testing.conf.bk"') if install_method is not None and install_method.name == 'custom': - self.exit(message=f'Tests can be run with the following command:\n' - f'\t{install_method.preview}') + messages.append(f'Tests can be run with the following command:\n' + f'\t{install_method.preview}') + if len(messages) > 0: + self.exit(message='\n\n'.join(messages)) else: self.exit() def set_default_executor(self) -> None: - label, name = self.state.get_executor() + label, name = self.state.get_batch_executor() + if name is None: + name = 'none' batch_warner = self.get_widget_by_id('warn-no-batch') if name == 'none': diff --git a/tests/installer/panels/batch_scheduler_panel.py b/tests/installer/panels/batch_scheduler_panel.py index 9784466a..c8bbb48f 100644 --- a/tests/installer/panels/batch_scheduler_panel.py +++ b/tests/installer/panels/batch_scheduler_panel.py @@ -6,7 +6,7 @@ from .panel import Panel from ..dialogs import TestJobsDialog from ..log import log -from ..state import Attr +from ..state import Attr, State from ..widgets import MSelect, ShortcutButton from textual import on @@ -220,6 +220,10 @@ class BatchSchedulerPanel(Panel): ('alt+t', 'toggle_test_job') ] + def __init__(self, state: State) -> None: + super().__init__(state) + self._auto_scheduler: Optional[str] = None + def _build_widgets(self) -> Widget: return Vertical( Label('Select and configure a batch system.', classes='header'), @@ -258,7 +262,7 @@ def _build_widgets(self) -> Widget: Input(id='mqueue-input'), classes='bs-col-2 form-row batch-valid' ), - Checkbox('Run [b bright_yellow]t[/b bright_yellow]est job', + Checkbox('Run [b bright_yellow]t[/b bright_yellow]est job', value=False, id='cb-run-test-job', classes='bs-col-3 m-t-1 batch-valid'), classes='w-100 form-row', id='batch-system-group-2' ), @@ -278,7 +282,11 @@ def _build_widgets(self) -> Widget: ) async def validate(self) -> bool: - if self.state.run_test_job and self.state.scheduler is not None: + run_test_job = self.get_widget_by_id('cb-run-test-job') + assert isinstance(run_test_job, Checkbox) + scheduler = self._get_scheduler() + + if run_test_job.value and scheduler != 'none' and scheduler != 'local': return await self.run_test_jobs() else: return True @@ -291,14 +299,19 @@ def label(self) -> str: def name(self) -> str: return 'scheduler' - async def activate(self) -> None: - self.update_attrs() + def _get_scheduler(self) -> str: selector = self.get_widget_by_id('batch-selector') assert isinstance(selector, Select) - sched = selector.selection - if sched is None or sched == 'none': - selector.focus(False) - elif sched == 'local': + value = selector.selection + assert isinstance(value, str) + return value + + async def activate(self) -> None: + self.update_attrs() + scheduler = self._get_scheduler() + if scheduler is None or scheduler == 'none': + self.get_widget_by_id('batch-selector').focus(False) + elif scheduler == 'local': self.app._focus_next() # type: ignore else: self.get_widget_by_id('account-input').focus(False) @@ -316,40 +329,48 @@ def update_attrs(self) -> None: @on(Select.Changed, '#batch-selector') def batch_system_selected(self) -> None: - selector = cast(Select[str], self.get_widget_by_id('batch-selector')) - sched = selector.selection - disabled = (sched is None or sched == 'none' or sched == 'local') - if disabled: - self.state.scheduler = None - else: - self.state.scheduler = sched + scheduler = self._get_scheduler() + + self.state.set_batch_executor(scheduler) + + disabled = (scheduler is None or scheduler == 'none' or scheduler == 'local') for widget in self.query('.batch-valid'): widget.disabled = disabled - if sched == 'local': + run_test_job = self.get_widget_by_id('cb-run-test-job') + assert isinstance(run_test_job, Checkbox) + run_test_job.value = not disabled + if scheduler == 'local': self.app._focus_next() # type: ignore else: self.get_widget_by_id('account-input').focus(False) def set_scheduler(self, name: str) -> None: + if name == '': + name = 'none' selector = self.get_widget_by_id('batch-selector') assert isinstance(selector, Select) selector.value = name + self._auto_scheduler = name @on(Input.Submitted, '#account-input') - def account_submitted(self) -> None: + def account_submitted(self, event: Input.Submitted) -> None: + self.state.update_conf('account', event.value) next = self.get_widget_by_id('queue-input') next.focus(False) @on(Input.Submitted, '#queue-input') def queue_submitted(self, event: Input.Submitted) -> None: + self.state.update_conf('queue_name', event.value) next = self.get_widget_by_id('mqueue-input') assert isinstance(next, Input) if next.value == '': next.value = event.input.value + self.state.update_conf('multi_node_queue_name', event.value) next.focus(False) @on(Input.Submitted, '#mqueue-input') - def mqueue_submitted(self) -> None: + def mqueue_submitted(self, event: Input.Submitted) -> None: + self.state.update_conf('multi_node_queue_name', event.value) self.app._focus_next() # type: ignore @on(Button.Pressed, '#btn-edit-attrs') @@ -405,12 +426,13 @@ async def _run_test_job(self, jd: TestJobsDialog, job_no: int, label: str, return False def _launch_job(self, test_name: str, rspec: Optional[ResourceSpecV1]) -> Job: - s = self.state.scheduler - assert s is not None + scheduler = self._get_scheduler() + assert scheduler is not None - ex = JobExecutor.get_instance(s) + ex = JobExecutor.get_instance(scheduler) attrs = JobAttributes() + account = self.state.conf.get('account', '') if rspec is not None and rspec.computed_node_count > 1: queue = self.state.conf.get('multi_node_queue_name', '') @@ -423,7 +445,7 @@ def _launch_job(self, test_name: str, rspec: Optional[ResourceSpecV1]) -> Job: for attr in self.state.attrs: if re.match(attr.filter, test_name): attrs.set_custom_attribute(attr.name, attr.value) - + log.write(f'attrs: {attrs}, rspec: {rspec}\n') job = Job(JobSpec('/bin/date', attributes=attrs, resources=rspec)) ex.submit(job) log.write('Job submitted\n') diff --git a/tests/installer/panels/intro_panel.py b/tests/installer/panels/intro_panel.py index a5e1f9dc..b404d7b2 100644 --- a/tests/installer/panels/intro_panel.py +++ b/tests/installer/panels/intro_panel.py @@ -41,4 +41,5 @@ async def activate(self) -> None: if result == 'quit': self.app.exit() if result == 'update': + self.state.backup_conf() self.app.disable_install() # type: ignore diff --git a/tests/installer/state.py b/tests/installer/state.py index f4cf2ef5..b45045e3 100644 --- a/tests/installer/state.py +++ b/tests/installer/state.py @@ -3,6 +3,7 @@ import json import os import pathlib +import shutil import types import requests @@ -19,6 +20,10 @@ Attr = namedtuple('Attr', ['filter', 'name', 'value']) +_EXECUTOR_LABELS = {'slurm': 'Slurm', 'pbs': 'PBS', 'cobalt': 'Cobalt', 'lsf': 'LSF', + 'none': 'None'} + + async def _to_thread(func, /, *args, **kwargs): # type: ignore loop = asyncio.get_running_loop() func_call = functools.partial(func, *args, **kwargs) @@ -43,10 +48,33 @@ def getoption(self, name: str) -> Union[str, int, bool, None]: return self.dict[name] +class Exec: + def __init__(self, pair: str) -> None: + array = pair.split(':') + assert len(array) > 0 and len(array) <= 3 + self.name = array[0] + self.launcher = '' + self.url = '' + if len(array) > 1: + self.launcher = array[1] + if len(array) > 2: + self.url = array[2] + + def __str__(self) -> str: + if self.launcher == '': + return self.name + else: + return f'{self.name}:{self.launcher}' + + def __repr__(self) -> str: + return str(self) + + class State: def __init__(self, conftest: types.ModuleType, ci_runner: types.ModuleType): self.conf = ci_runner.read_conf('testing.conf') self.env = conftest._discover_environment(ConfWrapper(self.conf)) + self.translate_launcher = conftest._translate_launcher log.write('Conf: ' + str(self.conf) + '\n') log.write(str(self.env) + '\n') self.disable_install = False @@ -56,6 +84,9 @@ def __init__(self, conftest: types.ModuleType, ci_runner: types.ModuleType): self.attrs = self._parse_attributes() self.run_test_job = True self.has_key = KEY_PATH.exists() + self.conf_backed_up = False + self.execs, self.batch_exec = self._parse_executors() + log.write(f'execs: {self.execs}, bexec: {self.batch_exec}\n') if self.has_key: with open(KEY_PATH, 'r') as f: self.key = f.read().strip() @@ -77,6 +108,7 @@ def _parse_attributes(self) -> List[Attr]: def _write_conf_value(self, name: str, value: str) -> None: self.conf[name] = value nlen = len(name) + found = False with open('testing.conf', 'r') as old: with open('testing.conf.new', 'w') as new: for line in old: @@ -85,8 +117,11 @@ def _write_conf_value(self, name: str, value: str) -> None: new.write(line) elif sline.startswith(name) and sline[nlen] in [' ', '\t', '=']: new.write(f'{name} = {value}\n') + found = True else: new.write(line) + if not found: + new.write(line) os.rename('testing.conf.new', 'testing.conf') def update_conf(self, name: str, value: str) -> None: @@ -95,6 +130,10 @@ def update_conf(self, name: str, value: str) -> None: else: self._write_conf_value(name, value) + def backup_conf(self) -> None: + shutil.copyfile('testing.conf', 'testing.conf.bk') + self.conf_backed_up = True + async def request(self, query: str, data: Dict[str, object], title: str, error_cb: Callable[[str, str], Awaitable[Optional[Dict[str, object]]]]) \ -> Optional[Dict[str, object]]: @@ -166,14 +205,63 @@ def _write_custom_attrs(self, attrs: List[Attr]) -> None: new.write(line) os.rename('testing.conf.new', 'testing.conf') - def get_executor(self) -> Tuple[str, str]: + def _executor_label(self, name: Optional[str]) -> Optional[str]: + if name in _EXECUTOR_LABELS: + return _EXECUTOR_LABELS[name] + else: + return name + + def get_auto_executor(self) -> Optional[str]: if self.env['has_slurm']: - return ('Slurm', 'slurm') + return 'slurm' if self.env['has_pbs']: - return ('PBS', 'pbs') + return 'pbs' if self.env['has_lsf']: - return ('LSF', 'lsf') + return 'lsf' if self.env['has_cobalt']: - return ('Cobalt', 'cobalt') + return 'cobalt' + + return 'none' + + def get_batch_executor(self) -> Tuple[Optional[str], Optional[str]]: + return self._executor_label(self.batch_exec), self.batch_exec + + def _parse_executors(self) -> Tuple[List[Exec], Optional[str]]: + conf_exec = self.conf.get('executors', '') + execs = [] + for exec in conf_exec.split(','): + exec = exec.strip() + if exec != '': + execs.append(Exec(exec)) + if len(execs) == 0: + execs = [Exec('auto')] + batch = None + for exec in execs: + if exec.name == 'auto' or exec.name == 'auto_q': + batch = self.get_auto_executor() + elif exec.name not in ['local', 'batch-test']: + batch = exec.name + + return execs, batch + + def _executors_str(self) -> str: + return ', '.join([str(x) for x in self.execs]) + + def set_batch_executor(self, scheduler: str) -> None: + auto_exec = self.get_auto_executor() + if auto_exec == scheduler or scheduler == 'none': + self.execs = [Exec('auto')] + elif scheduler == 'local' and auto_exec == 'none': + self.execs = [Exec('auto')] + else: + existing = None + for exec in self.execs: + if exec.name not in ['auto', 'auto_q', 'local', 'batch-test']: + exec.name = scheduler + exec.launcher = 'auto_l' + if not existing: + self.execs.append(Exec(scheduler)) + + log.write(f'set_be({scheduler}): execs: {self.execs}, str: {self._executors_str()}\n') - return ('None', 'none') + self._write_conf_value('executors', self._executors_str()) From 3b6040e96930a37768623c732e463b1f227929a0 Mon Sep 17 00:00:00 2001 From: hategan Date: Tue, 18 Feb 2025 19:42:14 -0800 Subject: [PATCH 17/33] Updated some of the form logic --- .../installer/panels/batch_scheduler_panel.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/installer/panels/batch_scheduler_panel.py b/tests/installer/panels/batch_scheduler_panel.py index c8bbb48f..4445ec5f 100644 --- a/tests/installer/panels/batch_scheduler_panel.py +++ b/tests/installer/panels/batch_scheduler_panel.py @@ -333,12 +333,8 @@ def batch_system_selected(self) -> None: self.state.set_batch_executor(scheduler) - disabled = (scheduler is None or scheduler == 'none' or scheduler == 'local') - for widget in self.query('.batch-valid'): - widget.disabled = disabled - run_test_job = self.get_widget_by_id('cb-run-test-job') - assert isinstance(run_test_job, Checkbox) - run_test_job.value = not disabled + self._update_controls(scheduler) + if scheduler == 'local': self.app._focus_next() # type: ignore else: @@ -350,7 +346,15 @@ def set_scheduler(self, name: str) -> None: selector = self.get_widget_by_id('batch-selector') assert isinstance(selector, Select) selector.value = name - self._auto_scheduler = name + self._update_controls(name) + + def _update_controls(self, scheduler: Optional[str]) -> None: + disabled = (scheduler is None or scheduler == 'none' or scheduler == 'local') + for widget in self.query('.batch-valid'): + widget.disabled = disabled + run_test_job = self.get_widget_by_id('cb-run-test-job') + assert isinstance(run_test_job, Checkbox) + run_test_job.value = not disabled @on(Input.Submitted, '#account-input') def account_submitted(self, event: Input.Submitted) -> None: From 5cc52e61bd095ed34edda4464c74e13362a64181 Mon Sep 17 00:00:00 2001 From: hategan Date: Tue, 18 Feb 2025 19:47:35 -0800 Subject: [PATCH 18/33] Populate batch scheduler inputs from config --- tests/installer/panels/batch_scheduler_panel.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tests/installer/panels/batch_scheduler_panel.py b/tests/installer/panels/batch_scheduler_panel.py index 4445ec5f..b86824e7 100644 --- a/tests/installer/panels/batch_scheduler_panel.py +++ b/tests/installer/panels/batch_scheduler_panel.py @@ -308,6 +308,11 @@ def _get_scheduler(self) -> str: async def activate(self) -> None: self.update_attrs() + + self._set_input('account-input', self.state.conf.get('account', '')) + self._set_input('queue-input', self.state.conf.get('queue_name', '')) + self._set_input('mqueue-input', self.state.conf.get('multi_node_queue_name', '')) + scheduler = self._get_scheduler() if scheduler is None or scheduler == 'none': self.get_widget_by_id('batch-selector').focus(False) @@ -316,6 +321,13 @@ async def activate(self) -> None: else: self.get_widget_by_id('account-input').focus(False) + def _set_input(self, id: str, value: Optional[str]) -> None: + if value is None: + value = '' + input = self.get_widget_by_id(id) + assert isinstance(input, Input) + input.value = value + def update_attrs(self) -> None: s = '' for attr in self.state.attrs: From 497ad5ff8140254d8eabe00483aa1fdc670851d8 Mon Sep 17 00:00:00 2001 From: hategan Date: Tue, 18 Feb 2025 20:05:05 -0800 Subject: [PATCH 19/33] Treat focus loss the same as pressing enter on most inputs. --- tests/installer/panels/basic_info_panel.py | 6 +++--- tests/installer/panels/batch_scheduler_panel.py | 8 ++++---- tests/installer/widgets.py | 13 ++++++++++++- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/tests/installer/panels/basic_info_panel.py b/tests/installer/panels/basic_info_panel.py index 0aac38b2..aaaa97c2 100644 --- a/tests/installer/panels/basic_info_panel.py +++ b/tests/installer/panels/basic_info_panel.py @@ -4,7 +4,7 @@ from .panel import Panel from ..log import log -from ..widgets import ShortcutButton +from ..widgets import ShortcutButton, MInput from textual import on from textual.containers import Vertical, Horizontal @@ -17,8 +17,8 @@ class BasicInfoPanel(Panel): def _build_widgets(self) -> Widget: - self.name_input = Input(placeholder='Enter machine name', id='name-input') - self.email_input = Input(placeholder='Enter email', id='email-input') + self.name_input = MInput(placeholder='Enter machine name', id='name-input') + self.email_input = MInput(placeholder='Enter email', id='email-input') return Vertical( Label('Some basic information', classes='header'), Label('The name should be something descriptive, such as ' diff --git a/tests/installer/panels/batch_scheduler_panel.py b/tests/installer/panels/batch_scheduler_panel.py index b86824e7..8d95a647 100644 --- a/tests/installer/panels/batch_scheduler_panel.py +++ b/tests/installer/panels/batch_scheduler_panel.py @@ -7,7 +7,7 @@ from ..dialogs import TestJobsDialog from ..log import log from ..state import Attr, State -from ..widgets import MSelect, ShortcutButton +from ..widgets import MSelect, ShortcutButton, MInput from textual import on from textual.app import ComposeResult @@ -246,7 +246,7 @@ def _build_widgets(self) -> Widget: ), Vertical( Label('Account/project:', classes='form-label'), - Input(id='account-input'), + MInput(id='account-input'), classes='bs-col-2 form-row batch-valid' ), classes='w-100 form-row', id='batch-system-group-1' @@ -254,12 +254,12 @@ def _build_widgets(self) -> Widget: Horizontal( Vertical( Label('Queue:', classes='form-label'), - Input(id='queue-input'), + MInput(id='queue-input'), classes='bs-col-1 form-row batch-valid' ), Vertical( Label('Multi-node queue:', classes='form-label'), - Input(id='mqueue-input'), + MInput(id='mqueue-input'), classes='bs-col-2 form-row batch-valid' ), Checkbox('Run [b bright_yellow]t[/b bright_yellow]est job', value=False, diff --git a/tests/installer/widgets.py b/tests/installer/widgets.py index 6bceb9e6..8573e84d 100644 --- a/tests/installer/widgets.py +++ b/tests/installer/widgets.py @@ -1,7 +1,8 @@ from textual.app import RenderResult from typing import Optional, Any from textual.binding import Binding -from textual.widgets import Select, Button, LoadingIndicator +from textual.events import Blur +from textual.widgets import Select, Button, LoadingIndicator, Input class MSelect(Select[str]): @@ -51,3 +52,13 @@ def render(self) -> RenderResult: text = super().render() text.plain = '......' # type: ignore return text + + +class MInput(Input): + # A version of input that also runs the submit action on blur (seems silly from a UI + # perspective to allow moving the focus from this input without the value being + # committed to whatever model is underneath. + + async def _on_blur(self, event: Blur) -> None: # type: ignore + super()._on_blur(event) + await self.action_submit() From 74e204bf767a38ba7c998b3a98fc621db0327dc8 Mon Sep 17 00:00:00 2001 From: hategan Date: Tue, 18 Feb 2025 20:19:06 -0800 Subject: [PATCH 20/33] Almost. We don't want a submit when switching to another window, so only do it when TAB is pressed. --- tests/installer/widgets.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/installer/widgets.py b/tests/installer/widgets.py index 8573e84d..5617a0fb 100644 --- a/tests/installer/widgets.py +++ b/tests/installer/widgets.py @@ -55,10 +55,12 @@ def render(self) -> RenderResult: class MInput(Input): - # A version of input that also runs the submit action on blur (seems silly from a UI - # perspective to allow moving the focus from this input without the value being + # A version of input that also runs the submit action when the tab button is pressed (seems + # silly from a UI perspective to allow moving the focus from this input without the value being # committed to whatever model is underneath. - async def _on_blur(self, event: Blur) -> None: # type: ignore - super()._on_blur(event) + BINDINGS = [('tab', 'tab_pressed')] + + + async def action_tab_pressed(self) -> None: await self.action_submit() From 189ef242a447a1a8eef5d433cb0f86a2d0e14325 Mon Sep 17 00:00:00 2001 From: hategan Date: Tue, 18 Feb 2025 20:32:47 -0800 Subject: [PATCH 21/33] Revert to normal tab processing and commit when validating panel. --- tests/installer/panels/basic_info_panel.py | 6 +++--- tests/installer/panels/batch_scheduler_panel.py | 17 +++++++++++++---- tests/installer/widgets.py | 12 ------------ 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/tests/installer/panels/basic_info_panel.py b/tests/installer/panels/basic_info_panel.py index aaaa97c2..0aac38b2 100644 --- a/tests/installer/panels/basic_info_panel.py +++ b/tests/installer/panels/basic_info_panel.py @@ -4,7 +4,7 @@ from .panel import Panel from ..log import log -from ..widgets import ShortcutButton, MInput +from ..widgets import ShortcutButton from textual import on from textual.containers import Vertical, Horizontal @@ -17,8 +17,8 @@ class BasicInfoPanel(Panel): def _build_widgets(self) -> Widget: - self.name_input = MInput(placeholder='Enter machine name', id='name-input') - self.email_input = MInput(placeholder='Enter email', id='email-input') + self.name_input = Input(placeholder='Enter machine name', id='name-input') + self.email_input = Input(placeholder='Enter email', id='email-input') return Vertical( Label('Some basic information', classes='header'), Label('The name should be something descriptive, such as ' diff --git a/tests/installer/panels/batch_scheduler_panel.py b/tests/installer/panels/batch_scheduler_panel.py index 8d95a647..0bab693c 100644 --- a/tests/installer/panels/batch_scheduler_panel.py +++ b/tests/installer/panels/batch_scheduler_panel.py @@ -7,7 +7,7 @@ from ..dialogs import TestJobsDialog from ..log import log from ..state import Attr, State -from ..widgets import MSelect, ShortcutButton, MInput +from ..widgets import MSelect, ShortcutButton from textual import on from textual.app import ComposeResult @@ -246,7 +246,7 @@ def _build_widgets(self) -> Widget: ), Vertical( Label('Account/project:', classes='form-label'), - MInput(id='account-input'), + Input(id='account-input'), classes='bs-col-2 form-row batch-valid' ), classes='w-100 form-row', id='batch-system-group-1' @@ -254,12 +254,12 @@ def _build_widgets(self) -> Widget: Horizontal( Vertical( Label('Queue:', classes='form-label'), - MInput(id='queue-input'), + Input(id='queue-input'), classes='bs-col-1 form-row batch-valid' ), Vertical( Label('Multi-node queue:', classes='form-label'), - MInput(id='mqueue-input'), + Input(id='mqueue-input'), classes='bs-col-2 form-row batch-valid' ), Checkbox('Run [b bright_yellow]t[/b bright_yellow]est job', value=False, @@ -286,11 +286,20 @@ async def validate(self) -> bool: assert isinstance(run_test_job, Checkbox) scheduler = self._get_scheduler() + self.state.update_conf('account', self._get_input('account-input')) + self.state.update_conf('queue_name', self._get_input('queue-input')) + self.state.update_conf('multi_node_queue_name', self._get_input('mqueue-input')) + if run_test_job.value and scheduler != 'none' and scheduler != 'local': return await self.run_test_jobs() else: return True + def _get_input(self, id: str) -> str: + input = self.get_widget_by_id(id) + assert isinstance(input, Input) + return input.value + @property def label(self) -> str: return 'Scheduler' diff --git a/tests/installer/widgets.py b/tests/installer/widgets.py index 5617a0fb..82b90243 100644 --- a/tests/installer/widgets.py +++ b/tests/installer/widgets.py @@ -52,15 +52,3 @@ def render(self) -> RenderResult: text = super().render() text.plain = '......' # type: ignore return text - - -class MInput(Input): - # A version of input that also runs the submit action when the tab button is pressed (seems - # silly from a UI perspective to allow moving the focus from this input without the value being - # committed to whatever model is underneath. - - BINDINGS = [('tab', 'tab_pressed')] - - - async def action_tab_pressed(self) -> None: - await self.action_submit() From 9b877736a6237fa0513a65508cface1f3017c420 Mon Sep 17 00:00:00 2001 From: hategan Date: Tue, 18 Feb 2025 20:33:22 -0800 Subject: [PATCH 22/33] Fixed checkbox css on utf terminals --- tests/installer/style.tcss | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/installer/style.tcss b/tests/installer/style.tcss index 8809b367..4e62691e 100644 --- a/tests/installer/style.tcss +++ b/tests/installer/style.tcss @@ -162,8 +162,8 @@ App.no-unicode ToggleButton .toggle--button { } ToggleButton.-on .toggle--button { - color: #000060; - background: #000060; + color: #00ff00; + background: #606060; } App.no-unicode ToggleButton.-on .toggle--button { @@ -175,8 +175,8 @@ ToggleButton:focus .toggle--button { } ToggleButton.-on:focus .toggle--button { - background: #0040a0; - color: #0040a0; + background: #606060; + color: #00ff00; } ToggleButton:focus .toggle--label { From bba2f17ad7fda4a840d0de076dfe4ed7ccaff5b6 Mon Sep 17 00:00:00 2001 From: hategan Date: Tue, 18 Feb 2025 22:12:32 -0800 Subject: [PATCH 23/33] Small fixes --- tests/installer/main.py | 4 +-- .../installer/panels/batch_scheduler_panel.py | 9 ++--- tests/installer/style.tcss | 36 ++++++++++++++++--- tests/installer/widgets.py | 3 +- 4 files changed, 36 insertions(+), 16 deletions(-) diff --git a/tests/installer/main.py b/tests/installer/main.py index 4c358625..25bf38d3 100644 --- a/tests/installer/main.py +++ b/tests/installer/main.py @@ -253,9 +253,9 @@ def set_default_executor(self) -> None: batch_warner = self.get_widget_by_id('warn-no-batch') if name == 'none': - batch_warner.visible = True + batch_warner.remove_class('hidden') else: - batch_warner.visible = False + batch_warner.add_class('hidden') self.scheduler_panel.set_scheduler(name) diff --git a/tests/installer/panels/batch_scheduler_panel.py b/tests/installer/panels/batch_scheduler_panel.py index 0bab693c..af3b948f 100644 --- a/tests/installer/panels/batch_scheduler_panel.py +++ b/tests/installer/panels/batch_scheduler_panel.py @@ -227,13 +227,6 @@ def __init__(self, state: State) -> None: def _build_widgets(self) -> Widget: return Vertical( Label('Select and configure a batch system.', classes='header'), - Vertical( - Label('Your system does not appear to have a batch scheduler. If you are certain ' - 'that this is wrong, you can select one below. If not, tests will be run ' - 'using non-batch executors.', classes='help-text media-large', - shrink=True, expand=True), - id='warn-no-batch' - ), Horizontal( Vertical( Label('Batch system:', classes='form-label'), @@ -249,6 +242,8 @@ def _build_widgets(self) -> Widget: Input(id='account-input'), classes='bs-col-2 form-row batch-valid' ), + Label('No batch scheduler detected.', classes='help-text', + id='warn-no-batch', shrink=True, expand=True), classes='w-100 form-row', id='batch-system-group-1' ), Horizontal( diff --git a/tests/installer/style.tcss b/tests/installer/style.tcss index 4e62691e..168c0f94 100644 --- a/tests/installer/style.tcss +++ b/tests/installer/style.tcss @@ -213,14 +213,14 @@ LoadingIndicator { .dialog Button { border-top: tall #80a0a0; - background: #408080; + background: #004040; border-bottom: tall #204040; } .dialog Button:focus { border-top: tall #a0ffff; background: #60a0a0; - border-top: tall #408080; + border-bottom: tall #408080; } .dialog Button.-error { @@ -433,11 +433,23 @@ Button.-primary { } Button.-primary:focus { - border-top: tall #80a0ff; + border-top: tall #a0ffff; background: #0080ff; border-bottom: tall #004080; } +Button.-error { + border-top: tall #800000; + background: #ff8080; + border-bottom: tall #000000; +} + +Button.-error:focus { + border-top: tall #ffa0a0; + background: #ff8080; + border-bottom: tall #800000; +} + .header { color: white; margin-bottom: 2; @@ -527,7 +539,17 @@ App.small .form-label { } #warn-no-batch { - height: auto; + height: 100%; + color: yellow; + width: 30%; + align: left middle; + margin-top: 2; + margin-left: 1; +} + +App.small #warn-no-batch { + margin-top: 1; + margin-left: 1; } #batch-warn { @@ -576,10 +598,14 @@ App.small .form-label { min-width: 16; width: 27; max-width: 32; - margin-top: 1; + margin-top: 2; margin-right: 2; } +App.small #cb-run-test-job { + margin-top: 1; +} + #btn-edit-attrs { margin-right: 1; width: 100%; diff --git a/tests/installer/widgets.py b/tests/installer/widgets.py index 82b90243..6bceb9e6 100644 --- a/tests/installer/widgets.py +++ b/tests/installer/widgets.py @@ -1,8 +1,7 @@ from textual.app import RenderResult from typing import Optional, Any from textual.binding import Binding -from textual.events import Blur -from textual.widgets import Select, Button, LoadingIndicator, Input +from textual.widgets import Select, Button, LoadingIndicator class MSelect(Select[str]): From dc4361dad833ba8b31f7d26fad9ba3752c90eefb Mon Sep 17 00:00:00 2001 From: hategan Date: Wed, 19 Feb 2025 13:11:04 -0800 Subject: [PATCH 24/33] Added warning when pip comes from a different version of Python. --- psij-ci-setup | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/psij-ci-setup b/psij-ci-setup index 62fe85a3..3146507c 100755 --- a/psij-ci-setup +++ b/psij-ci-setup @@ -291,6 +291,16 @@ check_python_version() { fi } +check_pip_version() { + PYTHON_VERSION=`$PYTHON --version | awk '{print $2}' | cut -d '.' -f 1,2` + PIP_PYTHON_VERSION=`$PIP --version | sed -nE 's/.*\(python\s(.*)\)/\1/p'` + + if [ "$PYTHON_VERSION" != "$PIP_PYTHON_VERSION" ]; then + echo -e "\e[33mWarning: The installed $PIP comes from Python $PIP_PYTHON_VERSION while your" + echo -e "Python interpreter has version $PYTHON_VERSION. This is likely to cause problems.\e[0m" + fi +} + not_cached() { if [ ! -d .packages ]; then return 0 @@ -353,6 +363,7 @@ while [ "$1" != "" ]; do done check_python_version +check_pip_version MYPATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" cd "$MYPATH" From f6199a033f1d1bf54c9065872694cb888245ad5b Mon Sep 17 00:00:00 2001 From: hategan Date: Wed, 19 Feb 2025 13:13:50 -0800 Subject: [PATCH 25/33] Don't just complain. Offer a possible solution. --- psij-ci-setup | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/psij-ci-setup b/psij-ci-setup index 3146507c..9f02ef33 100755 --- a/psij-ci-setup +++ b/psij-ci-setup @@ -296,8 +296,9 @@ check_pip_version() { PIP_PYTHON_VERSION=`$PIP --version | sed -nE 's/.*\(python\s(.*)\)/\1/p'` if [ "$PYTHON_VERSION" != "$PIP_PYTHON_VERSION" ]; then - echo -e "\e[33mWarning: The installed $PIP comes from Python $PIP_PYTHON_VERSION while your" - echo -e "Python interpreter has version $PYTHON_VERSION. This is likely to cause problems.\e[0m" + echo -e "\e[33mWarning: The installed $PIP comes from Python $PIP_PYTHON_VERSION while your Python" + echo -e "interpreter has version $PYTHON_VERSION. If this causes problems, please use a " + echo -e "virtual environment.\e[0m" fi } From af0a048f48b9356440fb3c960b3933c6f9bf28f9 Mon Sep 17 00:00:00 2001 From: hategan Date: Wed, 19 Feb 2025 13:50:20 -0800 Subject: [PATCH 26/33] Escape open square brackets since textual/rich can get confused. --- tests/installer/dialogs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/installer/dialogs.py b/tests/installer/dialogs.py index c5498555..539c9d6b 100644 --- a/tests/installer/dialogs.py +++ b/tests/installer/dialogs.py @@ -128,7 +128,7 @@ def compose(self) -> ComposeResult: Label('Running test jobs', classes='header', shrink=True, expand=True), Horizontal( Label('Single node job', id='label-job-1', classes='test-job-label'), - Label('[ ', classes='test-job-marker'), + Label(r'\[ ', classes='test-job-marker'), DottedLoadingIndicator(id='indicator-job-1', classes='test-job-indicator hidden'), Label('', id='status-job-1', classes='test-job-status'), Label(' ]', classes='test-job-marker'), @@ -136,7 +136,7 @@ def compose(self) -> ComposeResult: ), Horizontal( Label('Multi node job ', id='label-job-2', classes='test-job-label'), - Label('[ ', classes='test-job-marker'), + Label(r'\[ ', classes='test-job-marker'), DottedLoadingIndicator(id='indicator-job-2', classes='test-job-indicator hidden'), Label('', id='status-job-2', classes='test-job-status'), Label(' ]', classes='test-job-marker'), From 76ce47f974e08d959f96868b8ca61752cc514f69 Mon Sep 17 00:00:00 2001 From: hategan Date: Wed, 19 Feb 2025 15:13:33 -0800 Subject: [PATCH 27/33] Fix improperly aligned status labels on large screens. --- tests/installer/style.tcss | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/installer/style.tcss b/tests/installer/style.tcss index 168c0f94..489bf5da 100644 --- a/tests/installer/style.tcss +++ b/tests/installer/style.tcss @@ -776,10 +776,14 @@ Label.-error { } #method-status { - margin-top: 1; + margin-top: 2; color: #b04040; } +App.small #method-status { + margin-top: 1; +} + #method-preview { border: none; padding: 0; From 295cdd71a182ed4e461f195b35748b2914b5a5d8 Mon Sep 17 00:00:00 2001 From: hategan Date: Thu, 20 Feb 2025 10:28:18 -0800 Subject: [PATCH 28/33] Added --tee option to psij-ci-run (log AND print output) --- psij-ci-run | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/psij-ci-run b/psij-ci-run index f9b9b4af..9ed82d37 100755 --- a/psij-ci-run +++ b/psij-ci-run @@ -13,6 +13,9 @@ Usage: ./psij-ci-run [--help] [--repeat] [--reschedule HH:MM] [--log] --log Disables output and redirects to a log instead. Used when running the tests automatically. + --tee + Logs output as if --log was specified, but also prints + on stdout. EOF } @@ -34,6 +37,7 @@ fi REPEAT=0 RESCHEDULE=0 NO_OUT=0 +TEE=0 while [ "$1" != "" ]; do case "$1" in @@ -54,6 +58,10 @@ while [ "$1" != "" ]; do NO_OUT=1 shift ;; + --tee) + TEE=1 + shift + ;; *) echo "Unrecognized command line option: $1." usage @@ -62,12 +70,25 @@ while [ "$1" != "" ]; do esac done +tee() { + while read LINE; do + echo "$LINE" >&3 + echo "$LINE" >>./testing.log + done +} + if [ "$NO_OUT" == "1" ]; then exec 1<&- exec 2<&- exec 1>>./testing.log exec 2>&1 +elif [ "$TEE" == "1" ]; then + exec 3>&1- + exec 2<&- + + exec 1> >(tee) + exec 2>&1 fi if python --version 2>&1 | egrep -q 'Python 3\..*' >/dev/null 2>&1 ; then From e3975706ac5895fc3e44d3b776820c985cd2f234 Mon Sep 17 00:00:00 2001 From: hategan Date: Thu, 20 Feb 2025 10:28:45 -0800 Subject: [PATCH 29/33] Added tmux as a run method --- tests/installer/install_methods.py | 40 ++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/tests/installer/install_methods.py b/tests/installer/install_methods.py index b6cd6c61..f5f99820 100644 --- a/tests/installer/install_methods.py +++ b/tests/installer/install_methods.py @@ -185,7 +185,7 @@ def help_message(self) -> str: class Screen(InstallMethod): def __init__(self) -> None: - self.cmd = 'screen -d -m bash -c "./psij-ci-run --repeat --log"' + self.cmd = 'screen -d -m bash -c "./psij-ci-run --repeat --tee"' def is_available(self) -> Tuple[bool, Optional[str]]: if _succeeds('which screen'): @@ -218,6 +218,41 @@ def help_message(self) -> str: 'reboots.') +class Tmux(InstallMethod): + def __init__(self) -> None: + self.cmd = 'tmux new -d bash -c "./psij-ci-run --repeat --tee"' + + def is_available(self) -> Tuple[bool, Optional[str]]: + if _succeeds('which tmux'): + return True, None + else: + return False, 'not found' + + def already_installed(self) -> bool: + ec, out = _run('tmux list-sessions -F "#{pane_start_command}" | grep psij-ci-run') + return ec == 0 + + def install(self) -> None: + _must_succeed(self.cmd) + + @property + def preview(self) -> str: + return self.cmd + + @property + def name(self) -> str: + return 'tmux' + + @property + def label(self) -> str: + return 'tmux - another terminal multiplexer' + + @property + def help_message(self) -> str: + return ('Uses tmux to run tests in a tmux pane. Does not persist across ' + 'reboots.') + + class Custom(InstallMethod): def is_available(self) -> Tuple[bool, Optional[str]]: return True, None @@ -231,7 +266,7 @@ def install(self) -> None: @property def preview(self) -> str: cwd = os.getcwd() - return f'"{cwd}/psij-ci-run" --log' + return f'"{cwd}/psij-ci-run" --log --repeat' @property def name(self) -> str: @@ -251,6 +286,7 @@ def help_message(self) -> str: Crontab(), At(), Screen(), + Tmux(), Custom() ] From 10aa3ff9adbe8d1bda931ddaac6adb0dd90f3b28 Mon Sep 17 00:00:00 2001 From: hategan Date: Thu, 20 Feb 2025 17:24:12 -0800 Subject: [PATCH 30/33] Made thigs a bit more clear --- tests/installer/dialogs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/installer/dialogs.py b/tests/installer/dialogs.py index 539c9d6b..75668105 100644 --- a/tests/installer/dialogs.py +++ b/tests/installer/dialogs.py @@ -286,7 +286,7 @@ def compose(self) -> ComposeResult: assert self.method is not None yield Vertical( Label('Existing installation detected', classes='header'), - Label(f'An existing {self.method} installation of the tests was detected. Continuing ' + Label(f'An existing "{self.method}" installation of the tests was detected. Continuing ' 'will update the settings used by the existing installation.', classes='main-text', shrink=True, expand=True), From 869f2c2956b9d7abea5db04182da7ab0354076ff Mon Sep 17 00:00:00 2001 From: hategan Date: Thu, 20 Feb 2025 17:43:33 -0800 Subject: [PATCH 31/33] Fixed at detection leaving stragglers when warnings are printed by at before the job number --- tests/installer/install_methods.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/installer/install_methods.py b/tests/installer/install_methods.py index f5f99820..e7e5a7ae 100644 --- a/tests/installer/install_methods.py +++ b/tests/installer/install_methods.py @@ -1,4 +1,5 @@ import os +import re import subprocess import time from abc import ABC, abstractmethod @@ -135,11 +136,13 @@ def is_available(self) -> Tuple[bool, Optional[str]]: if ec != 0: return False, 'not found' time.sleep(0.2) - if out.startswith('job'): - job_no = out.split()[1] - _run(f'atrm {job_no}') - if 'No atd' in out: - return False, 'not running' + match = re.search('job ([0-9]+)', out) + if not match: + return False, 'error' + job_no = match.group(1) + _run(f'atrm {job_no}') + if 'No atd' in out: + return False, 'not running' if os.path.exists(path): os.remove(path) return False, 'unknown error' From b7ec0ff7af6bc88239884dc1a53e40b861c4b6c9 Mon Sep 17 00:00:00 2001 From: hategan Date: Mon, 12 May 2025 17:02:38 -0700 Subject: [PATCH 32/33] Fixing textual at 3.2.0 --- requirements-tests.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-tests.txt b/requirements-tests.txt index b275757d..dc2b5262 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -8,6 +8,6 @@ requests >= 2.25.1 pytest-cov pytest-timeout filelock >= 3.4, < 3.19 -textual +textual==3.2.0 tree-sitter tree-sitter-bash From eacc6250bc5ea6e50c08d3cf852d6aeeb03683c9 Mon Sep 17 00:00:00 2001 From: hategan Date: Mon, 12 May 2025 17:03:01 -0700 Subject: [PATCH 33/33] Updated ToggleButton patch for 3.2.0 --- tests/installer/terminal.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/installer/terminal.py b/tests/installer/terminal.py index a9d79bda..1ff7a7ca 100644 --- a/tests/installer/terminal.py +++ b/tests/installer/terminal.py @@ -119,16 +119,16 @@ def cell_len(text: str, _cell_len: Callable[[str], int] = cells.cached_cell_len) def patch_toggle_button(unicode: bool) -> None: from textual.widgets._toggle_button import ToggleButton from textual.widgets._radio_button import RadioButton - from rich.text import Text - from rich.style import Style + from textual.content import Content + from textual.style import Style class PatchedToggleButton(ToggleButton): @property - def _button(self) -> Text: - button_style = self.get_component_rich_style('toggle--button') - side_style = Style.from_color(self.colors[3].rich_color, - self.background_colors[1].rich_color) - return Text.assemble( + def _button(self) -> Content: + button_style = self.get_visual_style("toggle--button") + side_style = Style(self.colors[3], + self.background_colors[1]) + return Content.assemble( (self.BUTTON_LEFT, side_style), (self.BUTTON_INNER, button_style), (self.BUTTON_RIGHT, side_style)