From 65c58a4d7c50ffaf730fe5138ca194ccf6b432a0 Mon Sep 17 00:00:00 2001 From: void <> Date: Sun, 2 Jun 2024 19:32:11 +0000 Subject: [PATCH] (shmif) wire in ACCESSIBLITY defimpl hooks and allocate a primary slot This follows the pattern of the push-DEBUG segment to automatically build an internally managed attached segment for providing some kind of support should the client itself not provide an implementation. The current form simply maps it, converts into TUI and writes back that there is no 'alt' text. The next logical step is to use it and shmifsrv to spawn an afsrv_encode process latched to related-source window video buffer submission and use that with the tesseract_ocr mode or with a future LLM one to provide content description. FossilOrigin-Name: c01fceef633c2e616c2561ea2497bd57d20ebc66f0f005bc5e77bcb178957dc3 --- CHANGELOG.md | 2 + src/shmif/CMakeLists.txt | 1 + src/shmif/arcan_shmif_a11y.c | 92 +++++++++ src/shmif/arcan_shmif_control.c | 184 ++++-------------- src/shmif/arcan_shmif_control.h | 23 ++- src/shmif/arcan_shmif_debugif.c | 2 +- ...{arcan_shmif_debugif.h => shmif_defimpl.h} | 2 + src/shmif/shmif_privint.h | 150 ++++++++++++++ src/tools/adbginject/src/adbinject.c | 2 +- src/wayland/xwlwm/xwlwm.c | 2 +- 10 files changed, 303 insertions(+), 157 deletions(-) create mode 100644 src/shmif/arcan_shmif_a11y.c rename src/shmif/{arcan_shmif_debugif.h => shmif_defimpl.h} (84%) create mode 100644 src/shmif/shmif_privint.h diff --git a/CHANGELOG.md b/CHANGELOG.md index a3316d189..51a86974f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ ## Shmif * add interop helper for arcan\_shmif\_bchunk\_resolve to help translate fd-local path + * add a \_primary slot for ACCESSIBILITY + * add an internal-catch segment for ACCESSIBILITY ## Net * add -c file for assigning lua scriptable command-line overrides diff --git a/src/shmif/CMakeLists.txt b/src/shmif/CMakeLists.txt index cdcdf0830..97f9ad61e 100644 --- a/src/shmif/CMakeLists.txt +++ b/src/shmif/CMakeLists.txt @@ -56,6 +56,7 @@ set (SHMIF_SOURCES ${ASD}/shmif/arcan_shmif_control.c ${ASD}/shmif/arcan_shmif_sub.c ${ASD}/shmif/arcan_shmif_evpack.c + ${ASD}/shmif/arcan_shmif_a11y.c ${ASD}/engine/arcan_trace.c ${ASD}/shmif/platform/exec.c ) diff --git a/src/shmif/arcan_shmif_a11y.c b/src/shmif/arcan_shmif_a11y.c new file mode 100644 index 000000000..8c9560206 --- /dev/null +++ b/src/shmif/arcan_shmif_a11y.c @@ -0,0 +1,92 @@ +#include "arcan_shmif.h" +#include "arcan_shmif_interop.h" +#include "shmif_defimpl.h" +#include "shmif_privint.h" +#include + +#define ARCAN_TUI_DYNAMIC +#include "arcan_tui.h" + +struct a11y_meta { + struct tui_context* tui; +}; + +static void on_state(struct arcan_shmif_cont* p, int state) +{ + struct a11y_meta* M = p->priv->support_window_hook_data; + if (state == SUPPORT_EVENT_VSIGNAL){ +/* this is where we'd forward the video buffer to an external oracle + * and synch the output */ + } + else if (state == SUPPORT_EVENT_POLL){ + arcan_tui_process(&M->tui, 1, NULL, 0, 0); + } + else if (state == SUPPORT_EVENT_EXIT){ +/* could forward last_words as well, but it's likely the outer-wm would + * have this as part of a notification / window handling system that is + * accessible already so we don't need to do that here */ + p->priv->support_window_hook = NULL; + arcan_tui_destroy(M->tui, NULL); + free(M); + } +} + +bool arcan_shmif_a11yint_spawn( + struct arcan_shmif_cont* c, struct arcan_shmif_cont* p) +{ +/* + * only one active per parent + */ + if (p->priv->support_window_hook) + return false; + +/* regular thing to handle dynamic injection / loading */ + if (!arcan_tui_setup){ + void* openh = dlopen( +"libarcan_tui." +#ifndef __APPLE__ + "so" +#else + "dylib" +#endif + , RTLD_LAZY); + if (!arcan_tui_dynload(dlsym, openh)) + return false; + } + +/* + * there are a few possible options here. + * + * Most important is to mark that the client doesn't have an accessibility + * implemetation. This can be done as a simple message on the subject. + * + * The other is to hand over to a receiving oracle. With whisper support + * in afsrv_encode, we can latch the [c] segment and provide frame-hooks + * that routes the image data onwards to afsrv_encode, convert and attach + * the output into either tesseract or whisper or both. + * + * Yet another is to look for the symbols used by at-spi or accesskit and + * hook ourselves there, retrieve the tree and navigate it through this + * segment. + */ + + struct a11y_meta* a11y = malloc(sizeof(struct a11y_meta)); + *a11y = (struct a11y_meta){ + .tui = arcan_tui_setup(c, + NULL, &(struct tui_cbcfg){.tag = p}, sizeof(struct tui_cbcfg)) + }; + +/* set some basic 'whatever' */ + arcan_tui_wndhint(a11y->tui, NULL, + (struct tui_constraints){ + .min_rows = 2, .max_rows = 20, + .min_cols = 64, .max_cols = 240} + ); + arcan_tui_printf(a11y->tui, NULL, "no accesibility information available"); + arcan_tui_refresh(a11y->tui); + + p->priv->support_window_hook = on_state; + p->priv->support_window_hook_data = a11y; + + return true; +} diff --git a/src/shmif/arcan_shmif_control.c b/src/shmif/arcan_shmif_control.c index 38750009d..71f29e623 100644 --- a/src/shmif/arcan_shmif_control.c +++ b/src/shmif/arcan_shmif_control.c @@ -29,6 +29,9 @@ #include "arcan_shmif.h" #include "shmif_privext.h" +#include "shmif_privint.h" + +#include "shmif_defimpl.h" #include "platform/shmif_platform.h" #include @@ -57,17 +60,6 @@ enum debug_level { DETAILED = 2 }; -struct mstate { - union { - struct { - int32_t ax, ay, lx, ly; - uint8_t rel : 1; - uint8_t inrel : 1; - }; - uint8_t state[ASHMIF_MSTATE_SZ]; - }; -}; - _Static_assert(sizeof(struct mstate) == ASHMIF_MSTATE_SZ, "invalid mstate sz"); /* @@ -156,141 +148,12 @@ static ssize_t a12_cp(const char* conn_src, bool* weak); * (or forcibly exit if a FATALFAIL behavior has been set) and event triggered * functions will fail. */ -struct shmif_hidden { - struct arg_arr* args; - char* last_words; - - shmif_trigger_hook_fptr video_hook; - void* video_hook_data; - uint8_t vbuf_ind, vbuf_cnt; - bool vbuf_nbuf_active; - uint64_t vframe_id; - shmif_pixel* vbuf[ARCAN_SHMIF_VBUFC_LIM]; - - shmif_trigger_hook_fptr audio_hook; - void* audio_hook_data; - uint8_t abuf_ind, abuf_cnt; - shmif_asample* abuf[ARCAN_SHMIF_ABUFC_LIM]; - - shmif_reset_hook_fptr reset_hook; - void* reset_hook_tag; - -/* Initial contents gets dropped after first valid !initial call after open, - * otherwise we will be left with file descriptors in the process that might - * not get used. What argues against keeping them after is that we also need - * to track the descriptor carrying events and swap them out. The issue is - * relevant when it comes to the subsegments that don't have a preroll phase. */ - struct arcan_shmif_initial initial; - -/* The entire log mechanism is a bit dated, it was mainly for ARCAN_SHMIF_DEBUG - * environment set, but forwarding that to stderr when we have a real channel - * where it can be done, so this should be moved to the DEBUGIF mechanism */ - int log_event; - -/* Previously this was passed as environment variables that we are gradually - * moving away form. This is an opaque handle matching what the static build- - * time keystore (currently only _naive which uses a directory tree). Using - * device_node events can define this for us to pass on to arcan_net or used - * for signing states that we want signed. */ - int keystate_store; - - bool valid_initial : 1; - -/* "input" and "output" are poorly chosen names that stuck around for legacy, - * but basically ENCODER segments receive buffer contents rather than send it. */ - bool output : 1; - bool alive : 1; - -/* By default, the 'pause' mechanism is a trigger for the server- side to - * block in the calling thread into shmif functions until a resume- event - * has been received */ - bool paused : 1; - -/* When waiting for a descriptor to pair with an incoming event, if this is set - * the next pairing will not be forwarded to the client but instead consumed - * immediately. Main use is NEWSEGMENT for a forced DEBUG */ - bool autoclean : 1; - -/* Track an 'alternate connection path' that the server can update with a call - * to devicehint. A possible change here is for the alt_conn to be controlled - * by the user via an environment variable */ - char* alt_conn; - -/* The named key used to find the initial connection (if there is one) should - * be unliked on use. For special cases (SHMIF_DONT_UNLINK) this can be deferred - * and be left to the user. In these scenarios we need to keep the key around. */ - char* shm_key; - -/* User- provided setup flags and segment types are kept / tracked in order - * to re-issue events on a hard reset or migration */ - enum ARCAN_FLAGS flags; - int type; - enum shmif_ext_meta atype; - uint64_t guid[2]; - -/* The ingoing and outgoing event queues */ - struct arcan_evctx inev; - struct arcan_evctx outev; - -/* Typically not used, but some multithreaded clients that need locking controls - * have mutexes allocated and kept here, then we can log / warn / detect if a - * resize or migrate call is performed when it is unsafe */ - pthread_mutex_t lock; - bool in_lock, in_signal, in_migrate; - pthread_t lock_id; - pthread_t primary_id; - -/* during automatic pause, we want displayhint and fonthint events to queue and - * aggregate so we can return immediately on release, this pattern can be - * re-used for more events should they be needed (possibly CLOCK..) */ - struct arcan_event dh, fh; - int ph; /* bit 1, dh - bit 2 fh */ - -/* POSIX token passing is notoriously awful, in cases where we have to use the - * socket descriptor passing mechanism, we need to pair the descriptor with the - * corresponding event. This structure is used to track these states. */ - struct { - bool gotev, consumed; - bool handedover; - arcan_event ev; - file_handle fd; - } pev; - -/* When a NEWSEGMENT event has been provided (and descriptor- paired) the caller - * still needs to map it via a normal _acquire call. If that doesn't happen, the - * implementation of shmif is allowed to do whatever, thus we need to track the - * data needed for acquire */ - struct { - int epipe; - char key[256]; - } pseg; - - struct mstate mstate; - -/* The 'guard' structure is used by a separate monitoring thread that will - * track a pid or descriptor for aliveness. If the tracking fails, it will - * unlock semaphores and trigger an at_exit- like handler. This is practically - * necessary due to the poor multiplexation options for semaphores. */ - struct { - bool active; - -/* Fringe-wise, we need two DMSes, one set in shmpage and another using the - * guard-thread, then both need to be checked after every semaphore lock */ - _Atomic bool local_dms; - sem_handle semset[3]; - process_handle parent; - int parent_fd; - volatile uint8_t* _Atomic volatile dms; - pthread_mutex_t synch; - void (*exitf)(int val); - } guard; -}; /* We let one 'per process singleton' slot for an input and for an output * segment as a primitive discovery mechanism. This is managed (mostly) by the * caller, though cleaned up on certain calls like _drop etc. to avoid UAF. */ static struct { - struct arcan_shmif_cont* input, (* output); + struct arcan_shmif_cont* input, (* output), (*accessibility); } primary; static void* guard_thread(void* gstruct); @@ -409,10 +272,6 @@ static bool fd_event(struct arcan_shmif_cont* c, struct arcan_event* dst) return false; } -#ifdef SHMIF_DEBUG_IF -#include "arcan_shmif_debugif.h" -#endif - void arcan_shmif_defimpl( struct arcan_shmif_cont* newchild, int type, void* typetag) { @@ -422,6 +281,7 @@ void arcan_shmif_defimpl( return; } #endif + arcan_shmif_drop(newchild); } @@ -469,13 +329,24 @@ static void consume(struct arcan_shmif_cont* c) arcan_shmif_acquire(c, NULL, SEGID_DEBUG, SHMIF_NOREGISTER); if (pcont.addr){ - if (!arcan_shmif_debugint_spawn(&pcont, NULL, NULL)){ + if (!arcan_shmif_debugint_spawn(&pcont, NULL, NULL)) arcan_shmif_drop(&pcont); - } return; } } #endif + if (c->priv->pev.gotev && + c->priv->pev.ev.category == EVENT_TARGET && + c->priv->pev.ev.tgt.kind == TARGET_COMMAND_NEWSEGMENT && + c->priv->pev.ev.tgt.ioevs[2].iv == SEGID_ACCESSIBILITY){ + struct arcan_shmif_cont pcont = + arcan_shmif_acquire(c, NULL, SEGID_ACCESSIBILITY, 0); + if (pcont.addr){ + if (!arcan_shmif_a11yint_spawn(&pcont, c)) + arcan_shmif_drop(&pcont); + return; + } + } close(c->priv->pseg.epipe); c->priv->pseg.epipe = BADFD; @@ -806,6 +677,10 @@ static int process_events(struct arcan_shmif_cont* c, bool noks = false; int rv = 0; + if (priv->support_window_hook){ + priv->support_window_hook(c, SUPPORT_EVENT_POLL); + } + /* Select few events has a special queue position and can be delivered 'out of * order' from normal affairs. This is needed for displayhint/fonthint in WM * cases where a connection may be suspended for a long time and normal system @@ -2065,6 +1940,9 @@ unsigned arcan_shmif_signal(struct arcan_shmif_cont* ctx, int mask) if ( (mask & SHMIF_SIGVID) && priv->video_hook) mask = priv->video_hook(ctx); + if ( (mask & SHMIF_SIGVID) && priv->support_window_hook) + priv->support_window_hook(ctx, SUPPORT_EVENT_VSIGNAL); + if ( (mask & SHMIF_SIGAUD) && priv->audio_hook) mask = priv->audio_hook(ctx); @@ -2110,6 +1988,10 @@ void arcan_shmif_drop(struct arcan_shmif_cont* inctx) if (!inctx || !inctx->priv) return; + if (inctx->priv->support_window_hook){ + inctx->priv->support_window_hook(inctx, SUPPORT_EVENT_EXIT); + } + pthread_mutex_lock(&inctx->priv->lock); if (inctx->priv->valid_initial) @@ -2391,15 +2273,19 @@ struct arcan_shmif_cont* arcan_shmif_primary(enum arcan_shmif_type type) { if (type == SHMIF_INPUT) return primary.input; + else if (type == SHMIF_ACCESSIBILITY) + return primary.accessibility; else return primary.output; } -void arcan_shmif_setprimary(enum arcan_shmif_type type, - struct arcan_shmif_cont* seg) +void arcan_shmif_setprimary( + enum arcan_shmif_type type, struct arcan_shmif_cont* seg) { if (type == SHMIF_INPUT) primary.input = seg; + else if (type == SHMIF_ACCESSIBILITY) + primary.accessibility = seg; else primary.output = seg; } diff --git a/src/shmif/arcan_shmif_control.h b/src/shmif/arcan_shmif_control.h index 8f9cd58c7..9372737ad 100644 --- a/src/shmif/arcan_shmif_control.h +++ b/src/shmif/arcan_shmif_control.h @@ -198,14 +198,27 @@ static const int ARCAN_SHMPAGE_START_SZ = PP_SHMPAGE_STARTSZ; static const int ARCAN_SHMPAGE_ALIGN = PP_SHMPAGE_ALIGN; /* - * Two primary transfer operation types, from the perspective of the - * main arcan application (i.e. normally frameservers feed INPUT but - * specialized recording segments are flagged as OUTPUT. Internally, - * these have different synchronization rules. + * Three primary transfer operation types, from the perspective of the main + * arcan application (i.e. normally frameservers feed INPUT but specialized + * recording segments are flagged as OUTPUT. Internally, these have different + * synchronization rules. + * + * Accessibility is special. + * + * The server end will push it and the internal loop will map it and map it + * to the slot for the primary segment. The suggested approach is to call + * arcan_shmif_primary(SHMIF_ACCESSIBILITY) and if it returns a structure, + * extend it to arcan-tui and write into it the data that should be emphasised. + * + * The reason this is reserved and mapped internally like this is to allow a + * fallback where, if you provide nothing, we can route any _signal call + * through afsrv_decode and have it populate the accessibility segment on frame + * submit. */ enum arcan_shmif_type { SHMIF_INPUT = 1, - SHMIF_OUTPUT + SHMIF_OUTPUT, + SHMIF_ACCESSIBILITY }; /* diff --git a/src/shmif/arcan_shmif_debugif.c b/src/shmif/arcan_shmif_debugif.c index fc4db53de..ee224aecf 100644 --- a/src/shmif/arcan_shmif_debugif.c +++ b/src/shmif/arcan_shmif_debugif.c @@ -54,7 +54,7 @@ */ #include "arcan_shmif.h" #include "arcan_shmif_interop.h" -#include "arcan_shmif_debugif.h" +#include "shmif_defimpl.h" #include #include #include diff --git a/src/shmif/arcan_shmif_debugif.h b/src/shmif/shmif_defimpl.h similarity index 84% rename from src/shmif/arcan_shmif_debugif.h rename to src/shmif/shmif_defimpl.h index fc5b2675a..b94738a95 100644 --- a/src/shmif/arcan_shmif_debugif.h +++ b/src/shmif/shmif_defimpl.h @@ -14,4 +14,6 @@ bool arcan_shmif_debugint_spawn( struct arcan_shmif_cont* c, void* tuitag, struct debugint_ext_resolver* res); int arcan_shmif_debugint_alive(); +bool arcan_shmif_a11yint_spawn(struct arcan_shmif_cont* c, struct arcan_shmif_cont* p); + #endif diff --git a/src/shmif/shmif_privint.h b/src/shmif/shmif_privint.h new file mode 100644 index 000000000..247814554 --- /dev/null +++ b/src/shmif/shmif_privint.h @@ -0,0 +1,150 @@ +struct mstate { + union { + struct { + int32_t ax, ay, lx, ly; + uint8_t rel : 1; + uint8_t inrel : 1; + }; + uint8_t state[ASHMIF_MSTATE_SZ]; + }; +}; + +enum support_states { + SUPPORT_EVENT_VSIGNAL, + SUPPORT_EVENT_POLL, + SUPPORT_EVENT_EXIT +}; + +struct shmif_hidden { + struct arg_arr* args; + char* last_words; + + shmif_trigger_hook_fptr video_hook; + void* video_hook_data; + + void (*support_window_hook)(struct arcan_shmif_cont* c, int state); + void* support_window_hook_data; + + uint8_t vbuf_ind, vbuf_cnt; + bool vbuf_nbuf_active; + uint64_t vframe_id; + shmif_pixel* vbuf[ARCAN_SHMIF_VBUFC_LIM]; + + shmif_trigger_hook_fptr audio_hook; + void* audio_hook_data; + uint8_t abuf_ind, abuf_cnt; + shmif_asample* abuf[ARCAN_SHMIF_ABUFC_LIM]; + + shmif_reset_hook_fptr reset_hook; + void* reset_hook_tag; + +/* Initial contents gets dropped after first valid !initial call after open, + * otherwise we will be left with file descriptors in the process that might + * not get used. What argues against keeping them after is that we also need + * to track the descriptor carrying events and swap them out. The issue is + * relevant when it comes to the subsegments that don't have a preroll phase. */ + struct arcan_shmif_initial initial; + +/* The entire log mechanism is a bit dated, it was mainly for ARCAN_SHMIF_DEBUG + * environment set, but forwarding that to stderr when we have a real channel + * where it can be done, so this should be moved to the DEBUGIF mechanism */ + int log_event; + +/* Previously this was passed as environment variables that we are gradually + * moving away form. This is an opaque handle matching what the static build- + * time keystore (currently only _naive which uses a directory tree). Using + * device_node events can define this for us to pass on to arcan_net or used + * for signing states that we want signed. */ + int keystate_store; + + bool valid_initial : 1; + +/* "input" and "output" are poorly chosen names that stuck around for legacy, + * but basically ENCODER segments receive buffer contents rather than send it. */ + bool output : 1; + bool alive : 1; + +/* By default, the 'pause' mechanism is a trigger for the server- side to + * block in the calling thread into shmif functions until a resume- event + * has been received */ + bool paused : 1; + +/* When waiting for a descriptor to pair with an incoming event, if this is set + * the next pairing will not be forwarded to the client but instead consumed + * immediately. Main use is NEWSEGMENT for a forced DEBUG */ + bool autoclean : 1; + +/* Track an 'alternate connection path' that the server can update with a call + * to devicehint. A possible change here is for the alt_conn to be controlled + * by the user via an environment variable */ + char* alt_conn; + +/* The named key used to find the initial connection (if there is one) should + * be unliked on use. For special cases (SHMIF_DONT_UNLINK) this can be deferred + * and be left to the user. In these scenarios we need to keep the key around. */ + char* shm_key; + +/* User- provided setup flags and segment types are kept / tracked in order + * to re-issue events on a hard reset or migration */ + enum ARCAN_FLAGS flags; + int type; + enum shmif_ext_meta atype; + uint64_t guid[2]; + +/* The ingoing and outgoing event queues */ + struct arcan_evctx inev; + struct arcan_evctx outev; + +/* Typically not used, but some multithreaded clients that need locking controls + * have mutexes allocated and kept here, then we can log / warn / detect if a + * resize or migrate call is performed when it is unsafe */ + pthread_mutex_t lock; + bool in_lock, in_signal, in_migrate; + pthread_t lock_id; + pthread_t primary_id; + +/* during automatic pause, we want displayhint and fonthint events to queue and + * aggregate so we can return immediately on release, this pattern can be + * re-used for more events should they be needed (possibly CLOCK..) */ + struct arcan_event dh, fh; + int ph; /* bit 1, dh - bit 2 fh */ + +/* POSIX token passing is notoriously awful, in cases where we have to use the + * socket descriptor passing mechanism, we need to pair the descriptor with the + * corresponding event. This structure is used to track these states. */ + struct { + bool gotev, consumed; + bool handedover; + arcan_event ev; + file_handle fd; + } pev; + +/* When a NEWSEGMENT event has been provided (and descriptor- paired) the caller + * still needs to map it via a normal _acquire call. If that doesn't happen, the + * implementation of shmif is allowed to do whatever, thus we need to track the + * data needed for acquire */ + struct { + int epipe; + char key[256]; + } pseg; + + struct mstate mstate; + +/* The 'guard' structure is used by a separate monitoring thread that will + * track a pid or descriptor for aliveness. If the tracking fails, it will + * unlock semaphores and trigger an at_exit- like handler. This is practically + * necessary due to the poor multiplexation options for semaphores. */ + struct { + bool active; + +/* Fringe-wise, we need two DMSes, one set in shmpage and another using the + * guard-thread, then both need to be checked after every semaphore lock */ + _Atomic bool local_dms; + sem_handle semset[3]; + process_handle parent; + int parent_fd; + volatile uint8_t* _Atomic volatile dms; + pthread_mutex_t synch; + void (*exitf)(int val); + } guard; +}; diff --git a/src/tools/adbginject/src/adbinject.c b/src/tools/adbginject/src/adbinject.c index 5d0e037e4..1e16a523f 100644 --- a/src/tools/adbginject/src/adbinject.c +++ b/src/tools/adbginject/src/adbinject.c @@ -2,7 +2,7 @@ #include #include -#include "../../shmif/arcan_shmif_debugif.h" +#include "../../shmif/shmif_defimpl.h" volatile struct arcan_shmif_cont cont; static volatile bool hold_constructor = true; diff --git a/src/wayland/xwlwm/xwlwm.c b/src/wayland/xwlwm/xwlwm.c index a7763ea3b..2f386d46f 100644 --- a/src/wayland/xwlwm/xwlwm.c +++ b/src/wayland/xwlwm/xwlwm.c @@ -8,7 +8,7 @@ * sandboxed better and possibly used for a similar -rootless mode in Xarcan. */ #include "../../shmif/arcan_shmif.h" -#include "../../shmif/arcan_shmif_debugif.h" +#include "../../shmif/shmif_defimpl.h" #include #include #include