WIP implement own context classes

Author: almarklein

rendercanvas/MetalIOSurfaceHelper.m

@@ -0,0 +1,88 @@
+/*
+
+clang -dynamiclib -fobjc-arc \
+  -framework Foundation -framework Metal -framework IOSurface \
+  -arch x86_64 -arch arm64 \
+  -mmacosx-version-min=10.13 \
+  -o libMetalIOSurfaceHelper.dylib MetalIOSurfaceHelper.m
+
+*/
+#import <Foundation/Foundation.h>
+#import <Metal/Metal.h>
+#import <IOSurface/IOSurface.h>
+
+@interface MetalIOSurfaceHelper : NSObject
+@property (nonatomic, readonly) id<MTLDevice> device;
+@property (nonatomic, readonly) id<MTLTexture> texture;
+
+- (instancetype)initWithWidth:(NSUInteger)width
+                       height:(NSUInteger)height;
+
+- (void *)baseAddress;
+- (NSUInteger)bytesPerRow;
+@end
+
+
+@implementation MetalIOSurfaceHelper {
+    IOSurfaceRef _surf;
+}
+
+- (instancetype)initWithWidth:(NSUInteger)width
+                       height:(NSUInteger)height
+{
+    if ((self = [super init])) {
+        // Create Metal device
+        _device = MTLCreateSystemDefaultDevice();
+        if (!_device) {
+            NSLog(@"❌ Failed to create Metal device");
+            return nil;
+        }
+
+        // Create IOSurface properties
+        NSDictionary *props = @{
+            (id)kIOSurfaceWidth: @(width),
+            (id)kIOSurfaceHeight: @(height),
+            (id)kIOSurfaceBytesPerElement: @(4),
+            (id)kIOSurfacePixelFormat: @(0x42475241) // 'BGRA'
+        };
+
+        _surf = IOSurfaceCreate((__bridge CFDictionaryRef)props);
+        if (!_surf) {
+            NSLog(@"❌ Failed to create IOSurface");
+            return nil;
+        }
+
+        // Create texture from IOSurface
+        MTLTextureDescriptor *desc =
+            [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:MTLPixelFormatBGRA8Unorm
+                                                                width:width
+                                                               height:height
+                                                            mipmapped:NO];
+        desc.storageMode = MTLStorageModeShared;
+
+        _texture = [_device newTextureWithDescriptor:desc iosurface:_surf plane:0];
+        if (!_texture) {
+            NSLog(@"❌ Failed to create MTLTexture from IOSurface");
+            CFRelease(_surf);
+            return nil;
+        }
+    }
+    return self;
+}
+
+- (void *)baseAddress {
+    return IOSurfaceGetBaseAddress(_surf);
+}
+
+- (NSUInteger)bytesPerRow {
+    return IOSurfaceGetBytesPerRow(_surf);
+}
+
+- (void)dealloc {
+    if (_surf) {
+        CFRelease(_surf);
+        _surf = NULL;
+    }
+}
+
+@end

rendercanvas/__init__.py

@@ -8,5 +8,8 @@
 from . import _coreutils
 from ._enums import CursorShape, EventType, UpdateMode
 from .base import BaseRenderCanvas, BaseLoop
+from . import contexts
+from . import utils
+
 
 __all__ = ["BaseLoop", "BaseRenderCanvas", "CursorShape", "EventType", "UpdateMode"]

rendercanvas/_native_osx.py

@@ -0,0 +1,282 @@
+"""
+
+This uses rubicon to load objc classes, mainly for Cocoa (MacOS's
+windowing API). For rendering to bitmap we follow the super-fast
+approach of creating an IOSurface that is wrapped in a Metal texture.
+On Apple silicon, the memory for that texture is in RAM, so we can write
+directly to the texture, no copies. This approach is used by e.g. video
+viewers.
+
+However, because Python (via Rubicon) cannot pass or create pure C-level
+IOSurfaceRef pointers, which are required by Metal’s
+newTextureWithDescriptor:iosurface:plane; Rubicon can only work with
+actual Objective-C objects.
+
+Therefore this code relies on a mirco objc libary that is shipped along
+in rendercanvas. This dylib handles the C-level IOSurface creation and
+wraps it in a proper MTLTexture that Python can safely use.
+"""
+
+# ruff: noqa - for now
+
+import sys
+import time
+import ctypes
+
+import numpy as np
+from rubicon.objc import ObjCClass, objc_method, ObjCInstance
+
+
+NSApplication = ObjCClass("NSApplication")
+NSWindow = ObjCClass("NSWindow")
+NSObject = ObjCClass("NSObject")
+
+
+# Application and window
+app = NSApplication.sharedApplication
+
+
+SHADER = """
+#include <metal_stdlib>
+using namespace metal;
+
+struct VertexOut {
+    float4 position [[position]];
+    float2 texcoord;
+};
+
+vertex VertexOut vertex_main(uint vertexID [[vertex_id]]) {
+    float2 pos[3] = {
+        float2(-1.0, -1.0),
+        float2( 3.0, -1.0),
+        float2(-1.0,  3.0)
+    };
+    VertexOut out;
+    out.position = float4(pos[vertexID], 0.0, 1.0);
+    out.texcoord = (pos[vertexID] + 1.0) * 0.5;
+    return out;
+}
+
+fragment float4 fragment_main(VertexOut in [[stage_in]],
+                            texture2d<float> tex [[texture(0)]],
+                            sampler samp [[sampler(0)]]) {
+    constexpr sampler linearSampler(address::clamp_to_edge, filter::linear);
+    float4 color = tex.sample(linearSampler, in.texcoord);
+    return color;
+}
+"""
+
+
+class MetalRenderer(NSObject):
+    @objc_method
+    def initWithDevice_(self, device):  # -> ctypes.c_void_p:
+        self.init()
+        # self = ObjCInstance(send_message(self, "init"))
+        if self is None:
+            return None
+        self.device = device
+        self.queue = device.newCommandQueue()
+
+        # self.texture = win._texture
+
+        # --- Metal shader code ---
+
+        options = {}
+        error_placeholder = None  # ctypes.c_void_p()
+        library = device.newLibraryWithSource_options_error_(
+            SHADER, None, error_placeholder
+        )
+        if not library:
+            print("Shader compile failed:", error_placeholder)
+            return self
+
+        vertex_func = library.newFunctionWithName_("vertex_main")
+        frag_func = library.newFunctionWithName_("fragment_main")
+
+        desc = ObjCClass("MTLRenderPipelineDescriptor").alloc().init()
+        desc.vertexFunction = vertex_func
+        desc.fragmentFunction = frag_func
+        desc.colorAttachments.objectAtIndexedSubscript_(
+            0
+        ).pixelFormat = 80  # BGRA8Unorm
+
+        self.pipeline = device.newRenderPipelineStateWithDescriptor_error_(
+            desc, error_placeholder
+        )
+        if not self.pipeline:
+            print("Pipeline creation failed:", error_placeholder)
+        return self
+
+    @objc_method
+    def setTexture_(self, texture):
+        self.texture = texture
+
+    @objc_method
+    def drawInMTKView_(self, view):
+        drawable = view.currentDrawable
+        if drawable is None:
+            return
+
+        passdesc = ObjCClass("MTLRenderPassDescriptor").renderPassDescriptor()
+        passdesc.colorAttachments.objectAtIndexedSubscript_(
+            0
+        ).texture = drawable.texture
+        passdesc.colorAttachments.objectAtIndexedSubscript_(0).loadAction = 2  # Clear
+        passdesc.colorAttachments.objectAtIndexedSubscript_(0).storeAction = 1  # Store
+        passdesc.colorAttachments.objectAtIndexedSubscript_(
+            0
+        ).clearColor = view.clearColor
+
+        cmd_buf = self.queue.commandBuffer()
+        enc = cmd_buf.renderCommandEncoderWithDescriptor_(passdesc)
+
+        enc.setRenderPipelineState_(self.pipeline)
+        enc.setFragmentTexture_atIndex_(self.texture, 0)
+
+        enc.setRenderPipelineState_(self.pipeline)
+        enc.drawPrimitives_vertexStart_vertexCount_(3, 0, 3)
+        enc.endEncoding()
+        cmd_buf.presentDrawable_(drawable)
+        cmd_buf.commit()
+
+    @objc_method
+    def mtkView_drawableSizeWillChange_(self, view, newSize):
+        # Update if needed
+        print("resize", newSize)
+        pass
+
+
+class Window:
+    def __init__(self):
+        self._width = 512
+        self._height = 512
+
+        rect = (100, 100), (384, 216)
+
+        # Define window style
+        NSWindowStyleMaskTitled = 1 << 0
+        NSBackingStoreBuffered = 2
+        NSTitledWindowMask = 1 << 0
+        NSClosableWindowMask = 1 << 1
+        NSMiniaturizableWindowMask = 1 << 2
+        NSResizableWindowMask = 1 << 3
+        style_mask = (
+            NSTitledWindowMask
+            | NSClosableWindowMask
+            | NSMiniaturizableWindowMask
+            | NSResizableWindowMask
+        )
+
+        self._window = NSWindow.alloc().initWithContentRect_styleMask_backing_defer_(
+            rect, style_mask, NSBackingStoreBuffered, False
+        )
+        self._window.setTitle_("Rendercanvas cocoa demo")
+        self._window.makeKeyAndOrderFront_(None)
+
+        self._content_view = self._window.contentView
+
+        self._init_for_bitmap_present()
+
+    def _init_for_bitmap_present(self):
+        # Load our helper dylib to make its objc class available to rubicon.
+        ctypes.CDLL("./libMetalIOSurfaceHelper.dylib")
+
+        # Init our little helper helper
+        MetalIOSurfaceHelper = ObjCClass("MetalIOSurfaceHelper")
+        self._helper = MetalIOSurfaceHelper.alloc().initWithWidth_height_(
+            self._width, self._height
+        )
+        self._texture = self._helper.texture
+        self._device = self._helper.device
+
+        # Access CPU memory
+        base_addr = self._helper.baseAddress()
+        bytes_per_row = self._helper.bytesPerRow()
+
+        # Map array onto the shared memory
+        total_bytes = bytes_per_row * self._height
+        array_type = ctypes.c_uint8 * total_bytes
+        pixel_buf = array_type.from_address(base_addr.value)
+        self._texture_array = np.frombuffer(
+            pixel_buf, dtype=np.uint8, count=total_bytes
+        ).reshape(self._height, self._width, 4)
+
+        # Create MTKView
+        MTKView = ObjCClass("MTKView")
+        mtk_view = MTKView.alloc().initWithFrame_device_(
+            self._content_view.bounds, self._device
+        )
+        # Ensure we can write into the view's texture (not framebuffer-only) if we want to upload into it
+        try:
+            mtk_view.setFramebufferOnly_(False)
+        except Exception:
+            pass  # Not all setups require this call; ignore if not present
+
+        # TODO: use RGBA
+        # TODO: support yuv420p or something
+        # Choose pixel format. We'll assume BGRA8Unorm for Metal.
+        mtk_view.setColorPixelFormat_(80)  # MTLPixelFormatBGRA8Unorm
+
+        self._window.setContentView_(mtk_view)
+        self._window.makeKeyAndOrderFront_(None)  # todo: what does this do?
+
+        # Instantiate the renderer and set as delegate
+        # renderer = MetalRenderer.alloc().init()
+        renderer = MetalRenderer.alloc().initWithDevice_(self._device)
+        renderer.setTexture(self._texture)
+        mtk_view.setDelegate_(renderer)
+
+
+# Example: fill the buffer with a test pattern
+def fill_test_pattern(frame_index=0):
+    # simple moving gradient to prove updates work
+    t = frame_index % 256
+    # pixels_np[:, :, 0] = (np.arange(WIDTH, dtype=np.uint8) + t) & 0xFF  # R-ish
+    # pixels_np[:, :, 1] = (
+    #     np.arange(HEIGHT, dtype=np.uint8).reshape(HEIGHT, 1) + t
+    # ) & 0xFF  # G-ish
+    # pixels_np[:, :, 2] = t  # B constant
+    # pixels_np[:, :, 3] = 255  # full alpha
+
+    pixels_np = win._texture_array
+    pixels_np[:, :, 3] = 255
+    pixels_np[:t, :, :3] = 0
+    pixels_np[t:, :, :3] = 255
+
+
+def update_texture(frame_index):
+    fill_test_pattern(frame_index)
+
+    # mtk_view.setNeedsDisplay_(True)
+    return
+    pixel_buf = win._texture_array
+
+    bytes_per_row = win._width * 4
+    region = ((0, 0, 0), (win._width, win._height, 1))
+    renderer.texture.replaceRegion_mipmapLevel_withBytes_bytesPerRow_(
+        region, 0, pixel_buf, bytes_per_row
+    )
+
+
+win = Window()
+
+frame_index = 0
+while True:
+    frame_index += 1
+    # Drain events (non-blocking)
+    event = app.nextEventMatchingMask_untilDate_inMode_dequeue_(
+        0xFFFFFFFFFFFFFFFF,  # all events
+        None,  # don't wait
+        "kCFRunLoopDefaultMode",
+        True,
+    )
+    if event:
+        app.sendEvent_(event)
+
+    update_texture(frame_index)
+
+    app.updateWindows()
+
+    # your own update / render logic here
+    # (Metal drawInMTKView_ will get called by MTKView’s internal timer)
+    time.sleep(1 / 120)  # e.g. 120 Hz pacing