diff --git a/packages/app/studio/README.md b/packages/app/studio/README.md index 5b8e66a9f..c5abc1fb7 100644 --- a/packages/app/studio/README.md +++ b/packages/app/studio/README.md @@ -9,6 +9,11 @@ For a guided overview of the interface, see the [UI tour](../../docs/docs-user/u Guidance on saving and importing projects lives in the [file management guide](../../docs/docs-user/features/file-management.md). The [notepad feature](../../docs/docs-user/features/notepad.md) lets you store project notes using Markdown. Developer details about project storage and sessions can be found in the [projects documentation](../../docs/docs-dev/projects/overview.md). +The capture subsystem for recording audio and MIDI is described in the +[capture architecture docs](../../docs/docs-dev/architecture/capture/overview.md). +Users can follow the [recording workflow guide](../../docs/docs-user/workflows/recording.md) +to learn how to capture takes. + Quickly launch commands using the Spotlight search palette with Shift+Enter. Read the [user guide](../../docs/docs-user/features/search.md) or see the [developer docs](../../docs/docs-dev/ui/spotlight/overview.md). Exchange projects with other DAWs via the `.dawproject` format using the [DAWproject workflow](../../docs/docs-user/workflows/dawproject.md). Implementation diff --git a/packages/app/studio/src/project/ProjectDialogs.tsx b/packages/app/studio/src/project/ProjectDialogs.tsx index f5ec37b99..ce6fec647 100644 --- a/packages/app/studio/src/project/ProjectDialogs.tsx +++ b/packages/app/studio/src/project/ProjectDialogs.tsx @@ -1,5 +1,7 @@ /** * Collection of dialogs used for common project management operations. + * See the capture architecture docs for details on recording: + * {@link ../../../../docs/docs-dev/architecture/capture/overview.md}. */ import {Dialog} from "@/ui/components/Dialog" import {ExportStemsConfiguration, IconSymbol} from "@opendaw/studio-adapters" diff --git a/packages/docs/docs-dev/architecture/capture/audio.md b/packages/docs/docs-dev/architecture/capture/audio.md new file mode 100644 index 000000000..053f95482 --- /dev/null +++ b/packages/docs/docs-dev/architecture/capture/audio.md @@ -0,0 +1,8 @@ +# Audio Capture + +`CaptureAudio` wraps a browser `MediaStream` to provide microphone and line +input. When armed, it requests a stream matching the desired device and channel +configuration and feeds the data into a [`RecordingWorklet`](./worklet.md). + +Levels can be adjusted via a software gain stage before samples are written to +the recording buffer. diff --git a/packages/docs/docs-dev/architecture/capture/midi.md b/packages/docs/docs-dev/architecture/capture/midi.md new file mode 100644 index 000000000..2425ae97f --- /dev/null +++ b/packages/docs/docs-dev/architecture/capture/midi.md @@ -0,0 +1,6 @@ +# MIDI Capture + +`CaptureMidi` listens to Web MIDI input devices and forwards filtered note +messages to the recording subsystem. Each capture can restrict events to a +specific channel and normalises note on/off pairs before they are persisted in +the project timeline. diff --git a/packages/docs/docs-dev/architecture/capture/overview.md b/packages/docs/docs-dev/architecture/capture/overview.md new file mode 100644 index 000000000..42b5a1bde --- /dev/null +++ b/packages/docs/docs-dev/architecture/capture/overview.md @@ -0,0 +1,14 @@ +# Capture Architecture + +The capture layer manages audio and MIDI inputs that feed the recording system. +It instantiates `Capture` objects for audio units and coordinates their lifecycles +through the `CaptureManager`. + +- [Audio capture](./audio.md) describes how microphone and line inputs are + obtained from the browser. +- [MIDI capture](./midi.md) covers incoming note events. +- [Recording worklet](./worklet.md) explains how raw samples are buffered and + converted into peak data. + +This subsystem works in tandem with the [`Recording` API](../../../../studio/core/src/capture/Recording.ts) +which orchestrates the actual recording session. diff --git a/packages/docs/docs-dev/architecture/capture/worklet.md b/packages/docs/docs-dev/architecture/capture/worklet.md new file mode 100644 index 000000000..1ab12b89f --- /dev/null +++ b/packages/docs/docs-dev/architecture/capture/worklet.md @@ -0,0 +1,6 @@ +# Recording Worklet + +The `RecordingWorklet` is an `AudioWorkletNode` that receives frames from a +`MediaStream` and writes them into a ring buffer. It also computes peak data so +waveforms can be displayed immediately after recording. Once finalized, the +worklet stores audio and peak information through the sample storage service. diff --git a/packages/docs/docs-user/workflows/recording.md b/packages/docs/docs-user/workflows/recording.md new file mode 100644 index 000000000..df4af4877 --- /dev/null +++ b/packages/docs/docs-user/workflows/recording.md @@ -0,0 +1,18 @@ +# Recording Workflow + +Capture audio or MIDI directly onto the timeline. + +1. **Choose a track type.** Add an audio or instrument track depending on the + source you want to record. +2. **Arm the track.** Click the record icon on the track header to enable + input monitoring. +3. **Select the device.** Use the track's capture panel to pick the desired + microphone or MIDI device. +4. **Check levels.** Speak or play to ensure the meter responds without + clipping. +5. **Press record.** Hit the transport record button and perform. +6. **Stop when finished.** The take is placed on the timeline where it can be + edited like any other clip. + +For details on the underlying system see the +[developer capture docs](../../docs-dev/architecture/capture/overview.md). diff --git a/packages/docs/sidebarsDev.js b/packages/docs/sidebarsDev.js index d6b07f559..98f08bad5 100644 --- a/packages/docs/sidebarsDev.js +++ b/packages/docs/sidebarsDev.js @@ -59,6 +59,16 @@ module.exports = { "architecture/headless-vs-studio", "architecture/persistence", "architecture/opfs-samples", + { + type: "category", + label: "Capture", + items: [ + "architecture/capture/overview", + "architecture/capture/audio", + "architecture/capture/midi", + "architecture/capture/worklet", + ], + }, ], }, { diff --git a/packages/studio/core/src/RecordingWorklet.ts b/packages/studio/core/src/RecordingWorklet.ts index d6086c6f2..97fc62c93 100644 --- a/packages/studio/core/src/RecordingWorklet.ts +++ b/packages/studio/core/src/RecordingWorklet.ts @@ -70,7 +70,9 @@ class PeaksWriter implements Peaks, Peaks.Stage { /** * Captures audio from its input and exposes recorded data along with peak - * information. + * information. Recording is performed in the audio worklet thread while the + * class provides a loader style interface for retrieving the resulting + * {@link AudioData} and peak information. */ export class RecordingWorklet extends AudioWorkletNode implements Terminable, SampleLoader { readonly uuid: UUID.Format = UUID.generate() @@ -118,13 +120,22 @@ export class RecordingWorklet extends AudioWorkletNode implements Terminable, Sa }) } + /** Total number of frames recorded so far. */ get numberOfFrames(): int {return this.#output.length * RenderQuantum} + /** Recorded audio data once {@link finalize} has been called. */ get data(): Option {return this.#data} + /** Peak information for the recorded data if available. */ get peaks(): Option {return this.#peaks} + /** Loading state for consumers implementing {@link SampleLoader}. */ get state(): SampleLoaderState {return this.#state} + /** Part of {@link SampleLoader}; no-op for recordings. */ invalidate(): void {} + /** + * Observe loader state changes. Once loaded, the observer is invoked + * immediately and no subscription is kept. + */ subscribe(observer: Observer): Subscription { if (this.#state.type === "loaded") { observer(this.#state) @@ -162,6 +173,7 @@ export class RecordingWorklet extends AudioWorkletNode implements Terminable, Sa this.#setState({type: "loaded"}) } + /** Stop recording and release any buffered data. */ terminate(): void { this.#reader.stop() this.#isRecording = false diff --git a/packages/studio/core/src/capture/Capture.ts b/packages/studio/core/src/capture/Capture.ts index ffde91b3a..f00f4a3d2 100644 --- a/packages/studio/core/src/capture/Capture.ts +++ b/packages/studio/core/src/capture/Capture.ts @@ -13,6 +13,11 @@ import {CaptureBox} from "@opendaw/studio-adapters" import {RecordingContext} from "./RecordingContext" import {CaptureManager} from "./CaptureManager" +/** + * Base class for audio or MIDI capture units. A {@link Capture} wraps a + * {@link CaptureBox} and manages device selection and arming state for the + * associated {@link AudioUnitBox}. + */ export abstract class Capture implements Terminable { readonly #terminator = new Terminator() @@ -23,6 +28,11 @@ export abstract class Capture implements Te readonly #deviceId: MutableObservableValue> readonly #armed: DefaultObservableValue + /** + * @param manager Parent {@link CaptureManager} instance. + * @param audioUnitBox The audio unit this capture belongs to. + * @param captureBox Backing box containing persisted capture settings. + */ protected constructor(manager: CaptureManager, audioUnitBox: AudioUnitBox, captureBox: BOX) { this.#manager = manager this.#audioUnitBox = audioUnitBox @@ -42,18 +52,30 @@ export abstract class Capture implements Te ) } + /** Human readable label of the current device, if available. */ abstract get deviceLabel(): Option + /** Prepare the capture source for recording (e.g. open streams). */ abstract prepareRecording(context: RecordingContext): Promise + /** Begin capturing and return a handle that stops the session when terminated. */ abstract startRecording(context: RecordingContext): Terminable + /** UUID of the owning {@link AudioUnitBox}. */ get uuid(): UUID.Format {return this.#audioUnitBox.address.uuid} + /** Owning manager for this capture. */ get manager(): CaptureManager {return this.#manager} + /** The {@link AudioUnitBox} that this capture wraps. */ get audioUnitBox(): AudioUnitBox {return this.#audioUnitBox} + /** Underlying {@link CaptureBox} containing state. */ get captureBox(): BOX {return this.#captureBox} + /** Observable arming state. */ get armed(): MutableObservableValue {return this.#armed} + /** Observable selected device id. */ get deviceId(): MutableObservableValue> {return this.#deviceId} + /** Register a terminable that is disposed with this capture. */ own(terminable: T): T {return this.#terminator.own(terminable)} + /** Register multiple terminables to be disposed with this capture. */ ownAll(...terminables: ReadonlyArray): void {this.#terminator.ownAll(...terminables)} + /** Clean up all resources associated with this capture. */ terminate(): void {this.#terminator.terminate()} } \ No newline at end of file diff --git a/packages/studio/core/src/capture/CaptureAudio.ts b/packages/studio/core/src/capture/CaptureAudio.ts index f0ed6c9a2..2ca8be849 100644 --- a/packages/studio/core/src/capture/CaptureAudio.ts +++ b/packages/studio/core/src/capture/CaptureAudio.ts @@ -7,6 +7,10 @@ import {RecordAudio} from "./RecordAudio" import {RecordingContext} from "./RecordingContext" import {AudioDevices} from "../AudioDevices" +/** + * Handles capturing audio from a {@link MediaStream}. The capture will lazily + * create and update the stream based on the selected device and arming state. + */ export class CaptureAudio extends Capture { readonly #stream: MutableObservableOption @@ -15,6 +19,11 @@ export class CaptureAudio extends Capture { #requestChannels: Option<1 | 2> = Option.None #gainDb: number = 0.0 + /** + * @param manager Parent {@link CaptureManager}. + * @param audioUnitBox Audio unit the capture belongs to. + * @param captureBox Box storing capture configuration. + */ constructor(manager: CaptureManager, audioUnitBox: AudioUnitBox, captureBox: CaptureAudioBox) { super(manager, audioUnitBox, captureBox) @@ -43,26 +52,38 @@ export class CaptureAudio extends Capture { ) } + /** Gain to apply to the media stream in decibels. */ get gainDb(): number {return this.#gainDb} + /** Observable wrapper around the active `MediaStream`. */ get stream(): MutableObservableOption {return this.#stream} + /** Device id reported by the active {@link MediaStreamTrack}. */ get streamDeviceId(): Option { return this.streamMediaTrack.map(settings => settings.getSettings().deviceId ?? "") } + /** Human readable label of the active input device. */ get deviceLabel(): Option { return this.streamMediaTrack.map(track => track.label ?? "") } + /** First audio track from the current media stream. */ get streamMediaTrack(): Option { return this.#stream.flatMap(stream => Option.wrap(stream.getAudioTracks().at(0))) } + /** + * Ensure a media stream exists for the current device selection. Called + * before a recording session starts. + */ async prepareRecording(_: RecordingContext): Promise { return this.#streamGenerator() } + /** + * Begin recording the current stream into a {@link RecordingWorklet}. + */ startRecording({audioContext, worklets, project, engine, sampleManager}: RecordingContext): Terminable { const streamOption = this.#stream assert(streamOption.nonEmpty(), "Stream not prepared.") diff --git a/packages/studio/core/src/capture/CaptureManager.ts b/packages/studio/core/src/capture/CaptureManager.ts index 8040bb542..02cc5f88f 100644 --- a/packages/studio/core/src/capture/CaptureManager.ts +++ b/packages/studio/core/src/capture/CaptureManager.ts @@ -5,11 +5,19 @@ import {Capture} from "./Capture" import {CaptureMidi} from "./CaptureMidi" import {CaptureAudio} from "./CaptureAudio" +/** + * Tracks all {@link Capture} instances within a {@link Project}. The manager + * observes the project's audio units and creates the appropriate capture + * implementation for each unit. + */ export class CaptureManager implements Terminable { readonly #project: Project readonly #subscription: Subscription readonly #captures: SortedSet + /** + * @param project Project owning the capture devices. + */ constructor(project: Project) { this.#project = project this.#captures = UUID.newSet(unit => unit.uuid) @@ -27,15 +35,21 @@ export class CaptureManager implements Terminable { }) } + /** Owning project. */ get project(): Project {return this.#project} + /** Lookup a capture by the UUID of its audio unit. */ get(uuid: UUID.Format): Option {return this.#captures.opt(uuid)} + /** + * Return all captures that are armed and connected to an input. + */ filterArmed(): ReadonlyArray { return this.#captures.values() .filter(capture => capture.armed.getValue() && capture.audioUnitBox.input.pointerHub.nonEmpty()) } + /** Dispose all captures and internal subscriptions. */ terminate(): void { this.#subscription.terminate() this.#captures.forEach(capture => capture.terminate()) diff --git a/packages/studio/core/src/capture/CaptureMidi.ts b/packages/studio/core/src/capture/CaptureMidi.ts index 8db959bad..0daaccc54 100644 --- a/packages/studio/core/src/capture/CaptureMidi.ts +++ b/packages/studio/core/src/capture/CaptureMidi.ts @@ -20,6 +20,10 @@ import {CaptureManager} from "./CaptureManager" import {MidiDevices} from "../MidiDevices" import {Promises} from "@opendaw/lib-runtime" +/** + * Capture implementation for MIDI input devices. Streams MIDI messages from + * selected devices and forwards them to the recording pipeline. + */ export class CaptureMidi extends Capture { readonly #streamGenerator: Func> readonly #notifier = new Notifier() @@ -28,6 +32,11 @@ export class CaptureMidi extends Capture { #streaming: Option = Option.None + /** + * @param manager Parent {@link CaptureManager}. + * @param audioUnitBox Owning audio unit. + * @param captureMidiBox Backing box containing configuration. + */ constructor(manager: CaptureManager, audioUnitBox: AudioUnitBox, captureMidiBox: CaptureMidiBox) { super(manager, audioUnitBox, captureMidiBox) @@ -54,8 +63,12 @@ export class CaptureMidi extends Capture { ) } + /** Human readable label of the current MIDI device. */ get deviceLabel(): Option {return Option.wrap("MIDI coming soon.")} + /** + * Ensure the selected MIDI device is available before recording starts. + */ async prepareRecording(_: RecordingContext): Promise { const availableMidiDevices = MidiDevices.get() if (availableMidiDevices.isEmpty()) { @@ -72,6 +85,7 @@ export class CaptureMidi extends Capture { } } + /** Start streaming MIDI events into the project engine. */ startRecording({project, engine}: RecordingContext): Terminable { const availableMidiDevices = MidiDevices.inputs() assert(availableMidiDevices.nonEmpty(), "No MIDI input devices found") diff --git a/packages/studio/core/src/capture/RecordAudio.ts b/packages/studio/core/src/capture/RecordAudio.ts index 6eb841954..773402f5e 100644 --- a/packages/studio/core/src/capture/RecordAudio.ts +++ b/packages/studio/core/src/capture/RecordAudio.ts @@ -9,7 +9,9 @@ import {RecordTrack} from "./RecordTrack" import {RecordingWorklet} from "../RecordingWorklet" import {ColorCodes} from "../ColorCodes" +/** Utilities for recording audio into a project. */ export namespace RecordAudio { + /** Context required to start recording audio. */ type RecordAudioContext = { recordingWorklet: RecordingWorklet mediaStream: MediaStream @@ -21,6 +23,9 @@ export namespace RecordAudio { gainDb: number } + /** + * Begin recording the supplied media stream into the project timeline. + */ export const start = ( { recordingWorklet, mediaStream, sampleManager, audioContext, engine, project, capture, gainDb diff --git a/packages/studio/core/src/capture/RecordMidi.ts b/packages/studio/core/src/capture/RecordMidi.ts index 431600c9d..d9d847afe 100644 --- a/packages/studio/core/src/capture/RecordMidi.ts +++ b/packages/studio/core/src/capture/RecordMidi.ts @@ -19,7 +19,9 @@ import {Capture} from "./Capture" import {RecordTrack} from "./RecordTrack" import {ColorCodes} from "../ColorCodes" +/** Utilities for recording MIDI note data. */ export namespace RecordMidi { + /** Context required to start a MIDI recording. */ type RecordMidiContext = { notifier: Notifier, engine: Engine, @@ -27,6 +29,9 @@ export namespace RecordMidi { capture: Capture } + /** + * Begin recording MIDI events into the project timeline. + */ export const start = ({notifier, engine, project, capture}: RecordMidiContext): Terminable => { console.debug("RecordMidi.start") const beats = PPQN.fromSignature(1, project.timelineBox.signature.denominator.getValue()) diff --git a/packages/studio/core/src/capture/RecordTrack.ts b/packages/studio/core/src/capture/RecordTrack.ts index abcac6fee..d2d3472d2 100644 --- a/packages/studio/core/src/capture/RecordTrack.ts +++ b/packages/studio/core/src/capture/RecordTrack.ts @@ -3,7 +3,12 @@ import {asInstanceOf, int, UUID} from "@opendaw/lib-std" import {TrackType} from "@opendaw/studio-adapters" import {Editing} from "@opendaw/lib-box" +/** Helper utilities for locating tracks used during recording. */ export namespace RecordTrack { + /** + * Find an existing empty track matching the requested type or create a new + * track for the given {@link AudioUnitBox}. + */ export const findOrCreate = (editing: Editing, audioUnitBox: AudioUnitBox, type: TrackType): TrackBox => { let index: int = 0 | 0 for (const trackBox of audioUnitBox.tracks.pointerHub.incoming() diff --git a/packages/studio/core/src/capture/Recording.ts b/packages/studio/core/src/capture/Recording.ts index 3b04afe3b..a521c6af6 100644 --- a/packages/studio/core/src/capture/Recording.ts +++ b/packages/studio/core/src/capture/Recording.ts @@ -6,9 +6,21 @@ import {AudioUnitType} from "@opendaw/studio-enums" import {InstrumentFactories} from "../InstrumentFactories" import {Project} from "../Project" +/** + * Coordinates the recording lifecycle. This singleton orchestrates capture + * preparation, engine state and cleanup across multiple capture sources. + */ export class Recording { + /** Whether a recording session is currently active. */ static get isRecording(): boolean {return this.#isRecording} + /** + * Start a new recording session using the supplied context. + * + * @param context Runtime objects used during recording. + * @param countIn If `true`, the engine performs a count-in before + * recording starts. + */ static async start(context: RecordingContext, countIn: boolean): Promise { if (this.#isRecording) { return Promise.resolve(Terminable.Empty) @@ -48,6 +60,10 @@ export class Recording { return terminator } + /** + * Ensure that at least one capture is armed before recording starts. If no + * instruments exist a default Tape instrument will be created. + */ static #prepare({api, captureManager, editing, rootBox, userEditingManager}: Project): void { const captures = captureManager.filterArmed() const instruments = rootBox.audioUnits.pointerHub.incoming() diff --git a/packages/studio/core/src/capture/RecordingContext.ts b/packages/studio/core/src/capture/RecordingContext.ts index 733e4cd06..a0d9175a3 100644 --- a/packages/studio/core/src/capture/RecordingContext.ts +++ b/packages/studio/core/src/capture/RecordingContext.ts @@ -4,11 +4,20 @@ import {Project} from "../Project" import {Engine} from "../Engine" import {Worklets} from "../Worklets" +/** + * Runtime dependencies required by the recording subsystem. + */ export interface RecordingContext { + /** Project being recorded into. */ project: Project + /** Worklet factory used to create processing nodes. */ worklets: Worklets + /** Engine that manages transport and scheduling. */ engine: Engine + /** Audio context providing the clock and media graph. */ audioContext: AudioContext + /** Handles sample persistence. */ sampleManager: SampleManager + /** Provider used to request MIDI access when needed. */ requestMIDIAccess: Provider> } \ No newline at end of file