media encoders crate (#828)

* media encoders crate

* crate READMEs

Author: Brendonovich

Cargo.lock

@@ -11,6 +11,7 @@ cap-media = { path = "../media" }
 cap-project = { path = "../project" }
 cap-rendering = { path = "../rendering" }
 cap-audio = { path = "../audio" }
+cap-media-info = { path = "../media-info" }
 
 tokio = { workspace = true, features = ["macros", "process", "fs"] }
 cpal.workspace = true

crates/editor/Cargo.toml

@@ -1,8 +1,6 @@
 use crate::editor;
 use crate::playback::{self, PlaybackHandle};
 use cap_audio::AudioData;
-use cap_media::data::RawVideoFormat;
-use cap_media::data::VideoInfo;
 // use cap_media::feeds::AudioData;
 use cap_media::frame_ws::create_frame_ws;
 use cap_project::{CursorEvents, ProjectConfiguration, RecordingMeta, RecordingMetaInner, XY};

crates/editor/src/editor_instance.rs

@@ -1,9 +1,10 @@
 use std::{sync::Arc, time::Duration};
 
-use cap_media::data::{AudioInfo, FromSampleBytes};
-use cap_media::feeds::{AudioPlaybackBuffer, AudioSegment, AudioSegmentTrack};
+use cap_media::data::FromSampleBytes;
+use cap_media::feeds::{AudioPlaybackBuffer, AudioSegment};
 use cap_media::MediaError;
-use cap_project::{AudioConfiguration, ProjectConfiguration, XY};
+use cap_media_info::AudioInfo;
+use cap_project::{ProjectConfiguration, XY};
 use cap_rendering::{ProjectUniforms, RenderVideoConstants};
 use cpal::{
     traits::{DeviceTrait, HostTrait, StreamTrait},

crates/editor/src/playback.rs

@@ -10,6 +10,8 @@ cap-rendering = { path = "../rendering" }
 cap-editor = { path = "../editor" }
 cap-media = { path = "../media" }
 cap-flags = { path = "../flags" }
+cap-media-encoders = { path = "../media-encoders" }
+cap-media-info = { path = "../media-info" }
 
 tokio.workspace = true
 tempfile = "3.12.0"

crates/export/Cargo.toml

@@ -1,6 +1,6 @@
 use std::path::PathBuf;
 
-use cap_media::encoders::GifEncoderWrapper;
+use cap_media_encoders::GifEncoderWrapper;
 use cap_project::XY;
 use cap_rendering::{ProjectUniforms, RenderSegment, RenderedFrame};
 use futures::FutureExt;

crates/export/src/gif.rs

@@ -2,11 +2,9 @@ use std::{path::PathBuf, time::Duration};
 
 use crate::ExporterBase;
 use cap_editor::get_audio_segments;
-use cap_media::{
-    data::{RawVideoFormat, VideoInfo},
-    encoders::{AACEncoder, AudioEncoder, H264Encoder, MP4Input},
-    feeds::AudioRenderer,
-};
+use cap_media::feeds::AudioRenderer;
+use cap_media_encoders::{AACEncoder, AudioEncoder, H264Encoder, MP4File, MP4Input};
+use cap_media_info::{RawVideoFormat, VideoInfo};
 use cap_project::XY;
 use cap_rendering::{ProjectUniforms, RenderSegment, RenderedFrame};
 use futures::FutureExt;
@@ -79,7 +77,7 @@ impl Mp4ExportSettings {
         let encoder_thread = tokio::task::spawn_blocking(move || {
             trace!("Creating MP4File encoder");
 
-            let mut encoder = cap_media::encoders::MP4File::init(
+            let mut encoder = MP4File::init(
                 "output",
                 base.output_path.clone(),
                 |o| {
@@ -91,6 +89,7 @@ impl Mp4ExportSettings {
                     has_audio.then(|| {
                         AACEncoder::init("output_audio", AudioRenderer::info(), o)
                             .map(|v| v.boxed())
+                            .map_err(Into::into)
                     })
                 },
             )

crates/export/src/mp4.rs

@@ -0,0 +1,22 @@
+[package]
+name = "cap-media-encoders"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+cap-project = { path = "../project" }
+cap-flags = { path = "../flags" }
+cap-audio = { path = "../audio" }
+cap-fail = { path = "../fail" }
+cap-media-info = { path = "../media-info" }
+
+ffmpeg.workspace = true
+thiserror.workspace = true
+tracing.workspace = true
+gif = "0.13.3"
+
+[target.'cfg(target_os = "macos")'.dependencies]
+cidre = { workspace = true }
+
+[lints]
+workspace = true

crates/media-encoders/Cargo.toml

@@ -0,0 +1,98 @@
+# cap-media-encoders
+
+A comprehensive collection of media encoders for various output formats, designed to work seamlessly with the Cap media processing pipeline. The crate provides unified interfaces for encoding captured audio and video data into different formats using multiple underlying media frameworks.
+
+## Overview
+
+This crate serves as a unified interface for encoding captured media into various output formats, with FFmpeg providing cross-platform support and AVFoundation offering optimized macOS-specific encoding. All encoders support real-time frame queuing and proper finalization for streaming applications.
+
+## Available Encoders
+
+### FFmpeg-based Encoders
+
+**Video Encoders:**
+- `H264Encoder` - H.264/AVC video encoding with configurable quality presets and bitrate control
+- `MP4File` - Complete MP4 container handling H.264 video + audio stream muxing
+
+**Audio Encoders:**
+- `AACEncoder` - AAC audio encoding at 320kbps with automatic resampling and format conversion
+- `OpusEncoder` - Opus audio encoding at 128kbps optimized for voice and music
+- `OggFile` - Ogg container wrapper specifically designed for Opus audio streams
+
+### AVFoundation-based Encoders (macOS only)
+
+**Video + Audio:**
+- `MP4AVAssetWriterEncoder` - Native macOS encoder using AVAssetWriter for hardware-accelerated MP4 encoding
+  - Hardware acceleration support
+  - Pause/resume functionality for recording sessions
+  - Real-time encoding optimizations
+
+### Standalone Encoders
+
+**Animation:**
+- `GifEncoderWrapper` - GIF animation encoder with advanced features:
+  - Floyd-Steinberg dithering for quality color reduction
+  - Custom 256-color palette with grayscale fallback
+  - Configurable frame delay and infinite loop support
+
+## Common Interface
+
+### AudioEncoder Trait
+
+All audio encoders implement the `AudioEncoder` trait providing:
+- `queue_frame()` - Queue audio frames for encoding
+- `finish()` - Finalize encoding and flush remaining data
+- `boxed()` - Convert to boxed trait object for dynamic dispatch
+
+## Key Features
+
+### Format Handling
+- **Automatic conversion**: Seamless pixel format and sample format conversion between input and encoder requirements
+- **Resampling**: Built-in audio resampling for sample rate and format mismatches
+- **Validation**: Input validation with detailed error reporting
+
+### Performance Optimizations
+- **Hardware acceleration**: AVFoundation encoder leverages macOS VideoToolbox
+- **Threading**: Multi-threaded encoding support where available
+- **Real-time processing**: Optimized for live capture and streaming scenarios
+
+### Quality Control
+- **Configurable presets**: H.264 encoding supports Slow, Medium, and Ultrafast presets
+- **Bitrate control**: Intelligent bitrate calculation based on resolution and frame rate
+- **Format-specific optimizations**: Each encoder tuned for its target format characteristics
+
+## Dependencies
+
+- `cap-media-info` - Media stream information structures
+- `ffmpeg` - Cross-platform media processing (video and audio encoding)
+- `gif` - GIF image format encoding
+- `cidre` (macOS only) - AVFoundation bindings for native encoding
+
+## Integration
+
+The crate integrates with:
+- **cap-media-info** for media stream configuration and format definitions
+- **FFmpeg ecosystem** for broad codec and container support
+- **macOS AVFoundation** for optimized native encoding on Apple platforms
+- **Raw media pipelines** for direct frame processing
+
+## Error Handling
+
+Comprehensive error types for each encoder:
+- `H264EncoderError` - Video encoding failures
+- `AACEncoderError` / `OpusEncoderError` - Audio encoding issues
+- `GifEncodingError` - Animation encoding problems
+- `InitError` - Encoder initialization failures
+
+Each error type provides detailed context about failures including codec availability, format compatibility, and resource constraints.
+
+## Use Cases
+
+This crate is designed for applications requiring:
+- **Screen recording** with multiple output formats
+- **Live streaming** with real-time encoding
+- **Video conferencing** with adaptive quality
+- **Media conversion** between different formats
+- **Animation creation** from frame sequences
+
+The modular design allows applications to choose the most appropriate encoder for their specific requirements, whether prioritizing quality, performance, or compatibility.