From c2d8fb6984fcf8a0c0df55ef92ba4ed4aec191df Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sat, 29 Nov 2025 08:54:26 -0600 Subject: [PATCH 01/23] build: enable Werror by default --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index eb27747b1..fcf446ac5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -130,7 +130,7 @@ if(NOT WIN32) if(NOT ${USE_ASAN}) add_compile_definitions(_FORTIFY_SOURCE=2) endif() -add_compile_options(-Wformat -Werror=format-security) +add_compile_options(-Wformat -Werror=format-security -Werror) endif() if(MSVC) add_compile_options(/W4) From 6499b67e5bfe72b81c87583c265b5b6e279e49ac Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sat, 29 Nov 2025 09:12:47 -0600 Subject: [PATCH 02/23] debug: export loglevel for external users --- src/lib/debug.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/lib/debug.c b/src/lib/debug.c index 2a5295f3c..a2dcfe8f8 100644 --- a/src/lib/debug.c +++ b/src/lib/debug.c @@ -1,6 +1,12 @@ #include "internal.h" -ncloglevel_e loglevel = NCLOGLEVEL_SILENT; +#ifndef __MINGW32__ +#define API __attribute__((visibility("default"))) +#else +#define API __declspec(dllexport) +#endif + +API ncloglevel_e loglevel = NCLOGLEVEL_SILENT; void notcurses_debug(const notcurses* nc, FILE* debugfp){ fbuf f; From 03848af32dca1de4fb626cc6183b73aa28d74ed1 Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sat, 29 Nov 2025 08:59:23 -0600 Subject: [PATCH 03/23] audio: add SDL-backed output module --- CMakeLists.txt | 23 +++ src/media/audio-output.c | 400 +++++++++++++++++++++++++++++++++++++++ src/media/audio-output.h | 46 +++++ 3 files changed, 469 insertions(+) create mode 100644 src/media/audio-output.c create mode 100644 src/media/audio-output.h diff --git a/CMakeLists.txt b/CMakeLists.txt index fcf446ac5..e204cb91b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -116,6 +116,7 @@ pkg_check_modules(AVDEVICE REQUIRED libavdevice>=57.0) pkg_check_modules(AVFORMAT REQUIRED libavformat>=57.0) pkg_check_modules(AVUTIL REQUIRED libavutil>=56.0) pkg_check_modules(SWSCALE REQUIRED libswscale>=5.0) +pkg_check_modules(SWRESAMPLE REQUIRED libswresample>=4.0) set_property(GLOBAL APPEND PROPERTY PACKAGES_FOUND FFMpeg) elseif(${USE_OIIO}) pkg_check_modules(OPENIMAGEIO REQUIRED OpenImageIO>=2.1) @@ -320,6 +321,9 @@ endif() ############################################################################ # libnotcurses (multimedia shared library+static library) file(GLOB NCSRCS CONFIGURE_DEPENDS src/media/*.c src/media/*.cpp) +# Exclude audio-output.c from libnotcurses to avoid SDL2 being loaded at library load time +# audio-output.c will be compiled separately for ncplayer only +list(REMOVE_ITEM NCSRCS "${CMAKE_SOURCE_DIR}/src/media/audio-output.c") add_library(notcurses SHARED ${NCSRCS} ${COMPATSRC}) if(${USE_STATIC}) # can't build binaries against static notcurses until ffmpeg linking issues @@ -378,6 +382,7 @@ target_include_directories(notcurses "${AVFORMAT_INCLUDE_DIRS}" "${AVUTIL_INCLUDE_DIRS}" "${SWSCALE_INCLUDE_DIRS}" + "${SWRESAMPLE_INCLUDE_DIRS}" ) target_include_directories(notcurses-static PRIVATE @@ -386,6 +391,7 @@ target_include_directories(notcurses-static "${AVFORMAT_STATIC_INCLUDE_DIRS}" "${AVUTIL_STATIC_INCLUDE_DIRS}" "${SWSCALE_STATIC_INCLUDE_DIRS}" + "${SWRESAMPLE_STATIC_INCLUDE_DIRS}" ) target_link_libraries(notcurses PRIVATE @@ -393,6 +399,7 @@ target_link_libraries(notcurses "${AVDEVICE_LIBRARIES}" "${AVFORMAT_LIBRARIES}" "${SWSCALE_LIBRARIES}" + "${SWRESAMPLE_LIBRARIES}" "${AVUTIL_LIBRARIES}" ) target_link_libraries(notcurses-static @@ -401,6 +408,7 @@ target_link_libraries(notcurses-static "${AVDEVICE_STATIC_LIBRARIES}" "${AVFORMAT_STATIC_LIBRARIES}" "${SWSCALE_STATIC_LIBRARIES}" + "${SWRESAMPLE_STATIC_LIBRARIES}" "${AVUTIL_STATIC_LIBRARIES}" ) target_link_directories(notcurses @@ -409,6 +417,7 @@ target_link_directories(notcurses "${AVDEVICE_LIBRARY_DIRS}" "${AVFORMAT_LIBRARY_DIRS}" "${SWSCALE_LIBRARY_DIRS}" + "${SWRESAMPLE_LIBRARY_DIRS}" "${AVUTIL_LIBRARY_DIRS}" ) target_link_directories(notcurses-static @@ -417,6 +426,7 @@ target_link_directories(notcurses-static "${AVDEVICE_STATIC_LIBRARY_DIRS}" "${AVFORMAT_STATIC_LIBRARY_DIRS}" "${SWSCALE_STATIC_LIBRARY_DIRS}" + "${SWRESAMPLE_STATIC_LIBRARY_DIRS}" "${AVUTIL_STATIC_LIBRARY_DIRS}" ) elseif(${USE_OIIO}) @@ -851,10 +861,23 @@ target_include_directories(ncplayer "${CMAKE_REQUIRED_INCLUDES}" "${PROJECT_BINARY_DIR}/include" ) +if(${USE_FFMPEG}) +target_sources(ncplayer PRIVATE "${CMAKE_SOURCE_DIR}/src/media/audio-output.c") +endif() target_link_libraries(ncplayer PRIVATE notcurses++ + notcurses ) +if(${USE_FFMPEG}) +target_link_libraries(ncplayer + PRIVATE + "${AVCODEC_LIBRARIES}" + "${AVFORMAT_LIBRARIES}" + "${AVUTIL_LIBRARIES}" + "${SWRESAMPLE_LIBRARIES}" +) +endif() endif() endif() else() diff --git a/src/media/audio-output.c b/src/media/audio-output.c new file mode 100644 index 000000000..0abc76313 --- /dev/null +++ b/src/media/audio-output.c @@ -0,0 +1,400 @@ +#include "builddef.h" +#ifdef USE_FFMPEG + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "lib/internal.h" + +// Define API macro for visibility export +#ifndef __MINGW32__ +#define API __attribute__((visibility("default"))) +#else +#define API __declspec(dllexport) +#endif + +#define AUDIO_BUFFER_SIZE 4096 + +// SDL2 types - defined locally to avoid including SDL2/SDL.h (which would cause SDL2 to be linked) +typedef uint32_t SDL_AudioDeviceID; +typedef struct SDL_AudioSpec { + int freq; + uint16_t format; + uint8_t channels; + uint8_t silence; + uint16_t samples; + uint16_t padding; + uint32_t size; + void (*callback)(void* userdata, uint8_t* stream, int len); + void* userdata; +} SDL_AudioSpec; + +#define SDL_INIT_AUDIO 0x00000010 +#define SDL_AUDIO_ALLOW_FREQUENCY_CHANGE 0x00000001 +#define SDL_AUDIO_ALLOW_CHANNELS_CHANGE 0x00000002 +#define AUDIO_S16SYS 0x8010 + +// SDL2 function pointers - loaded dynamically to avoid loading SDL2 at library load time +typedef struct { + void* handle; + int (*SDL_Init)(uint32_t); + void (*SDL_QuitSubSystem)(uint32_t); + const char* (*SDL_GetError)(void); + SDL_AudioDeviceID (*SDL_OpenAudioDevice)(const char*, int, const SDL_AudioSpec*, SDL_AudioSpec*, int); + void (*SDL_PauseAudioDevice)(SDL_AudioDeviceID, int); + void (*SDL_CloseAudioDevice)(SDL_AudioDeviceID); + int (*SDL_SetHint)(const char*, const char*); +} sdl2_functions; + +static sdl2_functions sdl2 = {0}; + +typedef struct audio_output { + SDL_AudioDeviceID device_id; + SDL_AudioSpec want_spec; + SDL_AudioSpec have_spec; + uint8_t* buffer; + size_t buffer_size; + size_t buffer_pos; + size_t buffer_used; + pthread_mutex_t mutex; + pthread_cond_t cond; + bool playing; + bool paused; + double audio_clock; // Current audio position in seconds + uint64_t audio_pts; // Audio PTS in timebase units + int sample_rate; + int channels; + int sample_fmt; // AVSampleFormat (using int to avoid include issues) +} audio_output; + +static audio_output* g_audio = NULL; +static bool sdl_initialized = false; + +static void audio_callback(void* userdata, uint8_t* stream, int len) { + static int callback_count = 0; + callback_count++; + audio_output* ao = (audio_output*)userdata; + if (!ao || !ao->playing) { + memset(stream, 0, len); + if(callback_count <= 10){ + FILE* logfile = fopen("/tmp/ncplayer_audio.log", "a"); + if(logfile){ + fprintf(logfile, "audio_callback: Not playing (call %d, ao=%p, playing=%d)\n", callback_count, ao, ao ? ao->playing : 0); + fclose(logfile); + } + } + return; + } + + if(callback_count <= 10 || callback_count % 1000 == 0){ + FILE* logfile = fopen("/tmp/ncplayer_audio.log", "a"); + if(logfile){ + fprintf(logfile, "audio_callback: Call %d, len=%d, buffer_used=%zu\n", callback_count, len, ao->buffer_used); + fclose(logfile); + } + } + + pthread_mutex_lock(&ao->mutex); + + size_t bytes_to_write = len; + size_t bytes_available = ao->buffer_used; + + if (bytes_available == 0) { + // Buffer underrun - fill with silence + memset(stream, 0, len); + pthread_mutex_unlock(&ao->mutex); + return; + } + + if (bytes_to_write > bytes_available) { + bytes_to_write = bytes_available; + } + + memcpy(stream, ao->buffer + ao->buffer_pos, bytes_to_write); + ao->buffer_pos += bytes_to_write; + ao->buffer_used -= bytes_to_write; + + // Keep buffer data contiguous at the front to simplify writes + if (ao->buffer_used > 0 && ao->buffer_pos > 0) { + memmove(ao->buffer, ao->buffer + ao->buffer_pos, ao->buffer_used); + } + ao->buffer_pos = 0; + + // Update audio clock based on bytes written + if (ao->sample_rate > 0 && ao->channels > 0) { + double samples_written = bytes_to_write / (ao->channels * sizeof(int16_t)); + ao->audio_clock += samples_written / ao->sample_rate; + } + + pthread_cond_signal(&ao->cond); + pthread_mutex_unlock(&ao->mutex); + + // Fill remaining with silence if needed + if (bytes_to_write < (size_t)len) { + memset(stream + bytes_to_write, 0, len - bytes_to_write); + } +} + +// Load SDL2 dynamically to avoid loading it at program startup +static int load_sdl2(void) { + if (sdl2.handle) { + return 0; // Already loaded + } + + sdl2.handle = dlopen("libSDL2-2.0.so.0", RTLD_LAZY); + if (!sdl2.handle) { + sdl2.handle = dlopen("libSDL2.so", RTLD_LAZY); + } + if (!sdl2.handle) { + logerror("Failed to load SDL2: %s", dlerror()); + return -1; + } + + // Load SDL2 functions + sdl2.SDL_SetHint = (int (*)(const char*, const char*))dlsym(sdl2.handle, "SDL_SetHint"); + sdl2.SDL_Init = (int (*)(uint32_t))dlsym(sdl2.handle, "SDL_Init"); + sdl2.SDL_GetError = (const char* (*)(void))dlsym(sdl2.handle, "SDL_GetError"); + sdl2.SDL_OpenAudioDevice = (SDL_AudioDeviceID (*)(const char*, int, const SDL_AudioSpec*, SDL_AudioSpec*, int))dlsym(sdl2.handle, "SDL_OpenAudioDevice"); + sdl2.SDL_PauseAudioDevice = (void (*)(SDL_AudioDeviceID, int))dlsym(sdl2.handle, "SDL_PauseAudioDevice"); + sdl2.SDL_CloseAudioDevice = (void (*)(SDL_AudioDeviceID))dlsym(sdl2.handle, "SDL_CloseAudioDevice"); + sdl2.SDL_QuitSubSystem = (void (*)(uint32_t))dlsym(sdl2.handle, "SDL_QuitSubSystem"); + + if (!sdl2.SDL_Init || !sdl2.SDL_GetError || !sdl2.SDL_OpenAudioDevice || + !sdl2.SDL_PauseAudioDevice || !sdl2.SDL_CloseAudioDevice || !sdl2.SDL_QuitSubSystem) { + logerror("Failed to load SDL2 functions: %s", dlerror()); + dlclose(sdl2.handle); + sdl2.handle = NULL; + return -1; + } + + return 0; +} + +API audio_output* audio_output_init(int sample_rate, int channels, int sample_fmt) { + // Load SDL2 dynamically (only when audio is actually needed) + if (load_sdl2() < 0) { + return NULL; + } + + // Initialize SDL2 audio subsystem only once + // Use SDL_INIT_AUDIO only (not SDL_INIT_EVERYTHING) to avoid interfering with terminal + if (!sdl_initialized) { + // Don't let SDL2 install signal handlers that might interfere with terminal + if (sdl2.SDL_SetHint) { + sdl2.SDL_SetHint("SDL_HINT_NO_SIGNAL_HANDLERS", "1"); + } + if (sdl2.SDL_Init(SDL_INIT_AUDIO) < 0) { + logerror("SDL audio init failed: %s", sdl2.SDL_GetError()); + return NULL; + } + sdl_initialized = true; + } + + audio_output* ao = calloc(1, sizeof(audio_output)); + if (!ao) { + return NULL; + } + + ao->sample_rate = sample_rate; + ao->channels = channels; + ao->sample_fmt = sample_fmt; + + memset(&ao->want_spec, 0, sizeof(ao->want_spec)); + ao->want_spec.freq = sample_rate; + ao->want_spec.format = AUDIO_S16SYS; // Signed 16-bit samples, system byte order + ao->want_spec.channels = channels; + ao->want_spec.samples = AUDIO_BUFFER_SIZE; + ao->want_spec.callback = audio_callback; + ao->want_spec.userdata = ao; + + ao->device_id = sdl2.SDL_OpenAudioDevice(NULL, 0, &ao->want_spec, &ao->have_spec, + SDL_AUDIO_ALLOW_FREQUENCY_CHANGE | SDL_AUDIO_ALLOW_CHANNELS_CHANGE); + if (ao->device_id == 0) { + logerror("SDL_OpenAudioDevice failed: %s", sdl2.SDL_GetError()); + free(ao); + return NULL; + } + + // Update to actual specs + ao->sample_rate = ao->have_spec.freq; + ao->channels = ao->have_spec.channels; + + ao->buffer_size = sample_rate * channels * sizeof(int16_t) * 2; // 2 seconds buffer + ao->buffer = malloc(ao->buffer_size); + if (!ao->buffer) { + sdl2.SDL_CloseAudioDevice(ao->device_id); + free(ao); + return NULL; + } + + if (pthread_mutex_init(&ao->mutex, NULL) != 0) { + free(ao->buffer); + sdl2.SDL_CloseAudioDevice(ao->device_id); + free(ao); + return NULL; + } + + if (pthread_cond_init(&ao->cond, NULL) != 0) { + pthread_mutex_destroy(&ao->mutex); + free(ao->buffer); + sdl2.SDL_CloseAudioDevice(ao->device_id); + free(ao); + return NULL; + } + + ao->playing = false; + ao->paused = false; + ao->buffer_pos = 0; + ao->buffer_used = 0; + ao->audio_clock = 0.0; + + g_audio = ao; + return ao; +} + +API void audio_output_start(audio_output* ao) { + if (!ao) return; + ao->playing = true; + ao->paused = false; + if(sdl2.SDL_PauseAudioDevice){ + sdl2.SDL_PauseAudioDevice(ao->device_id, 0); + FILE* logfile = fopen("/tmp/ncplayer_audio.log", "a"); + if(logfile){ + fprintf(logfile, "audio_output_start: SDL_PauseAudioDevice called, device_id=%u\n", ao->device_id); + fclose(logfile); + } + } +} + +API void audio_output_pause(audio_output* ao) { + if (!ao) return; + ao->paused = true; + sdl2.SDL_PauseAudioDevice(ao->device_id, 1); +} + +API void audio_output_resume(audio_output* ao) { + if (!ao) return; + ao->paused = false; + sdl2.SDL_PauseAudioDevice(ao->device_id, 0); +} + +API int audio_output_write(audio_output* ao, const uint8_t* data, size_t len) { + if (!ao || !data) return -1; + + pthread_mutex_lock(&ao->mutex); + + // Wait if buffer is too full + size_t available_space = ao->buffer_size - ao->buffer_used; + while (len > available_space) { + pthread_cond_wait(&ao->cond, &ao->mutex); + if (!ao->playing) { + pthread_mutex_unlock(&ao->mutex); + return -1; + } + available_space = ao->buffer_size - ao->buffer_used; + } + + // Ensure data is contiguous at buffer start before writing + if (ao->buffer_pos > 0 && ao->buffer_used > 0) { + memmove(ao->buffer, ao->buffer + ao->buffer_pos, ao->buffer_used); + ao->buffer_pos = 0; + } + + memcpy(ao->buffer + ao->buffer_used, data, len); + ao->buffer_used += len; + + pthread_mutex_unlock(&ao->mutex); + return 0; +} + +API double audio_output_get_clock(audio_output* ao) { + if (!ao) return 0.0; + pthread_mutex_lock(&ao->mutex); + double clock = ao->audio_clock; + pthread_mutex_unlock(&ao->mutex); + return clock; +} + +API void audio_output_set_pts(audio_output* ao, uint64_t pts, double time_base) { + if (!ao) return; + pthread_mutex_lock(&ao->mutex); + ao->audio_pts = pts; + ao->audio_clock = pts * time_base; + pthread_mutex_unlock(&ao->mutex); +} + +API void audio_output_flush(audio_output* ao) { + if (!ao) return; + pthread_mutex_lock(&ao->mutex); + ao->buffer_pos = 0; + ao->buffer_used = 0; + ao->audio_clock = 0.0; + pthread_mutex_unlock(&ao->mutex); +} + +API void audio_output_destroy(audio_output* ao) { + if (!ao) return; + + ao->playing = false; + pthread_cond_broadcast(&ao->cond); + + sdl2.SDL_CloseAudioDevice(ao->device_id); + pthread_mutex_destroy(&ao->mutex); + pthread_cond_destroy(&ao->cond); + free(ao->buffer); + free(ao); + + if (g_audio == ao) { + g_audio = NULL; + } + + // Don't quit SDL subsystem here - it might be used by other instances + // Only quit if this was the last instance + if (g_audio == NULL && sdl2.handle) { + sdl2.SDL_QuitSubSystem(SDL_INIT_AUDIO); + sdl_initialized = false; + dlclose(sdl2.handle); + memset(&sdl2, 0, sizeof(sdl2)); + } +} + +API audio_output* audio_output_get_global(void) { + return g_audio; +} + +API bool audio_output_needs_data(audio_output* ao) { + if (!ao) return false; + pthread_mutex_lock(&ao->mutex); + // Need more data if buffer is less than 50% full + bool needs_data = (ao->buffer_used < ao->buffer_size / 2); + pthread_mutex_unlock(&ao->mutex); + return needs_data; +} + +API int audio_output_get_sample_rate(audio_output* ao) { + if (!ao) return 0; + pthread_mutex_lock(&ao->mutex); + int rate = ao->sample_rate; + pthread_mutex_unlock(&ao->mutex); + return rate; +} + +API int audio_output_get_channels(audio_output* ao) { + if (!ao) return 0; + pthread_mutex_lock(&ao->mutex); + int ch = ao->channels; + pthread_mutex_unlock(&ao->mutex); + return ch; +} + +#endif // USE_FFMPEG + diff --git a/src/media/audio-output.h b/src/media/audio-output.h new file mode 100644 index 000000000..08e8f84db --- /dev/null +++ b/src/media/audio-output.h @@ -0,0 +1,46 @@ +#ifndef AUDIO_OUTPUT_H +#define AUDIO_OUTPUT_H + +#include "builddef.h" +#ifdef USE_FFMPEG + +#include +#include +#include + +// Define API macro for visibility export +#ifndef __MINGW32__ +#define API __attribute__((visibility("default"))) +#else +#define API __declspec(dllexport) +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct audio_output audio_output; + +// sample_fmt is AVSampleFormat from libavutil, but we use int to avoid include issues +API audio_output* audio_output_init(int sample_rate, int channels, int sample_fmt); +API void audio_output_start(audio_output* ao); +API void audio_output_pause(audio_output* ao); +API void audio_output_resume(audio_output* ao); +API int audio_output_write(audio_output* ao, const uint8_t* data, size_t len); +API double audio_output_get_clock(audio_output* ao); +API void audio_output_set_pts(audio_output* ao, uint64_t pts, double time_base); +API void audio_output_flush(audio_output* ao); +API void audio_output_destroy(audio_output* ao); +API audio_output* audio_output_get_global(void); +// Check if audio buffer needs more data (returns true if buffer is less than 50% full) +API bool audio_output_needs_data(audio_output* ao); +API int audio_output_get_sample_rate(audio_output* ao); +API int audio_output_get_channels(audio_output* ao); + +#ifdef __cplusplus +} +#endif + +#endif // USE_FFMPEG +#endif // AUDIO_OUTPUT_H + From 4ce586d8b049a83a7000acf21f40d8c26185e3a1 Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sat, 29 Nov 2025 09:05:35 -0600 Subject: [PATCH 04/23] ffmpeg: add audio queue and resampler helpers --- src/media/ffmpeg.c | 574 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 572 insertions(+), 2 deletions(-) diff --git a/src/media/ffmpeg.c b/src/media/ffmpeg.c index 20ca50e1d..6fa9a240c 100644 --- a/src/media/ffmpeg.c +++ b/src/media/ffmpeg.c @@ -1,5 +1,8 @@ #include "builddef.h" #ifdef USE_FFMPEG +#include +#include +#include #include #include #include @@ -9,35 +12,160 @@ #include #include #include +#include +#include #include #include #include +#include +#include #include "lib/visual-details.h" #include "lib/internal.h" struct AVFormatContext; + +// Simple audio logging function - writes to /tmp/ncplayer_audio.log +static void audio_log(const char* fmt, ...){ + FILE* logfile = fopen("/tmp/ncplayer_audio.log", "a"); + if(!logfile){ + return; + } + va_list args; + va_start(args, fmt); + vfprintf(logfile, fmt, args); + va_end(args); + fflush(logfile); + fclose(logfile); +} struct AVCodecContext; struct AVFrame; struct AVCodec; struct AVCodecParameters; struct AVPacket; +#define AUDIO_PACKET_QUEUE_SIZE 8 + +typedef struct audio_packet_queue { + AVPacket* packets[AUDIO_PACKET_QUEUE_SIZE]; + int head; + int tail; + int count; +} audio_packet_queue; + typedef struct ncvisual_details { struct AVFormatContext* fmtctx; struct AVCodecContext* codecctx; // video codec context struct AVCodecContext* subtcodecctx; // subtitle codec context + struct AVCodecContext* audiocodecctx; // audio codec context struct AVFrame* frame; // frame as read/loaded/converted + struct AVFrame* audio_frame; // decoded audio frame struct AVCodec* codec; struct AVCodec* subtcodec; + struct AVCodec* audiocodec; struct AVPacket* packet; struct SwsContext* swsctx; struct SwsContext* rgbactx; + struct SwrContext* swrctx; // audio resampler context + int audio_out_channels; // output channel count for resampler (1 or 2) AVSubtitle subtitle; int stream_index; // match against this following av_read_frame() int sub_stream_index; // subtitle stream index, can be < 0 if no subtitles + int audio_stream_index; // audio stream index, can be < 0 if no audio bool packet_outstanding; + bool audio_packet_outstanding; // whether we have an audio packet waiting to be decoded + audio_packet_queue pending_audio_packets; // audio packets waiting to be sent + int64_t last_audio_frame_pts; // PTS of last processed audio frame (to prevent reprocessing) + pthread_mutex_t packet_mutex; // mutex for thread-safe packet reading + pthread_mutex_t audio_packet_mutex; // mutex for audio packet queue } ncvisual_details; +#define AUDIO_LOG_QUEUE_OPS 1 + +static void +audio_queue_init(audio_packet_queue* q){ + memset(q, 0, sizeof(*q)); +} + +static void +audio_queue_clear(audio_packet_queue* q){ + while(q->count > 0){ + av_packet_free(&q->packets[q->head]); + q->packets[q->head] = NULL; + q->head = (q->head + 1) % AUDIO_PACKET_QUEUE_SIZE; + q->count--; + } + q->head = q->tail = 0; +} + +static int +audio_queue_enqueue(audio_packet_queue* q, const AVPacket* pkt){ + if(q->count >= AUDIO_PACKET_QUEUE_SIZE){ + return -1; + } + AVPacket* copy = av_packet_alloc(); + if(!copy){ + return -1; + } + if(av_packet_ref(copy, pkt) < 0){ + av_packet_free(©); + return -1; + } + q->packets[q->tail] = copy; + q->tail = (q->tail + 1) % AUDIO_PACKET_QUEUE_SIZE; + q->count++; + return 0; +} + +static AVPacket* +audio_queue_peek(audio_packet_queue* q){ + if(q->count == 0){ + return NULL; + } + return q->packets[q->head]; +} + +static void +audio_queue_pop(audio_packet_queue* q){ + if(q->count == 0){ + return; + } + av_packet_free(&q->packets[q->head]); + q->packets[q->head] = NULL; + q->head = (q->head + 1) % AUDIO_PACKET_QUEUE_SIZE; + q->count--; +} + +static void +ffmpeg_drain_pending_audio_locked(ncvisual_details* deets){ + if(!deets->audiocodecctx){ + audio_queue_clear(&deets->pending_audio_packets); + deets->audio_packet_outstanding = false; + return; + } + while(deets->pending_audio_packets.count > 0){ + AVPacket* pkt = audio_queue_peek(&deets->pending_audio_packets); + if(!pkt){ + break; + } + int ret = avcodec_send_packet(deets->audiocodecctx, pkt); + if(ret == 0){ + audio_queue_pop(&deets->pending_audio_packets); + deets->audio_packet_outstanding = true; + continue; + }else if(ret == AVERROR(EAGAIN)){ + // Decoder still full; keep outstanding flag set because queue still populated. + deets->audio_packet_outstanding = true; + return; + }else{ + // Drop problematic packet and keep trying remaining ones. + audio_queue_pop(&deets->pending_audio_packets); + } + } + if(deets->pending_audio_packets.count == 0){ + deets->audio_packet_outstanding = false; + } +} + #define IMGALLOCALIGN 64 uint64_t ffmpeg_pkt_duration(const AVFrame* frame){ @@ -322,13 +450,17 @@ ffmpeg_decode(ncvisual* n){ av_packet_unref(n->details->packet); } int averr; - if((averr = av_read_frame(n->details->fmtctx, n->details->packet)) < 0){ + // Single point for reading packets - no mutex needed as only video decoder reads + averr = av_read_frame(n->details->fmtctx, n->details->packet); + if(averr < 0){ /*if(averr != AVERROR_EOF){ fprintf(stderr, "Error reading frame info (%s)\n", av_err2str(averr)); }*/ return averr2ncerr(averr); } unref = true; + + // Handle subtitle packets if(n->details->packet->stream_index == n->details->sub_stream_index){ int result = 0, ret; avsubtitle_free(&n->details->subtitle); @@ -336,8 +468,75 @@ ffmpeg_decode(ncvisual* n){ if(ret >= 0 && result){ // FIXME? } + continue; // Continue reading, this wasn't a video packet } - }while(n->details->packet->stream_index != n->details->stream_index); + + // Handle audio packets - send them to audio decoder (audio thread will receive frames) + if(n->details->packet->stream_index == n->details->audio_stream_index){ + if(n->details->audiocodecctx){ + // Validate packet and codec context before sending + if(!n->details->packet || !n->details->audiocodecctx){ + av_packet_unref(n->details->packet); + continue; + } + + // Check packet validity - must have data and size + if(n->details->packet->size <= 0 || !n->details->packet->data){ + // Empty or invalid packet - skip it + av_packet_unref(n->details->packet); + continue; + } + + static int audio_packet_count = 0; + audio_packet_count++; + + pthread_mutex_lock(&n->details->audio_packet_mutex); + ffmpeg_drain_pending_audio_locked(n->details); + + // Now try to send the new packet + int audio_ret = avcodec_send_packet(n->details->audiocodecctx, n->details->packet); + if(audio_ret == 0){ + n->details->audio_packet_outstanding = true; + if(audio_packet_count <= 10 || audio_packet_count % 100 == 0){ + audio_log("ffmpeg_decode: Sent audio packet %d successfully\n", audio_packet_count); + } + av_packet_unref(n->details->packet); + }else if(audio_ret == AVERROR(EAGAIN)){ + // Decoder full - save this packet for next time + if(audio_queue_enqueue(&n->details->pending_audio_packets, n->details->packet) == 0){ + n->details->audio_packet_outstanding = true; + if(audio_packet_count <= 10 || audio_packet_count % 100 == 0){ + audio_log("ffmpeg_decode: Audio decoder full (EAGAIN), queuing packet %d (queued=%d)\n", + audio_packet_count, n->details->pending_audio_packets.count); + } + }else{ + // Already have pending packet - drop this one (queue full) + if(audio_packet_count <= 10 || audio_packet_count % 100 == 0){ + audio_log("ffmpeg_decode: Audio queue full, dropping packet %d\n", audio_packet_count); + } + } + av_packet_unref(n->details->packet); + }else if(audio_ret == AVERROR_EOF){ + // EOF - this is normal at end of stream + av_packet_unref(n->details->packet); + }else{ + // Error - log it but don't crash + if(audio_packet_count <= 10){ + audio_log("ffmpeg_decode: Error sending audio packet %d: %d (skipping)\n", audio_packet_count, audio_ret); + } + av_packet_unref(n->details->packet); + } + + pthread_mutex_unlock(&n->details->audio_packet_mutex); + }else{ + av_packet_unref(n->details->packet); + } + continue; // Continue reading, this wasn't a video packet + } + + // This is a video packet - break out of loop to process it + break; + }while(1); n->details->packet_outstanding = true; int averr = avcodec_send_packet(n->details->codecctx, n->details->packet); if(averr < 0){ @@ -378,7 +577,27 @@ ffmpeg_details_init(void){ memset(deets, 0, sizeof(*deets)); deets->stream_index = -1; deets->sub_stream_index = -1; + deets->audio_stream_index = -1; + deets->last_audio_frame_pts = AV_NOPTS_VALUE; // No frame processed yet + deets->audio_out_channels = 0; // Will be set when resampler is initialized + audio_queue_init(&deets->pending_audio_packets); + if(pthread_mutex_init(&deets->packet_mutex, NULL) != 0){ + free(deets); + return NULL; + } + if(pthread_mutex_init(&deets->audio_packet_mutex, NULL) != 0){ + pthread_mutex_destroy(&deets->packet_mutex); + free(deets); + return NULL; + } if((deets->frame = av_frame_alloc()) == NULL){ + pthread_mutex_destroy(&deets->packet_mutex); + free(deets); + return NULL; + } + if((deets->audio_frame = av_frame_alloc()) == NULL){ + av_frame_free(&deets->frame); + pthread_mutex_destroy(&deets->packet_mutex); free(deets); return NULL; } @@ -437,6 +656,27 @@ ffmpeg_from_file(const char* filename){ }else{ ncv->details->sub_stream_index = -1; } + // Find audio stream (similar to subtitle detection) + if((averr = av_find_best_stream(ncv->details->fmtctx, AVMEDIA_TYPE_AUDIO, -1, -1, +#if LIBAVFORMAT_VERSION_MAJOR >= 59 + (const AVCodec**)&ncv->details->audiocodec, 0)) >= 0){ +#else + &ncv->details->audiocodec, 0)) >= 0){ +#endif + ncv->details->audio_stream_index = averr; + AVStream* ast = ncv->details->fmtctx->streams[ncv->details->audio_stream_index]; + if((ncv->details->audiocodecctx = avcodec_alloc_context3(ncv->details->audiocodec)) == NULL){ + goto err; + } + if(avcodec_parameters_to_context(ncv->details->audiocodecctx, ast->codecpar) < 0){ + goto err; + } + if(avcodec_open2(ncv->details->audiocodecctx, ncv->details->audiocodec, NULL) < 0){ + goto err; + } + }else{ + ncv->details->audio_stream_index = -1; + } //fprintf(stderr, "FRAME FRAME: %p\n", ncv->details->frame); if((ncv->details->packet = av_packet_alloc()) == NULL){ // fprintf(stderr, "Couldn't allocate packet for %s\n", filename); @@ -481,6 +721,119 @@ ffmpeg_from_file(const char* filename){ return NULL; } +// Decode audio packet and return number of samples decoded +// Returns: >0 = samples decoded, 0 = need more data, <0 = error +// This function is safe to call - it doesn't interfere with video decoding +static int +ffmpeg_decode_audio_internal(ncvisual* ncv, AVPacket* packet){ + if(!ncv->details->audiocodecctx || ncv->details->audio_stream_index < 0){ + return -1; + } + + // Send packet to decoder if provided + if(packet && packet->stream_index == ncv->details->audio_stream_index){ + int averr = avcodec_send_packet(ncv->details->audiocodecctx, packet); + if(averr < 0 && averr != AVERROR(EAGAIN) && averr != AVERROR_EOF){ + return averr2ncerr(averr); + } + ncv->details->audio_packet_outstanding = (averr == 0); + } + + // If no packet outstanding, return 0 (need more data) + if(!ncv->details->audio_packet_outstanding){ + return 0; + } + + // Receive decoded frame + int averr = avcodec_receive_frame(ncv->details->audiocodecctx, ncv->details->audio_frame); + if(averr == 0){ + ncv->details->audio_packet_outstanding = false; + return ncv->details->audio_frame->nb_samples; + }else if(averr == AVERROR(EAGAIN)){ + return 0; // need more packets + }else if(averr == AVERROR_EOF){ + return 1; // EOF + }else{ + return averr2ncerr(averr); + } +} + +// Initialize audio resampler for converting to output format +// Returns 0 on success, <0 on error +// This function is safe to call - it doesn't interfere with video decoding +static int +ffmpeg_init_audio_resampler_internal(ncvisual* ncv, int out_sample_rate, int out_channels){ + if(!ncv->details->audiocodecctx || ncv->details->audio_stream_index < 0){ + return -1; + } + + AVCodecContext* acodecctx = ncv->details->audiocodecctx; + + // Free existing resampler if any + if(ncv->details->swrctx){ + swr_free(&ncv->details->swrctx); + } + + // Use older API with channel masks (more stable) + uint64_t in_channel_mask = 0; + uint64_t out_channel_mask = 0; + + // Get input channel count + int in_ch_count = 0; + if(acodecctx->ch_layout.nb_channels > 0){ + in_ch_count = acodecctx->ch_layout.nb_channels; + }else{ + // Fallback for older FFmpeg versions + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" + in_ch_count = acodecctx->channels; + #pragma GCC diagnostic pop + } + + // Map channel count to standard layout masks + if(in_ch_count == 1){ + in_channel_mask = AV_CH_LAYOUT_MONO; + }else if(in_ch_count == 2){ + in_channel_mask = AV_CH_LAYOUT_STEREO; + }else if(in_ch_count == 6){ + in_channel_mask = AV_CH_LAYOUT_5POINT1; + }else if(in_ch_count == 4){ + in_channel_mask = AV_CH_LAYOUT_QUAD; + }else if(in_ch_count == 8){ + in_channel_mask = AV_CH_LAYOUT_7POINT1; + }else{ + // Default to stereo for unknown channel counts + in_channel_mask = AV_CH_LAYOUT_STEREO; + in_ch_count = 2; + } + + out_channel_mask = (out_channels == 1) ? AV_CH_LAYOUT_MONO : AV_CH_LAYOUT_STEREO; + + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" + ncv->details->swrctx = swr_alloc_set_opts( + NULL, + out_channel_mask, AV_SAMPLE_FMT_S16, out_sample_rate, + in_channel_mask, acodecctx->sample_fmt, acodecctx->sample_rate, + 0, NULL); + #pragma GCC diagnostic pop + + if(!ncv->details->swrctx){ + return -1; + } + + int ret = swr_init(ncv->details->swrctx); + if(ret < 0){ + swr_free(&ncv->details->swrctx); + return -1; + } + + // Store output channel count for buffer size calculations + ncv->details->audio_out_channels = out_channels; + + return 0; +} + // iterate over the decoded frames, calling streamer() with curry for each. // frames carry a presentation time relative to the beginning, so we get an // initial timestamp, and check each frame against the elapsed time to sync @@ -719,13 +1072,20 @@ static void ffmpeg_details_destroy(ncvisual_details* deets){ // avcodec_close() is deprecated; avcodec_free_context() suffices avcodec_free_context(&deets->subtcodecctx); + avcodec_free_context(&deets->audiocodecctx); avcodec_free_context(&deets->codecctx); av_frame_free(&deets->frame); + av_frame_free(&deets->audio_frame); + audio_queue_clear(&deets->pending_audio_packets); + av_frame_free(&deets->audio_frame); + swr_free(&deets->swrctx); sws_freeContext(deets->rgbactx); sws_freeContext(deets->swsctx); av_packet_free(&deets->packet); avformat_close_input(&deets->fmtctx); avsubtitle_free(&deets->subtitle); + pthread_mutex_destroy(&deets->audio_packet_mutex); + pthread_mutex_destroy(&deets->packet_mutex); free(deets); } @@ -740,6 +1100,216 @@ ffmpeg_destroy(ncvisual* ncv){ } } +// Public API functions for audio handling (called from play.cpp) +// Define API macro for visibility export +#ifndef __MINGW32__ +#define API __attribute__((visibility("default"))) +#else +#define API __declspec(dllexport) +#endif + +API void +ffmpeg_audio_request_packets(ncvisual* ncv){ + if(!ncv || !ncv->details){ + return; + } + pthread_mutex_lock(&ncv->details->audio_packet_mutex); + ffmpeg_drain_pending_audio_locked(ncv->details); + pthread_mutex_unlock(&ncv->details->audio_packet_mutex); +} + +// Check if the visual has an audio stream +API bool +ffmpeg_has_audio(ncvisual* ncv){ + return ncv && ncv->details && ncv->details->audio_stream_index >= 0 && + ncv->details->audiocodecctx != NULL; +} + +// Decode audio packet (public wrapper) +API int +ffmpeg_decode_audio(ncvisual* ncv, AVPacket* packet){ + return ffmpeg_decode_audio_internal(ncv, packet); +} + +// Initialize audio resampler (public wrapper) +API int +ffmpeg_init_audio_resampler(ncvisual* ncv, int out_sample_rate, int out_channels){ + return ffmpeg_init_audio_resampler_internal(ncv, out_sample_rate, out_channels); +} + +// Try to get a decoded audio frame (packets are read by video decoder) +// Returns: >0 = samples decoded, 0 = no frame available, <0 = error +// This function is called by the audio thread - it doesn't read packets +// IMPORTANT: avcodec_receive_frame can only return a frame once per packet sent +API int +ffmpeg_get_decoded_audio_frame(ncvisual* ncv){ + if(!ncv || !ncv->details || ncv->details->audio_stream_index < 0){ + return -1; + } + + // Try to receive a decoded frame (non-blocking) + // This will only return a frame once per packet - subsequent calls return EAGAIN + static int receive_call_count = 0; + receive_call_count++; + int pending_after = 0; + bool outstanding = false; + pthread_mutex_lock(&ncv->details->audio_packet_mutex); + int averr = avcodec_receive_frame(ncv->details->audiocodecctx, ncv->details->audio_frame); + pending_after = ncv->details->pending_audio_packets.count; + outstanding = ncv->details->audio_packet_outstanding; + if(averr == 0){ + static int frame_counter = 0; + frame_counter++; + if(pending_after == 0){ + ncv->details->audio_packet_outstanding = false; + outstanding = false; + } + pthread_mutex_unlock(&ncv->details->audio_packet_mutex); + if(receive_call_count <= 20 || receive_call_count % 100 == 0 || frame_counter % 50 == 0){ + if(frame_counter <= 20 || frame_counter % 200 == 0){ + audio_log("ffmpeg_get_decoded_audio_frame: Frame received, pending queue=%d, total_frames=%d\n", + pending_after, frame_counter); + } + } + + // Check if this is a new frame (different PTS) or the same one we already processed + int64_t current_pts = ncv->details->audio_frame->pts; + if(current_pts == ncv->details->last_audio_frame_pts && current_pts != AV_NOPTS_VALUE){ + // Same frame as before - don't process again + // This can happen if avcodec_receive_frame is called multiple times + // Note: avcodec_receive_frame should only return each frame once, so this + // check is defensive. If we see this frequently, there's a bug elsewhere. + if(receive_call_count <= 20){ + audio_log("ffmpeg_get_decoded_audio_frame: Duplicate PTS detected (call %d)\n", receive_call_count); + } + return 0; + } + ncv->details->last_audio_frame_pts = current_pts; + if(receive_call_count <= 20 || receive_call_count % 100 == 0){ + audio_log("ffmpeg_get_decoded_audio_frame: Received frame (call %d, samples=%d, pts=%" PRId64 ")\n", + receive_call_count, ncv->details->audio_frame->nb_samples, current_pts); + } + return ncv->details->audio_frame->nb_samples; + }else if(averr == AVERROR(EAGAIN)){ + pthread_mutex_unlock(&ncv->details->audio_packet_mutex); + // Need more packets - video decoder will provide them + // This is normal - the decoder needs more packets before it can produce a frame + (void)outstanding; + return 0; + }else if(averr == AVERROR_EOF){ + pthread_mutex_unlock(&ncv->details->audio_packet_mutex); + audio_log("ffmpeg_get_decoded_audio_frame: EOF (call %d)\n", receive_call_count); + return 1; // EOF + }else{ + pthread_mutex_unlock(&ncv->details->audio_packet_mutex); + audio_log("ffmpeg_get_decoded_audio_frame: Error %d (call %d)\n", averr, receive_call_count); + return -1; // Error + } +} + +// Resample audio frame to output format +// Returns: number of samples output, <0 on error +// NOTE: This should be called immediately after ffmpeg_get_decoded_audio_frame returns >0 +// The frame data is only valid until the next call to avcodec_receive_frame +API int +ffmpeg_resample_audio(ncvisual* ncv, uint8_t** out_data, int* out_samples){ + if(!ncv || !ncv->details || !ncv->details->swrctx || !ncv->details->audio_frame){ + return -1; + } + + AVFrame* frame = ncv->details->audio_frame; + if(frame->nb_samples <= 0){ + return 0; + } + + // Calculate output buffer size + int64_t out_count = swr_get_out_samples(ncv->details->swrctx, frame->nb_samples); + if(out_count < 0){ + return -1; + } + + // Use OUTPUT channel count (from resampler), not input channel count + int out_channels = ncv->details->audio_out_channels; + if(out_channels <= 0){ + // Fallback: use input channel count, but limit to 2 + out_channels = ncv->details->audiocodecctx->ch_layout.nb_channels; + if(out_channels == 0){ + // Fallback for older FFmpeg + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" + out_channels = ncv->details->audiocodecctx->channels; + #pragma GCC diagnostic pop + } + // Limit to stereo max + if(out_channels > 2){ + out_channels = 2; + } + } + + int out_size = av_samples_get_buffer_size(NULL, out_channels, out_count, + AV_SAMPLE_FMT_S16, 1); + if(out_size < 0){ + return -1; + } + + // Allocate output buffer + *out_data = malloc(out_size); + if(!*out_data){ + return -1; + } + + // Perform resampling + uint8_t* out_ptr = *out_data; + int samples_out = swr_convert(ncv->details->swrctx, &out_ptr, out_count, + (const uint8_t**)frame->data, frame->nb_samples); + + if(samples_out < 0){ + free(*out_data); + *out_data = NULL; + return -1; + } + + *out_samples = samples_out; + return samples_out * out_channels * sizeof(int16_t); // Return bytes +} + +// Get the decoded audio frame (for getting sample rate, channels, etc.) +API AVFrame* +ffmpeg_get_audio_frame(ncvisual* ncv){ + if(!ncv || !ncv->details){ + return NULL; + } + return ncv->details->audio_frame; +} + +// Get audio sample rate from codec context +API int +ffmpeg_get_audio_sample_rate(ncvisual* ncv){ + if(!ncv || !ncv->details || !ncv->details->audiocodecctx){ + return 44100; // Default + } + return ncv->details->audiocodecctx->sample_rate > 0 ? + ncv->details->audiocodecctx->sample_rate : 44100; +} + +// Get audio channel count from codec context +API int +ffmpeg_get_audio_channels(ncvisual* ncv){ + if(!ncv || !ncv->details || !ncv->details->audiocodecctx){ + return 2; // Default stereo + } + AVCodecContext* acodecctx = ncv->details->audiocodecctx; + int channels = acodecctx->ch_layout.nb_channels; + if(channels == 0){ + // Fallback for older FFmpeg + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wdeprecated-declarations" + channels = acodecctx->channels; + #pragma GCC diagnostic pop + } + return channels > 0 ? channels : 2; // Default stereo +} + ncvisual_implementation local_visual_implementation = { .visual_init = ffmpeg_init, .visual_printbanner = ffmpeg_printbanner, From 0f0d2a756c7f5eea0e6f4fa22ad529b548ec0833 Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sat, 29 Nov 2025 09:06:00 -0600 Subject: [PATCH 05/23] ncplayer: drive audio thread and drop lagging frames --- src/media/audio-output.c | 25 +---- src/media/ffmpeg.c | 54 +-------- src/player/play.cpp | 234 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 238 insertions(+), 75 deletions(-) diff --git a/src/media/audio-output.c b/src/media/audio-output.c index 0abc76313..22f61b78d 100644 --- a/src/media/audio-output.c +++ b/src/media/audio-output.c @@ -10,8 +10,9 @@ #include #include #include -#include +#include #include +#include #include #include "lib/internal.h" @@ -80,29 +81,12 @@ static audio_output* g_audio = NULL; static bool sdl_initialized = false; static void audio_callback(void* userdata, uint8_t* stream, int len) { - static int callback_count = 0; - callback_count++; audio_output* ao = (audio_output*)userdata; if (!ao || !ao->playing) { memset(stream, 0, len); - if(callback_count <= 10){ - FILE* logfile = fopen("/tmp/ncplayer_audio.log", "a"); - if(logfile){ - fprintf(logfile, "audio_callback: Not playing (call %d, ao=%p, playing=%d)\n", callback_count, ao, ao ? ao->playing : 0); - fclose(logfile); - } - } return; } - if(callback_count <= 10 || callback_count % 1000 == 0){ - FILE* logfile = fopen("/tmp/ncplayer_audio.log", "a"); - if(logfile){ - fprintf(logfile, "audio_callback: Call %d, len=%d, buffer_used=%zu\n", callback_count, len, ao->buffer_used); - fclose(logfile); - } - } - pthread_mutex_lock(&ao->mutex); size_t bytes_to_write = len; @@ -267,11 +251,6 @@ API void audio_output_start(audio_output* ao) { ao->paused = false; if(sdl2.SDL_PauseAudioDevice){ sdl2.SDL_PauseAudioDevice(ao->device_id, 0); - FILE* logfile = fopen("/tmp/ncplayer_audio.log", "a"); - if(logfile){ - fprintf(logfile, "audio_output_start: SDL_PauseAudioDevice called, device_id=%u\n", ao->device_id); - fclose(logfile); - } } } diff --git a/src/media/ffmpeg.c b/src/media/ffmpeg.c index 6fa9a240c..9a7339580 100644 --- a/src/media/ffmpeg.c +++ b/src/media/ffmpeg.c @@ -2,7 +2,7 @@ #ifdef USE_FFMPEG #include #include -#include +#include #include #include #include @@ -24,19 +24,6 @@ struct AVFormatContext; -// Simple audio logging function - writes to /tmp/ncplayer_audio.log -static void audio_log(const char* fmt, ...){ - FILE* logfile = fopen("/tmp/ncplayer_audio.log", "a"); - if(!logfile){ - return; - } - va_list args; - va_start(args, fmt); - vfprintf(logfile, fmt, args); - va_end(args); - fflush(logfile); - fclose(logfile); -} struct AVCodecContext; struct AVFrame; struct AVCodec; @@ -497,35 +484,13 @@ ffmpeg_decode(ncvisual* n){ int audio_ret = avcodec_send_packet(n->details->audiocodecctx, n->details->packet); if(audio_ret == 0){ n->details->audio_packet_outstanding = true; - if(audio_packet_count <= 10 || audio_packet_count % 100 == 0){ - audio_log("ffmpeg_decode: Sent audio packet %d successfully\n", audio_packet_count); - } - av_packet_unref(n->details->packet); }else if(audio_ret == AVERROR(EAGAIN)){ // Decoder full - save this packet for next time if(audio_queue_enqueue(&n->details->pending_audio_packets, n->details->packet) == 0){ n->details->audio_packet_outstanding = true; - if(audio_packet_count <= 10 || audio_packet_count % 100 == 0){ - audio_log("ffmpeg_decode: Audio decoder full (EAGAIN), queuing packet %d (queued=%d)\n", - audio_packet_count, n->details->pending_audio_packets.count); - } - }else{ - // Already have pending packet - drop this one (queue full) - if(audio_packet_count <= 10 || audio_packet_count % 100 == 0){ - audio_log("ffmpeg_decode: Audio queue full, dropping packet %d\n", audio_packet_count); - } - } - av_packet_unref(n->details->packet); - }else if(audio_ret == AVERROR_EOF){ - // EOF - this is normal at end of stream - av_packet_unref(n->details->packet); - }else{ - // Error - log it but don't crash - if(audio_packet_count <= 10){ - audio_log("ffmpeg_decode: Error sending audio packet %d: %d (skipping)\n", audio_packet_count, audio_ret); } - av_packet_unref(n->details->packet); } + av_packet_unref(n->details->packet); pthread_mutex_unlock(&n->details->audio_packet_mutex); }else{ @@ -1165,12 +1130,6 @@ ffmpeg_get_decoded_audio_frame(ncvisual* ncv){ outstanding = false; } pthread_mutex_unlock(&ncv->details->audio_packet_mutex); - if(receive_call_count <= 20 || receive_call_count % 100 == 0 || frame_counter % 50 == 0){ - if(frame_counter <= 20 || frame_counter % 200 == 0){ - audio_log("ffmpeg_get_decoded_audio_frame: Frame received, pending queue=%d, total_frames=%d\n", - pending_after, frame_counter); - } - } // Check if this is a new frame (different PTS) or the same one we already processed int64_t current_pts = ncv->details->audio_frame->pts; @@ -1179,16 +1138,9 @@ ffmpeg_get_decoded_audio_frame(ncvisual* ncv){ // This can happen if avcodec_receive_frame is called multiple times // Note: avcodec_receive_frame should only return each frame once, so this // check is defensive. If we see this frequently, there's a bug elsewhere. - if(receive_call_count <= 20){ - audio_log("ffmpeg_get_decoded_audio_frame: Duplicate PTS detected (call %d)\n", receive_call_count); - } return 0; } ncv->details->last_audio_frame_pts = current_pts; - if(receive_call_count <= 20 || receive_call_count % 100 == 0){ - audio_log("ffmpeg_get_decoded_audio_frame: Received frame (call %d, samples=%d, pts=%" PRId64 ")\n", - receive_call_count, ncv->details->audio_frame->nb_samples, current_pts); - } return ncv->details->audio_frame->nb_samples; }else if(averr == AVERROR(EAGAIN)){ pthread_mutex_unlock(&ncv->details->audio_packet_mutex); @@ -1198,11 +1150,9 @@ ffmpeg_get_decoded_audio_frame(ncvisual* ncv){ return 0; }else if(averr == AVERROR_EOF){ pthread_mutex_unlock(&ncv->details->audio_packet_mutex); - audio_log("ffmpeg_get_decoded_audio_frame: EOF (call %d)\n", receive_call_count); return 1; // EOF }else{ pthread_mutex_unlock(&ncv->details->audio_packet_mutex); - audio_log("ffmpeg_get_decoded_audio_frame: Error %d (call %d)\n", averr, receive_call_count); return -1; // Error } } diff --git a/src/player/play.cpp b/src/player/play.cpp index 3eb042e6e..204e55900 100644 --- a/src/player/play.cpp +++ b/src/player/play.cpp @@ -10,13 +10,37 @@ #include #include #include +#include +#include +#include +#include +#include +extern "C" { +#include +#include +#include +#include +} #include #include #include #include "compat/compat.h" +#include "media/audio-output.h" using namespace ncpp; +// Forward declarations for audio API functions from ffmpeg.c +extern "C" { +bool ffmpeg_has_audio(ncvisual* ncv); +int ffmpeg_get_decoded_audio_frame(ncvisual* ncv); +int ffmpeg_resample_audio(ncvisual* ncv, uint8_t** out_data, int* out_samples); +AVFrame* ffmpeg_get_audio_frame(ncvisual* ncv); +int ffmpeg_init_audio_resampler(ncvisual* ncv, int out_sample_rate, int out_channels); +int ffmpeg_get_audio_sample_rate(ncvisual* ncv); +int ffmpeg_get_audio_channels(ncvisual* ncv); +void ffmpeg_audio_request_packets(ncvisual* ncv); +} + static void usage(std::ostream& os, const char* name, int exitcode) __attribute__ ((noreturn)); @@ -42,6 +66,9 @@ struct marshal { int framecount; bool quiet; ncblitter_e blitter; // can be changed while streaming, must propagate out + uint64_t last_abstime_ns; + uint64_t avg_frame_ns; + uint64_t dropped_frames; }; // frame count is in the curry. original time is kept in n's userptr. @@ -75,6 +102,38 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, stdn->printf(0, NCAlign::Left, "frame %06d (%s)", marsh->framecount, notcurses_str_blitter(vopts->blitter)); } + + const uint64_t target_ns = timespec_to_ns(abstime); + const uint64_t prev_ns = marsh->last_abstime_ns; + if(prev_ns > 0 && target_ns > prev_ns){ + uint64_t delta = target_ns - prev_ns; + if(delta < NANOSECS_IN_SEC * 10){ + if(marsh->avg_frame_ns == 0){ + marsh->avg_frame_ns = delta; + }else{ + marsh->avg_frame_ns = (marsh->avg_frame_ns * 7 + delta) / 8; + } + } + } + marsh->last_abstime_ns = target_ns; + const uint64_t default_frame_ns = 41666667ull; // ~24fps + const uint64_t expected_frame_ns = marsh->avg_frame_ns ? marsh->avg_frame_ns : default_frame_ns; + + struct timespec nowts; + clock_gettime(CLOCK_MONOTONIC, &nowts); + uint64_t now_ns = timespec_to_ns(&nowts); + int64_t lag_ns = (int64_t)now_ns - (int64_t)target_ns; + const uint64_t drop_threshold_ns = expected_frame_ns * 3 / 2; // drop once 1.5 frames behind + if(lag_ns > (int64_t)drop_threshold_ns){ + marsh->dropped_frames++; + return 0; + } + if(lag_ns < 0){ + struct timespec sleep_ts; + ns_to_timespec((uint64_t)(-lag_ns), &sleep_ts); + clock_nanosleep(CLOCK_MONOTONIC, 0, &sleep_ts, NULL); + } + struct ncplane* subp = ncvisual_subtitle_plane(*stdn, ncv); const int64_t h = ns / (60 * 60 * NANOSECS_IN_SEC); ns -= h * (60 * 60 * NANOSECS_IN_SEC); @@ -106,8 +165,14 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, keyp = nc.get(false, &ni); } if(keyp == 0){ + // Timeout - check if we should continue break; } + // Check for 'q' key immediately to allow quitting + if(keyp == 'q' && ni.evtype != EvType::Release){ + ncplane_destroy(subp); + return 1; + } // we don't care about key release events, especially the enter // release that starts so many interactive programs under Kitty if(ni.evtype == EvType::Release){ @@ -297,6 +362,112 @@ auto handle_opts(int argc, char** argv, notcurses_options& opts, bool* quiet, return optind; } +// Audio thread data structure +struct audio_thread_data { + ncvisual* ncv; + audio_output* ao; + std::atomic* running; + std::mutex* mutex; +}; + +// Audio thread function - processes decoded frames (video decoder reads packets) +static void audio_thread_func(audio_thread_data* data) { + ncvisual* ncv = data->ncv; + audio_output* ao = data->ao; + std::atomic* running = data->running; + + if(!ffmpeg_has_audio(ncv) || !ao){ + return; + } + + // Initialize resampler - convert FROM codec format TO output format + int out_sample_rate = audio_output_get_sample_rate(ao); + if(out_sample_rate <= 0){ + out_sample_rate = 44100; + } + int out_channels = audio_output_get_channels(ao); + if(out_channels <= 0){ + out_channels = ffmpeg_get_audio_channels(ncv); + } + // Limit to stereo max for output + if(out_channels > 2){ + out_channels = 2; + } + + if(ffmpeg_init_audio_resampler(ncv, out_sample_rate, out_channels) < 0){ + return; + } + + int frame_count = 0; + int consecutive_eagain = 0; + auto last_log = std::chrono::steady_clock::now(); + int frames_since_log = 0; + while(*running){ + ffmpeg_audio_request_packets(ncv); + // Get decoded audio frame (packets are read by video decoder) + // IMPORTANT: avcodec_receive_frame can only return each frame once + // So we must process the frame immediately and not call get_decoded_audio_frame again + // until we've finished processing + int samples = ffmpeg_get_decoded_audio_frame(ncv); + if(samples > 0){ + do{ + consecutive_eagain = 0; + uint8_t* out_data = nullptr; + int out_samples = 0; + int bytes = ffmpeg_resample_audio(ncv, &out_data, &out_samples); + if(bytes > 0 && out_data){ + audio_output_write(ao, out_data, bytes); + frame_count++; + frames_since_log++; + auto now = std::chrono::steady_clock::now(); + auto elapsed = std::chrono::duration_cast(now - last_log).count(); + if(elapsed >= 1000){ + frames_since_log = 0; + last_log = now; + } + free(out_data); + } + samples = ffmpeg_get_decoded_audio_frame(ncv); + }while(samples > 0 && audio_output_needs_data(ao)); + if(!audio_output_needs_data(ao)){ + usleep(1000); + } + continue; + }else if(samples == 1){ + // EOF - don't break, just wait for more data (video might still be playing) + consecutive_eagain = 0; + usleep(10000); // 10ms delay on EOF + }else if(samples < 0){ + // Error + break; + }else{ + consecutive_eagain++; + ffmpeg_audio_request_packets(ncv); + if(!audio_output_needs_data(ao)){ + usleep(1000); + } + } + } + + // Flush any remaining frames at EOF + int flush_count = 0; + while(*running){ + int samples = ffmpeg_get_decoded_audio_frame(ncv); + if(samples > 0){ + uint8_t* out_data = nullptr; + int out_samples = 0; + int bytes = ffmpeg_resample_audio(ncv, &out_data, &out_samples); + if(bytes > 0 && out_data){ + audio_output_write(ao, out_data, bytes); + free(out_data); + flush_count++; + } + }else{ + break; // No more frames + } + } +} + int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, ncscale_e scalemode, ncblitter_e blitter, bool quiet, bool loop, @@ -318,6 +489,14 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, nopts.flags = NCPLANE_OPTION_MARGINALIZED; ncplane* n = nullptr; ncplane* clin = nullptr; + + // Audio-related variables (declared at function scope for cleanup) + audio_output* ao = nullptr; + std::thread* audio_thread = nullptr; + std::atomic audio_running(false); + std::mutex audio_mutex; + audio_thread_data* audio_data = nullptr; + for(auto i = 0 ; i < argc ; ++i){ std::unique_ptr ncv; ncv = std::make_unique(argv[i]); @@ -359,13 +538,53 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, ncplane_scrollup_child(*stdn, clin); } ncplane_erase(n); + + // Initialize audio if available + ncvisual* raw_ncv = *ncv; // Get underlying ncvisual pointer + if(ffmpeg_has_audio(raw_ncv)){ + // Use fixed output format for audio output (resampler will convert) + int sample_rate = 44100; // Fixed output sample rate + int channels = ffmpeg_get_audio_channels(raw_ncv); + // Limit to stereo max for output + if(channels > 2){ + channels = 2; + } + + ao = audio_output_init(sample_rate, channels, AV_SAMPLE_FMT_S16); + if(ao){ + audio_running = true; + audio_data = new audio_thread_data{raw_ncv, ao, &audio_running, &audio_mutex}; + audio_thread = new std::thread(audio_thread_func, audio_data); + audio_output_start(ao); + } + } + do{ struct marshal marsh = { .framecount = 0, .quiet = quiet, .blitter = vopts.blitter, + .last_abstime_ns = 0, + .avg_frame_ns = 0, + .dropped_frames = 0, }; r = ncv->stream(&vopts, timescale, perframe, &marsh); + // Stop audio thread + if(audio_thread){ + audio_running = false; + audio_thread->join(); + delete audio_thread; + audio_thread = nullptr; + } + if(audio_data){ + delete audio_data; + audio_data = nullptr; + } + if(ao){ + audio_output_destroy(ao); + ao = nullptr; + } + free(stdn->get_userptr()); stdn->set_userptr(nullptr); if(r == 0){ @@ -425,6 +644,21 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, return 0; err: + // Cleanup audio resources + if(audio_thread){ + audio_running = false; + audio_thread->join(); + delete audio_thread; + audio_thread = nullptr; + } + if(audio_data){ + delete audio_data; + audio_data = nullptr; + } + if(ao){ + audio_output_destroy(ao); + ao = nullptr; + } free(ncplane_userptr(n)); ncplane_destroy(n); return -1; From 8f7dfd55600997b3828607d0d57643a685026df0 Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sat, 29 Nov 2025 10:44:57 -0600 Subject: [PATCH 06/23] api: centralize export macro --- include/notcurses/api.h | 9 +++++++++ include/notcurses/direct.h | 6 +----- include/notcurses/nckeys.h | 6 +----- include/notcurses/notcurses.h | 6 +----- src/lib/debug.c | 7 +------ src/lib/internal.h | 6 +----- src/media/audio-output.c | 8 +------- src/media/audio-output.h | 8 +------- src/media/ffmpeg.c | 6 +----- 9 files changed, 17 insertions(+), 45 deletions(-) create mode 100644 include/notcurses/api.h diff --git a/include/notcurses/api.h b/include/notcurses/api.h new file mode 100644 index 000000000..f5b07035e --- /dev/null +++ b/include/notcurses/api.h @@ -0,0 +1,9 @@ +#ifdef API +#undef API +#endif + +#ifdef __MINGW32__ +# define API __declspec(dllexport) +#else +# define API __attribute__((visibility("default"))) +#endif diff --git a/include/notcurses/direct.h b/include/notcurses/direct.h index d7dbc3afe..dfca3f457 100644 --- a/include/notcurses/direct.h +++ b/include/notcurses/direct.h @@ -2,6 +2,7 @@ #define NOTCURSES_DIRECT #include +#include #ifdef __cplusplus extern "C" { @@ -11,11 +12,6 @@ extern "C" { #define static API #endif -#ifndef __MINGW32__ -#define API __attribute__((visibility("default"))) -#else -#define API __declspec(dllexport) -#endif #define ALLOC __attribute__((malloc)) __attribute__((warn_unused_result)) // ncdirect_init() will call setlocale() to inspect the current locale. If diff --git a/include/notcurses/nckeys.h b/include/notcurses/nckeys.h index aef9d4e60..bb9636f5f 100644 --- a/include/notcurses/nckeys.h +++ b/include/notcurses/nckeys.h @@ -3,6 +3,7 @@ #include #include +#include #ifdef __cplusplus extern "C" { @@ -12,11 +13,6 @@ extern "C" { #define static API #endif -#ifndef __MINGW32__ -#define API __attribute__((visibility("default"))) -#else -#define API __declspec(dllexport) -#endif #define ALLOC __attribute__((malloc)) __attribute__((warn_unused_result)) // Synthesized input events, i.e. any input event we can report that isn't diff --git a/include/notcurses/notcurses.h b/include/notcurses/notcurses.h index 7d6b07b55..c88c072ea 100644 --- a/include/notcurses/notcurses.h +++ b/include/notcurses/notcurses.h @@ -15,6 +15,7 @@ #include #include #include +#include #ifdef __cplusplus extern "C" { @@ -28,11 +29,6 @@ extern "C" { #define static API #endif -#ifndef __MINGW32__ -#define API __attribute__((visibility("default"))) -#else -#define API __declspec(dllexport) -#endif #define ALLOC __attribute__((malloc)) __attribute__((warn_unused_result)) // Get a human-readable string describing the running Notcurses version. diff --git a/src/lib/debug.c b/src/lib/debug.c index a2dcfe8f8..c9c7a66b7 100644 --- a/src/lib/debug.c +++ b/src/lib/debug.c @@ -1,10 +1,5 @@ #include "internal.h" - -#ifndef __MINGW32__ -#define API __attribute__((visibility("default"))) -#else -#define API __declspec(dllexport) -#endif +#include API ncloglevel_e loglevel = NCLOGLEVEL_SILENT; diff --git a/src/lib/internal.h b/src/lib/internal.h index 4b49994f9..47311b4af 100644 --- a/src/lib/internal.h +++ b/src/lib/internal.h @@ -8,6 +8,7 @@ #include "notcurses/ncport.h" #include "notcurses/notcurses.h" #include "notcurses/direct.h" +#include // KEY_EVENT is defined by both ncurses.h (prior to 6.3) and wincon.h. since we // don't use either definition, kill it before inclusion of ncurses.h. #undef KEY_EVENT @@ -38,11 +39,6 @@ #include "lib/gpm.h" -#ifndef __MINGW32__ -#define API __attribute__((visibility("default"))) -#else -#define API __declspec(dllexport) -#endif #define ALLOC __attribute__((malloc)) __attribute__((warn_unused_result)) #ifdef __cplusplus diff --git a/src/media/audio-output.c b/src/media/audio-output.c index 22f61b78d..68d27aaab 100644 --- a/src/media/audio-output.c +++ b/src/media/audio-output.c @@ -15,13 +15,7 @@ #include #include #include "lib/internal.h" - -// Define API macro for visibility export -#ifndef __MINGW32__ -#define API __attribute__((visibility("default"))) -#else -#define API __declspec(dllexport) -#endif +#include #define AUDIO_BUFFER_SIZE 4096 diff --git a/src/media/audio-output.h b/src/media/audio-output.h index 08e8f84db..34278fc99 100644 --- a/src/media/audio-output.h +++ b/src/media/audio-output.h @@ -7,13 +7,7 @@ #include #include #include - -// Define API macro for visibility export -#ifndef __MINGW32__ -#define API __attribute__((visibility("default"))) -#else -#define API __declspec(dllexport) -#endif +#include #ifdef __cplusplus extern "C" { diff --git a/src/media/ffmpeg.c b/src/media/ffmpeg.c index 9a7339580..903805a6a 100644 --- a/src/media/ffmpeg.c +++ b/src/media/ffmpeg.c @@ -21,6 +21,7 @@ #include #include "lib/visual-details.h" #include "lib/internal.h" +#include struct AVFormatContext; @@ -1067,11 +1068,6 @@ ffmpeg_destroy(ncvisual* ncv){ // Public API functions for audio handling (called from play.cpp) // Define API macro for visibility export -#ifndef __MINGW32__ -#define API __attribute__((visibility("default"))) -#else -#define API __declspec(dllexport) -#endif API void ffmpeg_audio_request_packets(ncvisual* ncv){ From 6b5dd19ab365c2cecd9d3317f77f907e1d8853f2 Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sat, 29 Nov 2025 10:58:37 -0600 Subject: [PATCH 07/23] ffmpeg: enable multi-threaded decode --- src/media/ffmpeg.c | 47 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/src/media/ffmpeg.c b/src/media/ffmpeg.c index 903805a6a..a1ea19f79 100644 --- a/src/media/ffmpeg.c +++ b/src/media/ffmpeg.c @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -19,6 +20,8 @@ #include #include #include +#include +#include #include "lib/visual-details.h" #include "lib/internal.h" #include @@ -40,6 +43,41 @@ typedef struct audio_packet_queue { int count; } audio_packet_queue; +static int +ffmpeg_detect_thread_count(void){ + static int cached = 0; + if(cached > 0){ + return cached; + } + int threads = 0; + const char* env = getenv("NCPLAYER_FFMPEG_THREADS"); + if(env && *env){ + char* endptr = NULL; + long parsed = strtol(env, &endptr, 10); + if(endptr != env && parsed > 0 && parsed < INT_MAX){ + threads = (int)parsed; + } + } + if(threads <= 0){ + int avcpus = av_cpu_count(); + if(avcpus > 0){ + threads = avcpus; + }else{ +#ifdef _SC_NPROCESSORS_ONLN + long syscpus = sysconf(_SC_NPROCESSORS_ONLN); + if(syscpus > 0 && syscpus < INT_MAX){ + threads = (int)syscpus; + } +#endif + } + } + if(threads <= 0){ + threads = 1; + } + cached = threads; + return cached; +} + typedef struct ncvisual_details { struct AVFormatContext* fmtctx; struct AVCodecContext* codecctx; // video codec context @@ -670,6 +708,15 @@ ffmpeg_from_file(const char* filename){ if(avcodec_parameters_to_context(ncv->details->codecctx, st->codecpar) < 0){ goto err; } + int vthreads = ffmpeg_detect_thread_count(); + if(vthreads > 1){ + ncv->details->codecctx->thread_count = vthreads; + if(ncv->details->codec->capabilities & AV_CODEC_CAP_FRAME_THREADS){ + ncv->details->codecctx->thread_type = FF_THREAD_FRAME; + }else{ + ncv->details->codecctx->thread_type = FF_THREAD_SLICE; + } + } if(avcodec_open2(ncv->details->codecctx, ncv->details->codec, NULL) < 0){ //fprintf(stderr, "Couldn't open codec for %s (%s)\n", filename, av_err2str(*averr)); goto err; From 98ac8e0b3558fcd920ea130c6e58fe7535916dec Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sun, 30 Nov 2025 07:51:32 -0600 Subject: [PATCH 08/23] ncplayer: add seek keys and fps overlay --- include/ncpp/Visual.hh | 10 ++ include/notcurses/notcurses.h | 9 ++ src/lib/internal.h | 2 + src/lib/visual.c | 14 +++ src/media/ffmpeg.c | 119 +++++++++++++++++++ src/media/none.c | 1 + src/media/oiio-indep.c | 2 + src/player/play.cpp | 217 ++++++++++++++++++++++++++-------- 8 files changed, 325 insertions(+), 49 deletions(-) diff --git a/include/ncpp/Visual.hh b/include/ncpp/Visual.hh index 115c95e08..6ab084c40 100644 --- a/include/ncpp/Visual.hh +++ b/include/ncpp/Visual.hh @@ -91,6 +91,16 @@ namespace ncpp return error_guard (ncvisual_simple_streamer (visual, vopts, tspec, curry), -1); } + bool seek (double seconds) const NOEXCEPT_MAYBE + { + return error_guard (ncvisual_seek (visual, seconds), -1); + } + + int64_t frame_index () const NOEXCEPT_MAYBE + { + return ncvisual_frame_index (visual); + } + int polyfill (int y, int x, uint32_t rgba) const NOEXCEPT_MAYBE { return error_guard (ncvisual_polyfill_yx (visual, y, x, rgba), -1); diff --git a/include/notcurses/notcurses.h b/include/notcurses/notcurses.h index c88c072ea..435e1e18b 100644 --- a/include/notcurses/notcurses.h +++ b/include/notcurses/notcurses.h @@ -3431,6 +3431,15 @@ API int ncvisual_decode(struct ncvisual* nc) API int ncvisual_decode_loop(struct ncvisual* nc) __attribute__ ((nonnull (1))); +// Seek within a visual by the provided number of seconds. Positive values move +// forward, negative backwards. Not all backends support seeking. +API int ncvisual_seek(struct ncvisual* n, double seconds) + __attribute__ ((nonnull (1))); + +// Return zero-based index of last decoded frame, or -1 if unavailable +API int64_t ncvisual_frame_index(const struct ncvisual* n) + __attribute__ ((nonnull (1))); + // Rotate the visual 'rads' radians. Only M_PI/2 and -M_PI/2 are supported at // the moment, but this might change in the future. API int ncvisual_rotate(struct ncvisual* n, double rads) diff --git a/src/lib/internal.h b/src/lib/internal.h index 47311b4af..372796352 100644 --- a/src/lib/internal.h +++ b/src/lib/internal.h @@ -1803,6 +1803,8 @@ typedef struct ncvisual_implementation { int rowalign; // rowstride base, can be 0 for no padding // do a persistent resize, changing the ncv itself int (*visual_resize)(struct ncvisual* ncv, unsigned rows, unsigned cols); + int (*visual_seek)(struct ncvisual* ncv, double seconds); + int64_t (*visual_frame_index)(const struct ncvisual* ncv); void (*visual_destroy)(struct ncvisual* ncv); bool canopen_images; bool canopen_videos; diff --git a/src/lib/visual.c b/src/lib/visual.c index f0ae4b6d7..22f572af4 100644 --- a/src/lib/visual.c +++ b/src/lib/visual.c @@ -50,6 +50,20 @@ int ncvisual_decode_loop(ncvisual* nc){ return visual_implementation->visual_decode_loop(nc); } +int ncvisual_seek(ncvisual* nc, double seconds){ + if(!visual_implementation->visual_seek){ + return -1; + } + return visual_implementation->visual_seek(nc, seconds); +} + +int64_t ncvisual_frame_index(const ncvisual* nc){ + if(!visual_implementation->visual_frame_index){ + return -1; + } + return visual_implementation->visual_frame_index(nc); +} + ncvisual* ncvisual_from_file(const char* filename){ if(!visual_implementation->visual_from_file){ return NULL; diff --git a/src/media/ffmpeg.c b/src/media/ffmpeg.c index a1ea19f79..42fcb745a 100644 --- a/src/media/ffmpeg.c +++ b/src/media/ffmpeg.c @@ -101,6 +101,8 @@ typedef struct ncvisual_details { bool audio_packet_outstanding; // whether we have an audio packet waiting to be decoded audio_packet_queue pending_audio_packets; // audio packets waiting to be sent int64_t last_audio_frame_pts; // PTS of last processed audio frame (to prevent reprocessing) + int64_t last_video_frame_pts; // PTS of last displayed video frame + int64_t decoded_frames; pthread_mutex_t packet_mutex; // mutex for thread-safe packet reading pthread_mutex_t audio_packet_mutex; // mutex for audio packet queue } ncvisual_details; @@ -571,9 +573,25 @@ ffmpeg_decode(ncvisual* n){ //fprintf(stderr, "good decode! %d/%d %d %p\n", n->details->frame->height, n->details->frame->width, n->rowstride, f->data); ncvisual_set_data(n, f->data[0], false); force_rgba(n); + int64_t vpts = f->pts; + if(vpts == AV_NOPTS_VALUE){ + vpts = f->best_effort_timestamp; + } + n->details->last_video_frame_pts = vpts; + if(n->details->decoded_frames >= 0){ + ++n->details->decoded_frames; + } return 0; } +static int64_t +ffmpeg_frame_index(const ncvisual* ncv){ + if(!ncv || !ncv->details){ + return -1; + } + return ncv->details->decoded_frames; +} + static ncvisual_details* ffmpeg_details_init(void){ ncvisual_details* deets = malloc(sizeof(*deets)); @@ -583,6 +601,8 @@ ffmpeg_details_init(void){ deets->sub_stream_index = -1; deets->audio_stream_index = -1; deets->last_audio_frame_pts = AV_NOPTS_VALUE; // No frame processed yet + deets->last_video_frame_pts = AV_NOPTS_VALUE; + deets->decoded_frames = 0; deets->audio_out_channels = 0; // Will be set when resampler is initialized audio_queue_init(&deets->pending_audio_packets); if(pthread_mutex_init(&deets->packet_mutex, NULL) != 0){ @@ -1113,6 +1133,31 @@ ffmpeg_destroy(ncvisual* ncv){ } } +// Public API functions used by ncplayer +API double +ffmpeg_get_video_position_seconds(const ncvisual* ncv){ + if(!ncv || !ncv->details || !ncv->details->fmtctx){ + return -1.0; + } + int stream_index = ncv->details->stream_index; + if(stream_index < 0 || stream_index >= (int)ncv->details->fmtctx->nb_streams){ + return -1.0; + } + AVStream* stream = ncv->details->fmtctx->streams[stream_index]; + if(!stream){ + return -1.0; + } + double time_base = av_q2d(stream->time_base); + if(time_base <= 0.0){ + return -1.0; + } + int64_t pts = ncv->details->last_video_frame_pts; + if(pts == AV_NOPTS_VALUE){ + return -1.0; + } + return pts * time_base; +} + // Public API functions for audio handling (called from play.cpp) // Define API macro for visibility export @@ -1303,6 +1348,78 @@ ffmpeg_get_audio_channels(ncvisual* ncv){ return channels > 0 ? channels : 2; // Default stereo } +API int +ffmpeg_seek_relative(ncvisual* ncv, double seconds){ + if(!ncv || !ncv->details || !ncv->details->fmtctx || seconds == 0.0){ + return 0; + } + if(!ncv->details->codecctx){ + return -1; + } + AVFormatContext* fmt = ncv->details->fmtctx; + int stream_index = ncv->details->stream_index; + if(stream_index < 0 || stream_index >= (int)fmt->nb_streams){ + return -1; + } + AVStream* stream = fmt->streams[stream_index]; + double time_base = av_q2d(stream->time_base); + if(time_base == 0.0){ + return -1; + } + int64_t current = ncv->details->last_video_frame_pts; + if(current == AV_NOPTS_VALUE){ + current = (stream->start_time != AV_NOPTS_VALUE) ? stream->start_time : 0; + } + int64_t offset = seconds / time_base; + int64_t target = current + offset; + if(target < 0){ + target = 0; + } + int flags = 0; + if(seconds < 0){ + flags |= AVSEEK_FLAG_BACKWARD; + } + if(av_seek_frame(fmt, stream_index, target, flags) < 0){ + // clamp to valid duration if available + if(stream->duration > 0 && target > stream->duration){ + target = stream->duration; + if(av_seek_frame(fmt, stream_index, target, AVSEEK_FLAG_BACKWARD) < 0){ + return -1; + } + }else{ + return -1; + } + } + avcodec_flush_buffers(ncv->details->codecctx); + ncv->details->packet_outstanding = false; + av_packet_unref(ncv->details->packet); + ncv->details->last_video_frame_pts = target; + double frame_rate = av_q2d(stream->avg_frame_rate); + if(frame_rate <= 0.0){ + frame_rate = 1.0 / av_q2d(stream->time_base); + if(frame_rate <= 0.0){ + frame_rate = 30.0; + } + } + double seconds_pos = target * time_base; + if(seconds_pos < 0){ + seconds_pos = 0; + } + ncv->details->decoded_frames = (int64_t)(seconds_pos * frame_rate); + if(ncv->details->audiocodecctx){ + avcodec_flush_buffers(ncv->details->audiocodecctx); + pthread_mutex_lock(&ncv->details->audio_packet_mutex); + audio_queue_clear(&ncv->details->pending_audio_packets); + ncv->details->audio_packet_outstanding = false; + pthread_mutex_unlock(&ncv->details->audio_packet_mutex); + ncv->details->last_audio_frame_pts = AV_NOPTS_VALUE; + } + if(ffmpeg_decode(ncv) < 0){ + return -1; + } + return 0; +} + ncvisual_implementation local_visual_implementation = { .visual_init = ffmpeg_init, .visual_printbanner = ffmpeg_printbanner, @@ -1315,6 +1432,8 @@ ncvisual_implementation local_visual_implementation = { .visual_stream = ffmpeg_stream, .visual_subtitle = ffmpeg_subtitle, .visual_resize = ffmpeg_resize, + .visual_seek = ffmpeg_seek_relative, + .visual_frame_index = ffmpeg_frame_index, .visual_destroy = ffmpeg_destroy, .rowalign = 64, // ffmpeg wants multiples of IMGALIGN (64) .canopen_images = true, diff --git a/src/media/none.c b/src/media/none.c index cf239bc19..a5b468d62 100644 --- a/src/media/none.c +++ b/src/media/none.c @@ -10,6 +10,7 @@ printbanner(fbuf* f){ ncvisual_implementation local_visual_implementation = { .visual_printbanner = printbanner, + .visual_frame_index = NULL, }; #endif diff --git a/src/media/oiio-indep.c b/src/media/oiio-indep.c index c5acbc779..64c867ff2 100644 --- a/src/media/oiio-indep.c +++ b/src/media/oiio-indep.c @@ -85,6 +85,8 @@ ncvisual_implementation local_visual_implementation = { .visual_decode_loop = oiio_decode_loop, .visual_stream = oiio_stream, .visual_resize = oiio_resize, + .visual_seek = NULL, + .visual_frame_index = NULL, .visual_destroy = oiio_destroy, .canopen_images = true, .canopen_videos = false, diff --git a/src/player/play.cpp b/src/player/play.cpp index 204e55900..049c66182 100644 --- a/src/player/play.cpp +++ b/src/player/play.cpp @@ -39,6 +39,7 @@ int ffmpeg_init_audio_resampler(ncvisual* ncv, int out_sample_rate, int out_chan int ffmpeg_get_audio_sample_rate(ncvisual* ncv); int ffmpeg_get_audio_channels(ncvisual* ncv); void ffmpeg_audio_request_packets(ncvisual* ncv); +double ffmpeg_get_video_position_seconds(const ncvisual* ncv); } static void usage(std::ostream& os, const char* name, int exitcode) @@ -62,6 +63,15 @@ void usage(std::ostream& o, const char* name, int exitcode){ exit(exitcode); } +enum class PlaybackRequest { + None, + Quit, + NextFile, + PrevFile, + RestartFile, + Seek, +}; + struct marshal { int framecount; bool quiet; @@ -69,8 +79,15 @@ struct marshal { uint64_t last_abstime_ns; uint64_t avg_frame_ns; uint64_t dropped_frames; + bool show_fps; + double current_fps; + double current_drop_pct; + PlaybackRequest request; + double seek_delta; }; +static constexpr double kNcplayerSeekSeconds = 5.0; +static constexpr double kNcplayerSeekMinutes = 2.0 * 60.0; // frame count is in the curry. original time is kept in n's userptr. auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, const struct timespec* abstime, void* vmarshal) -> int { @@ -84,14 +101,42 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, ncplane_set_userptr(vopts->n, start); } std::unique_ptr stdn(nc.get_stdplane()); + static auto last_fps_sample = std::chrono::steady_clock::now(); + static int frames_since_sample = 0; // negative framecount means don't print framecount/timing (quiet mode) if(marsh->framecount >= 0){ - ++marsh->framecount; + ++frames_since_sample; + int64_t idx = ncvisual_frame_index(ncv); + if(idx >= 0){ + marsh->framecount = idx; + }else{ + ++marsh->framecount; + } + auto now = std::chrono::steady_clock::now(); + auto elapsed = std::chrono::duration_cast(now - last_fps_sample).count(); + if(elapsed >= 1000){ + if(elapsed > 0){ + marsh->current_fps = frames_since_sample * 1000.0 / elapsed; + const uint64_t total_attempted = marsh->framecount + marsh->dropped_frames; + marsh->current_drop_pct = total_attempted ? (100.0 * marsh->dropped_frames / total_attempted) : 0.0; + } + frames_since_sample = 0; + last_fps_sample = now; + } } stdn->set_fg_rgb(0x80c080); - struct timespec now; - clock_gettime(CLOCK_MONOTONIC, &now); - int64_t ns = timespec_to_ns(&now) - timespec_to_ns(start); + int64_t display_ns; + double media_seconds = ffmpeg_get_video_position_seconds(ncv); + if(media_seconds >= 0.0){ + display_ns = (int64_t)(media_seconds * (double)NANOSECS_IN_SEC); + }else{ + struct timespec now; + clock_gettime(CLOCK_MONOTONIC, &now); + display_ns = timespec_to_ns(&now) - timespec_to_ns(start); + } + if(display_ns < 0){ + display_ns = 0; + } marsh->blitter = vopts->blitter; if(marsh->blitter == NCBLIT_DEFAULT){ marsh->blitter = ncvisual_media_defblitter(nc, vopts->scaling); @@ -99,8 +144,14 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, if(!marsh->quiet){ // FIXME put this on its own plane if we're going to be erase()ing it stdn->erase(); - stdn->printf(0, NCAlign::Left, "frame %06d (%s)", marsh->framecount, - notcurses_str_blitter(vopts->blitter)); + if(marsh->show_fps){ + stdn->printf(0, NCAlign::Left, "frame %06d FPS %.2f drops %.1f%% (%s)", + marsh->framecount, marsh->current_fps, marsh->current_drop_pct, + notcurses_str_blitter(vopts->blitter)); + }else{ + stdn->printf(0, NCAlign::Left, "frame %06d (%s)", marsh->framecount, + notcurses_str_blitter(vopts->blitter)); + } } const uint64_t target_ns = timespec_to_ns(abstime); @@ -135,15 +186,16 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, } struct ncplane* subp = ncvisual_subtitle_plane(*stdn, ncv); - const int64_t h = ns / (60 * 60 * NANOSECS_IN_SEC); - ns -= h * (60 * 60 * NANOSECS_IN_SEC); - const int64_t m = ns / (60 * NANOSECS_IN_SEC); - ns -= m * (60 * NANOSECS_IN_SEC); - const int64_t s = ns / NANOSECS_IN_SEC; - ns -= s * NANOSECS_IN_SEC; + int64_t remaining = display_ns; + const int64_t h = remaining / (60 * 60 * NANOSECS_IN_SEC); + remaining -= h * (60 * 60 * NANOSECS_IN_SEC); + const int64_t m = remaining / (60 * NANOSECS_IN_SEC); + remaining -= m * (60 * NANOSECS_IN_SEC); + const int64_t s = remaining / NANOSECS_IN_SEC; + remaining -= s * NANOSECS_IN_SEC; if(!marsh->quiet){ stdn->printf(0, NCAlign::Right, "%02" PRId64 ":%02" PRId64 ":%02" PRId64 ".%04" PRId64, - h, m, s, ns / 1000000); + h, m, s, remaining / 1000000); } if(!nc.render()){ return -1; @@ -169,7 +221,8 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, break; } // Check for 'q' key immediately to allow quitting - if(keyp == 'q' && ni.evtype != EvType::Release){ + if((keyp == 'q' || keyp == 'Q') && ni.evtype != EvType::Release){ + marsh->request = PlaybackRequest::Quit; ncplane_destroy(subp); return 1; } @@ -200,18 +253,29 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, continue; }else if(keyp >= '7' && keyp <= '9' && !ncinput_alt_p(&ni) && !ncinput_ctrl_p(&ni)){ continue; // don't error out - }else if(keyp == NCKey::Up){ - // FIXME move backwards significantly + }else if(keyp == 'f' || keyp == 'F'){ + marsh->show_fps = !marsh->show_fps; continue; + }else if(keyp == NCKey::Up){ + marsh->request = PlaybackRequest::Seek; + marsh->seek_delta = -kNcplayerSeekMinutes; + ncplane_destroy(subp); + return 2; }else if(keyp == NCKey::Down){ - // FIXME move forwards significantly - continue; + marsh->request = PlaybackRequest::Seek; + marsh->seek_delta = kNcplayerSeekMinutes; + ncplane_destroy(subp); + return 2; }else if(keyp == NCKey::Right){ - // FIXME move forwards - continue; + marsh->request = PlaybackRequest::Seek; + marsh->seek_delta = kNcplayerSeekSeconds; + ncplane_destroy(subp); + return 2; }else if(keyp == NCKey::Left){ - // FIXME move backwards - continue; + marsh->request = PlaybackRequest::Seek; + marsh->seek_delta = -kNcplayerSeekSeconds; + ncplane_destroy(subp); + return 2; }else if(keyp != 'q'){ continue; } @@ -490,6 +554,7 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, ncplane* n = nullptr; ncplane* clin = nullptr; + bool show_fps_overlay = false; // Audio-related variables (declared at function scope for cleanup) audio_output* ao = nullptr; std::thread* audio_thread = nullptr; @@ -539,27 +604,24 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, } ncplane_erase(n); - // Initialize audio if available - ncvisual* raw_ncv = *ncv; // Get underlying ncvisual pointer - if(ffmpeg_has_audio(raw_ncv)){ - // Use fixed output format for audio output (resampler will convert) - int sample_rate = 44100; // Fixed output sample rate - int channels = ffmpeg_get_audio_channels(raw_ncv); - // Limit to stereo max for output - if(channels > 2){ - channels = 2; - } - - ao = audio_output_init(sample_rate, channels, AV_SAMPLE_FMT_S16); - if(ao){ - audio_running = true; - audio_data = new audio_thread_data{raw_ncv, ao, &audio_running, &audio_mutex}; - audio_thread = new std::thread(audio_thread_func, audio_data); - audio_output_start(ao); + PlaybackRequest pending_request = PlaybackRequest::None; + while(true){ + bool restart_stream = false; + if(ffmpeg_has_audio(*ncv)){ + int sample_rate = 44100; + int channels = ffmpeg_get_audio_channels(*ncv); + if(channels > 2){ + channels = 2; + } + ao = audio_output_init(sample_rate, channels, AV_SAMPLE_FMT_S16); + if(ao){ + audio_running = true; + audio_data = new audio_thread_data{*ncv, ao, &audio_running, &audio_mutex}; + audio_thread = new std::thread(audio_thread_func, audio_data); + audio_output_start(ao); + } } - } - do{ struct marshal marsh = { .framecount = 0, .quiet = quiet, @@ -567,9 +629,15 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, .last_abstime_ns = 0, .avg_frame_ns = 0, .dropped_frames = 0, + .show_fps = show_fps_overlay, + .current_fps = 0.0, + .current_drop_pct = 0.0, + .request = PlaybackRequest::None, + .seek_delta = 0.0, }; r = ncv->stream(&vopts, timescale, perframe, &marsh); - // Stop audio thread + pending_request = marsh.request; + show_fps_overlay = marsh.show_fps; if(audio_thread){ audio_running = false; audio_thread->join(); @@ -587,7 +655,18 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, free(stdn->get_userptr()); stdn->set_userptr(nullptr); - if(r == 0){ + restart_stream = false; + if(pending_request == PlaybackRequest::Seek){ + double delta = marsh.seek_delta; + if(ncvisual_seek(*ncv, delta) == 0){ + pending_request = PlaybackRequest::None; + restart_stream = true; + }else{ + pending_request = PlaybackRequest::None; + } + } + + if(!restart_stream && r == 0){ vopts.blitter = marsh.blitter; if(!loop){ if(displaytime < 0){ @@ -608,13 +687,13 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, nc.refresh(nullptr, nullptr); }else if(ni.id >= '0' && ni.id <= '6'){ blitter = vopts.blitter = static_cast(ni.id - '0'); - --i; // rerun same input with the new blitter + --i; break; }else if(ni.id >= '7' && ni.id <= '9'){ - --i; // just absorb the input + --i; break; }else if(ni.id == NCKey::Resize){ - --i; // rerun with the new size + --i; if(!nc.refresh(&dimy, &dimx)){ goto err; } @@ -622,7 +701,6 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, } }while(ni.id != ' '); }else{ - // FIXME do we still want to honor keybindings when timing out? struct timespec ts; ts.tv_sec = displaytime; ts.tv_nsec = (displaytime - ts.tv_sec) * NANOSECS_IN_SEC; @@ -631,15 +709,56 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, }else{ ncv->decode_loop(); } - ncplane_destroy(clin); + + if(loop && pending_request == PlaybackRequest::None){ + restart_stream = true; + } } - }while(loop && r == 0); + + if(!restart_stream){ + break; + } + } + if(clin){ + ncplane_destroy(clin); + clin = nullptr; + } if(r < 0){ // positive is intentional abort std::cerr << "Error while playing " << argv[i] << std::endl; goto err; } free(ncplane_userptr(n)); ncplane_destroy(n); + if(pending_request == PlaybackRequest::Quit){ + return 0; + } + if(pending_request == PlaybackRequest::PrevFile){ + if(i >= 1){ + i -= 2; + }else{ + i = -1; + } + continue; + } + if(pending_request == PlaybackRequest::RestartFile){ + if(i >= 0){ + --i; + } + continue; + } + if(pending_request == PlaybackRequest::Quit){ + return 0; + }else if(pending_request == PlaybackRequest::PrevFile){ + if(i >= 1){ + i -= 2; + }else{ + i = -1; + } + continue; + }else if(pending_request == PlaybackRequest::RestartFile){ + --i; + continue; + } } return 0; From 62c729e1467a7d9d17fec287c03ffa09196ca351 Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sun, 30 Nov 2025 10:06:54 -0600 Subject: [PATCH 09/23] get subtitles working - WIP --- src/media/ffmpeg.c | 321 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 276 insertions(+), 45 deletions(-) diff --git a/src/media/ffmpeg.c b/src/media/ffmpeg.c index 42fcb745a..a28af1afe 100644 --- a/src/media/ffmpeg.c +++ b/src/media/ffmpeg.c @@ -1,10 +1,14 @@ #include "builddef.h" #ifdef USE_FFMPEG +#include #include +#include +#include #include #include #include #include +#include #include #include #include @@ -24,8 +28,13 @@ #include #include "lib/visual-details.h" #include "lib/internal.h" +#include "lib/logging.h" #include +#define SUBLOG_INFO(fmt, ...) loginfo("[subtitle] " fmt, ##__VA_ARGS__) +#define SUBLOG_DEBUG(fmt, ...) logdebug("[subtitle] " fmt, ##__VA_ARGS__) +#define SUBLOG_WARN(fmt, ...) logwarn("[subtitle] " fmt, ##__VA_ARGS__) + struct AVFormatContext; struct AVCodecContext; @@ -34,6 +43,8 @@ struct AVCodec; struct AVCodecParameters; struct AVPacket; +double ffmpeg_get_video_position_seconds(const ncvisual* ncv); + #define AUDIO_PACKET_QUEUE_SIZE 8 typedef struct audio_packet_queue { @@ -43,6 +54,52 @@ typedef struct audio_packet_queue { int count; } audio_packet_queue; +static int +ass_detect_text_field(const uint8_t* header, size_t size){ + if(!header || size == 0){ + return 9; + } + const char* data = (const char*)header; + const char* end = data + size; + const int default_index = 9; + while(data < end){ + const char* line_start = data; + const char* linebreak = memchr(line_start, '\n', end - line_start); + size_t linelen = linebreak ? (size_t)(linebreak - line_start) : (size_t)(end - line_start); + if(linelen >= 7 && !strncasecmp(line_start, "Format:", 7)){ + const char* cursor = line_start + 7; + while(cursor < line_start + linelen && isspace((unsigned char)*cursor)){ + ++cursor; + } + int field = 0; + const char* token = cursor; + for(const char* p = cursor ; p <= line_start + linelen ; ++p){ + if(p == line_start + linelen || *p == ',' || *p == '\r'){ + const char* t = token; + while(t < p && isspace((unsigned char)*t)){ + ++t; + } + const char* q = p; + while(q > t && isspace((unsigned char)*(q - 1))){ + --q; + } + if(q > t){ + size_t toklen = q - t; + if(!strncasecmp(t, "Text", toklen)){ + return field; + } + } + ++field; + token = p + 1; + } + } + return default_index; + } + data = linebreak ? linebreak + 1 : end; + } + return default_index; +} + static int ffmpeg_detect_thread_count(void){ static int cached = 0; @@ -94,6 +151,12 @@ typedef struct ncvisual_details { struct SwrContext* swrctx; // audio resampler context int audio_out_channels; // output channel count for resampler (1 or 2) AVSubtitle subtitle; + bool subtitle_active; + double subtitle_start_time; + double subtitle_end_time; + double subtitle_time_base; + int subtitle_text_field; + bool subtitle_logged; int stream_index; // match against this following av_read_frame() int sub_stream_index; // subtitle stream index, can be < 0 if no subtitles int audio_stream_index; // audio stream index, can be < 0 if no audio @@ -247,59 +310,126 @@ print_frame_summary(const AVCodecContext* cctx, const AVFrame* f){ }*/ static char* -deass(const char* ass){ - // SSA/ASS formats: - // Dialogue: Marked=0,0:02:40.65,0:02:41.79,Wolf main,Cher,0000,0000,0000,,Et les enregistrements de ses ondes delta ? - // FIXME more - if(strncmp(ass, "Dialogue:", strlen("Dialogue:"))){ +deass(const char* ass, int text_field_index){ + if(!ass){ return NULL; } - const char* delim = strchr(ass, ','); - int commas = 0; // we want 8 - while(delim && commas < 8){ - delim = strchr(delim + 1, ','); - ++commas; + const char* dialog = strstr(ass, "Dialogue:"); + const char* payload = dialog ? dialog + strlen("Dialogue:") : ass; + while(*payload == ' '){ + ++payload; + } + const char* text = payload; + int field = 0; + bool in_field_brace = false; + bool found_field = false; + for(const char* src = payload ; *src ; ++src){ + if(*src == '{'){ + in_field_brace = true; + }else if(*src == '}'){ + in_field_brace = false; + } + if(!in_field_brace && *src == ','){ + ++field; + if(field == text_field_index){ + text = src + 1; + found_field = true; + break; + } + } + } + if(!found_field){ + const char* lastcomma = strrchr(payload, ','); + if(lastcomma){ + text = lastcomma + 1; + } } - if(!delim){ + while(*text == ' ' || *text == '\t'){ + ++text; + } + size_t textlen = strlen(text); + char* buf = malloc(textlen + 1); + if(!buf){ return NULL; } - // handle ASS syntax...\i0, \b0, etc. - char* dup = strdup(delim + 1); - char* c = dup; - while(*c){ - if(*c == '\\'){ - *c = ' '; - ++c; - if(*c){ - *c = ' ';; + char* dst = buf; + bool in_brace = false; + for(const char* src = text ; *src ; ++src){ + if(in_brace){ + if(*src == '}'){ + in_brace = false; } + continue; + } + if(*src == '{'){ + in_brace = true; + continue; } - ++c; + if(*src == '\\'){ + ++src; + if(!*src){ + break; + } + if(*src == 'N' || *src == 'n'){ + *dst++ = '\n'; + } + continue; + } + *dst++ = *src; + } + *dst = '\0'; + char* trimmed = buf; + while(*trimmed && isspace((unsigned char)*trimmed)){ + ++trimmed; + } + size_t len = strlen(trimmed); + while(len > 0 && isspace((unsigned char)trimmed[len - 1])){ + trimmed[--len] = '\0'; } - return dup; + char* out = len ? strdup(trimmed) : NULL; + free(buf); + return out; } static struct ncplane* -subtitle_plane_from_text(ncplane* parent, const char* text){ - if(parent == NULL){ -//logerror("need a parent plane\n"); +subtitle_plane_from_text(ncplane* parent, const char* text, bool* logged_flag){ + if(parent == NULL || text == NULL){ return NULL; } - int width = ncstrwidth(text, NULL, NULL); - if(width <= 0){ -//logwarn("couldn't extract subtitle from %s\n", text); + char* dup = strdup(text); + if(!dup){ return NULL; } - int rows = (width + ncplane_dim_x(parent) - 1) / ncplane_dim_x(parent); + char* trimmed = dup; + while(*trimmed && isspace((unsigned char)*trimmed)){ + ++trimmed; + } + size_t len = strlen(trimmed); + while(len > 0 && isspace((unsigned char)trimmed[len - 1])){ + trimmed[--len] = '\0'; + } + if(len == 0){ + free(dup); + return NULL; + } + int linecount = 1; + for(size_t i = 0 ; i < len ; ++i){ + if(trimmed[i] == '\n' || trimmed[i] == '\r'){ + ++linecount; + if(trimmed[i] == '\r' && trimmed[i + 1] == '\n'){ + ++i; + } + } + } struct ncplane_options nopts = { - .y = ncplane_dim_y(parent) - (rows + 1), - .rows = rows, + .y = ncplane_dim_y(parent) - (linecount + 1), + .rows = linecount, .cols = ncplane_dim_x(parent), .name = "subt", }; struct ncplane* n = ncplane_create(parent, &nopts); if(n == NULL){ -//logerror("error creating subtitle plane\n"); + free(dup); return NULL; } uint64_t channels = 0; @@ -307,34 +437,61 @@ subtitle_plane_from_text(ncplane* parent, const char* text){ ncchannels_set_fg_rgb8(&channels, 0x88, 0x88, 0x88); ncplane_stain(n, -1, -1, 0, 0, channels, channels, channels, channels); ncchannels_set_fg_default(&channels); - ncplane_puttext(n, 0, NCALIGN_LEFT, text, NULL); + if(logged_flag && !*logged_flag){ + SUBLOG_DEBUG("rendering subtitle text: \"%s\"", trimmed); + *logged_flag = true; + } + ncplane_puttext(n, 0, NCALIGN_CENTER, trimmed, NULL); ncchannels_set_bg_alpha(&channels, NCALPHA_TRANSPARENT); ncplane_set_base(n, " ", 0, channels); + free(dup); return n; } static uint32_t palette[NCPALETTESIZE]; struct ncplane* ffmpeg_subtitle(ncplane* parent, const ncvisual* ncv){ + if(!ncv->details->subtitle_active || ncv->details->subtitle.num_rects == 0){ + return NULL; + } + double now = ffmpeg_get_video_position_seconds(ncv); + if(now >= 0.0){ + if(now < ncv->details->subtitle_start_time){ + return NULL; + } + if(now > ncv->details->subtitle_end_time){ + SUBLOG_DEBUG("subtitle expired at %.3f (end %.3f)", now, ncv->details->subtitle_end_time); + avsubtitle_free(&ncv->details->subtitle); + ncv->details->subtitle_active = false; + return NULL; + } + } for(unsigned i = 0 ; i < ncv->details->subtitle.num_rects ; ++i){ // it is possible that there are more than one subtitle rects present, // but we only bother dealing with the first one we find FIXME? const AVSubtitleRect* rect = ncv->details->subtitle.rects[i]; if(rect->type == SUBTITLE_ASS){ - char* ass = deass(rect->ass); - struct ncplane* n = NULL; - if(ass){ - n = subtitle_plane_from_text(parent, ass); + char* ass = rect->ass ? deass(rect->ass, ncv->details->subtitle_text_field) : NULL; + if(!ass && rect->text){ + ass = strdup(rect->text); + } + if(!ass){ + SUBLOG_DEBUG("ASS subtitle missing text, skipping"); + return NULL; + } + struct ncplane* n = subtitle_plane_from_text(parent, ass, &ncv->details->subtitle_logged); + if(!n){ + SUBLOG_WARN("failed to render ASS subtitle: \"%s\"", ass); } free(ass); return n; - }else if(rect->type == SUBTITLE_TEXT){; - return subtitle_plane_from_text(parent, rect->text); + }else if(rect->type == SUBTITLE_TEXT){ + return subtitle_plane_from_text(parent, rect->text, &ncv->details->subtitle_logged); }else if(rect->type == SUBTITLE_BITMAP){ // there are technically up to AV_NUM_DATA_POINTERS planes, but we // only try to work with the first FIXME? if(rect->linesize[0] != rect->w){ -//logwarn("bitmap subtitle size %d != width %d\n", rect->linesize[0], rect->w); + SUBLOG_WARN("bitmap data linesize %d != width %d", rect->linesize[0], rect->w); continue; } struct notcurses* nc = ncplane_notcurses(parent); @@ -347,6 +504,7 @@ struct ncplane* ffmpeg_subtitle(ncplane* parent, const ncvisual* ncv){ rect->w, rect->w, NCPALETTESIZE, 1, palette); if(v == NULL){ + SUBLOG_WARN("failed to construct ncvisual for bitmap subtitle"); return NULL; } int rows = (rect->h + cellpxx - 1) / cellpxy; @@ -359,6 +517,7 @@ struct ncplane* ffmpeg_subtitle(ncplane* parent, const ncvisual* ncv){ struct ncplane* vn = ncplane_create(parent, &nopts); if(vn == NULL){ ncvisual_destroy(v); + SUBLOG_WARN("failed to allocate ncplane for bitmap subtitle"); return NULL; } struct ncvisual_options vopts = { @@ -369,6 +528,7 @@ struct ncplane* ffmpeg_subtitle(ncplane* parent, const ncvisual* ncv){ if(ncvisual_blit(nc, v, &vopts) == NULL){ ncplane_destroy(vn); ncvisual_destroy(v); + SUBLOG_WARN("failed to blit bitmap subtitle rect"); return NULL; } ncvisual_destroy(v); @@ -490,11 +650,52 @@ ffmpeg_decode(ncvisual* n){ // Handle subtitle packets if(n->details->packet->stream_index == n->details->sub_stream_index){ - int result = 0, ret; - avsubtitle_free(&n->details->subtitle); - ret = avcodec_decode_subtitle2(n->details->subtcodecctx, &n->details->subtitle, &result, n->details->packet); + int result = 0; + AVSubtitle decoded = {0}; + int ret = avcodec_decode_subtitle2(n->details->subtcodecctx, &decoded, &result, n->details->packet); if(ret >= 0 && result){ - // FIXME? + SUBLOG_DEBUG("packet pts=%" PRId64 " produced %u rects (format %d)", + n->details->packet->pts, decoded.num_rects, + decoded.format); + avsubtitle_free(&n->details->subtitle); + n->details->subtitle = decoded; + if(decoded.num_rects > 0){ + double base_time = ffmpeg_get_video_position_seconds(n); + int64_t subtitle_pts = decoded.pts; + if(subtitle_pts == AV_NOPTS_VALUE){ + subtitle_pts = n->details->packet->pts; + } + if(subtitle_pts != AV_NOPTS_VALUE && n->details->subtitle_time_base > 0.0){ + base_time = subtitle_pts * n->details->subtitle_time_base; + } + if(base_time < 0.0){ + base_time = 0.0; + } + const double start_offset = decoded.start_display_time / 1000.0; + double end_offset = decoded.end_display_time / 1000.0; + double start_time = base_time + start_offset; + double end_time = (end_offset > 0.0) ? base_time + end_offset + : start_time + 5.0; + if(end_time <= start_time){ + end_time = start_time + 5.0; + } + n->details->subtitle_start_time = start_time; + n->details->subtitle_end_time = end_time; + n->details->subtitle_active = true; + n->details->subtitle_logged = false; + SUBLOG_DEBUG("subtitle scheduled start=%.3f end=%.3f pts=%" PRId64 + " start_ms=%u end_ms=%u base=%.3f tb=%.6f", + start_time, end_time, subtitle_pts, + decoded.start_display_time, decoded.end_display_time, + base_time, n->details->subtitle_time_base); + }else{ + n->details->subtitle_active = false; + } + }else{ + if(ret < 0){ + SUBLOG_WARN("decode failure (%d) on pts=%" PRId64, ret, n->details->packet->pts); + } + avsubtitle_free(&decoded); } continue; // Continue reading, this wasn't a video packet } @@ -625,6 +826,12 @@ ffmpeg_details_init(void){ free(deets); return NULL; } + deets->subtitle_active = false; + deets->subtitle_start_time = 0.0; + deets->subtitle_end_time = 0.0; + deets->subtitle_time_base = 0.0; + deets->subtitle_text_field = 9; + deets->subtitle_logged = false; } return deets; } @@ -672,11 +879,35 @@ ffmpeg_from_file(const char* filename){ //fprintf(stderr, "Couldn't allocate decoder for %s\n", filename); goto err; } - // FIXME do we need avcodec_parameters_to_context() here? + AVStream* sst = ncv->details->fmtctx->streams[ncv->details->sub_stream_index]; + ncv->details->subtitle_time_base = av_q2d(sst->time_base); + if(ncv->details->subtitle_time_base <= 0.0){ + ncv->details->subtitle_time_base = 1.0 / AV_TIME_BASE; + } + if(ncv->details->subtcodecctx->subtitle_header && + ncv->details->subtcodecctx->subtitle_header_size > 0){ + ncv->details->subtitle_text_field = + ass_detect_text_field(ncv->details->subtcodecctx->subtitle_header, + ncv->details->subtcodecctx->subtitle_header_size); + } + if(avcodec_parameters_to_context(ncv->details->subtcodecctx, sst->codecpar) < 0){ + goto err; + } if(avcodec_open2(ncv->details->subtcodecctx, ncv->details->subtcodec, NULL) < 0){ //fprintf(stderr, "Couldn't open codec for %s (%s)\n", filename, av_err2str(*averr)); goto err; } + const char* subtitle_codec_name = "unknown"; + int subtitle_codec_id = -1; + if(ncv->details->subtcodec){ + subtitle_codec_id = (int)ncv->details->subtcodec->id; + if(ncv->details->subtcodec->name){ + subtitle_codec_name = ncv->details->subtcodec->name; + } + } + SUBLOG_INFO("stream %d (%s) discovered with codec id %d", + ncv->details->sub_stream_index, + subtitle_codec_name, subtitle_codec_id); }else{ ncv->details->sub_stream_index = -1; } From fee25f3fd410b4a6243931fd52aa4e6f1d633b08 Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sun, 30 Nov 2025 10:46:15 -0600 Subject: [PATCH 10/23] fixup --- src/media/ffmpeg.c | 86 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 74 insertions(+), 12 deletions(-) diff --git a/src/media/ffmpeg.c b/src/media/ffmpeg.c index a28af1afe..56399ad78 100644 --- a/src/media/ffmpeg.c +++ b/src/media/ffmpeg.c @@ -100,6 +100,19 @@ ass_detect_text_field(const uint8_t* header, size_t size){ return default_index; } +static int64_t +ass_text_hash(const char* text){ + if(!text){ + return 0; + } + int64_t hash = 1469598103934665603ull; + while(*text){ + hash ^= (unsigned char)(*text++); + hash *= 1099511628211ull; + } + return hash; +} + static int ffmpeg_detect_thread_count(void){ static int cached = 0; @@ -157,6 +170,7 @@ typedef struct ncvisual_details { double subtitle_time_base; int subtitle_text_field; bool subtitle_logged; + int64_t subtitle_last_logged_hash; int stream_index; // match against this following av_read_frame() int sub_stream_index; // subtitle stream index, can be < 0 if no subtitles int audio_stream_index; // audio stream index, can be < 0 if no audio @@ -414,17 +428,56 @@ subtitle_plane_from_text(ncplane* parent, const char* text, bool* logged_flag){ } int linecount = 1; for(size_t i = 0 ; i < len ; ++i){ - if(trimmed[i] == '\n' || trimmed[i] == '\r'){ + if(trimmed[i] == '\r'){ + trimmed[i] = '\n'; + } + if(trimmed[i] == '\n'){ ++linecount; - if(trimmed[i] == '\r' && trimmed[i + 1] == '\n'){ - ++i; - } } } + + int parent_cols = ncplane_dim_x(parent); + if(parent_cols <= 0){ + free(dup); + return NULL; + } + int maxwidth = 0; + char* walker = trimmed; + for(int line = 0 ; line < linecount ; ++line){ + char* next = strchr(walker, '\n'); + if(next){ + *next = '\0'; + } + int w = ncstrwidth(walker, NULL, NULL); + if(w > maxwidth){ + maxwidth = w; + } + if(next){ + *next = '\n'; + walker = next + 1; + }else{ + break; + } + } + if(maxwidth <= 0){ + free(dup); + return NULL; + } + int cols = maxwidth + 2; + if(cols > parent_cols){ + cols = parent_cols; + } + if(cols <= 0){ + free(dup); + return NULL; + } + int xpos = (parent_cols - cols) / 2; + struct ncplane_options nopts = { .y = ncplane_dim_y(parent) - (linecount + 1), + .x = xpos, .rows = linecount, - .cols = ncplane_dim_x(parent), + .cols = cols, .name = "subt", }; struct ncplane* n = ncplane_create(parent, &nopts); @@ -433,17 +486,18 @@ subtitle_plane_from_text(ncplane* parent, const char* text, bool* logged_flag){ return NULL; } uint64_t channels = 0; - ncchannels_set_fg_alpha(&channels, NCALPHA_HIGHCONTRAST); - ncchannels_set_fg_rgb8(&channels, 0x88, 0x88, 0x88); - ncplane_stain(n, -1, -1, 0, 0, channels, channels, channels, channels); - ncchannels_set_fg_default(&channels); + ncchannels_set_fg_alpha(&channels, NCALPHA_TRANSPARENT); + ncchannels_set_bg_alpha(&channels, NCALPHA_TRANSPARENT); + ncplane_set_base(n, " ", 0, channels); + + ncplane_set_fg_alpha(n, NCALPHA_HIGHCONTRAST); + ncplane_set_fg_rgb8(n, 0x88, 0x88, 0x88); + ncplane_set_bg_alpha(n, NCALPHA_TRANSPARENT); if(logged_flag && !*logged_flag){ SUBLOG_DEBUG("rendering subtitle text: \"%s\"", trimmed); *logged_flag = true; } - ncplane_puttext(n, 0, NCALIGN_CENTER, trimmed, NULL); - ncchannels_set_bg_alpha(&channels, NCALPHA_TRANSPARENT); - ncplane_set_base(n, " ", 0, channels); + ncplane_puttext(n, 0, NCALIGN_LEFT, trimmed, NULL); free(dup); return n; } @@ -471,6 +525,13 @@ struct ncplane* ffmpeg_subtitle(ncplane* parent, const ncvisual* ncv){ // but we only bother dealing with the first one we find FIXME? const AVSubtitleRect* rect = ncv->details->subtitle.rects[i]; if(rect->type == SUBTITLE_ASS){ + if(rect->ass){ + int64_t hash = ass_text_hash(rect->ass); + if(hash != ncv->details->subtitle_last_logged_hash){ + SUBLOG_DEBUG("raw ASS text: \"%s\"", rect->ass); + ncv->details->subtitle_last_logged_hash = hash; + } + } char* ass = rect->ass ? deass(rect->ass, ncv->details->subtitle_text_field) : NULL; if(!ass && rect->text){ ass = strdup(rect->text); @@ -832,6 +893,7 @@ ffmpeg_details_init(void){ deets->subtitle_time_base = 0.0; deets->subtitle_text_field = 9; deets->subtitle_logged = false; + deets->subtitle_last_logged_hash = 0; } return deets; } From b7fa055107720906cf8b8d4388979edd06e9156c Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sun, 30 Nov 2025 11:26:55 -0600 Subject: [PATCH 11/23] fixup --- src/media/ffmpeg.c | 130 ++++++--------------------------------------- 1 file changed, 17 insertions(+), 113 deletions(-) diff --git a/src/media/ffmpeg.c b/src/media/ffmpeg.c index 56399ad78..d95c3d9e4 100644 --- a/src/media/ffmpeg.c +++ b/src/media/ffmpeg.c @@ -62,11 +62,14 @@ ass_detect_text_field(const uint8_t* header, size_t size){ const char* data = (const char*)header; const char* end = data + size; const int default_index = 9; + bool in_events = false; while(data < end){ const char* line_start = data; const char* linebreak = memchr(line_start, '\n', end - line_start); size_t linelen = linebreak ? (size_t)(linebreak - line_start) : (size_t)(end - line_start); - if(linelen >= 7 && !strncasecmp(line_start, "Format:", 7)){ + if(linelen >= 8 && !strncasecmp(line_start, "[Events]", 8)){ + in_events = true; + }else if(in_events && linelen >= 7 && !strncasecmp(line_start, "Format:", 7)){ const char* cursor = line_start + 7; while(cursor < line_start + linelen && isspace((unsigned char)*cursor)){ ++cursor; @@ -100,19 +103,6 @@ ass_detect_text_field(const uint8_t* header, size_t size){ return default_index; } -static int64_t -ass_text_hash(const char* text){ - if(!text){ - return 0; - } - int64_t hash = 1469598103934665603ull; - while(*text){ - hash ^= (unsigned char)(*text++); - hash *= 1099511628211ull; - } - return hash; -} - static int ffmpeg_detect_thread_count(void){ static int cached = 0; @@ -170,7 +160,8 @@ typedef struct ncvisual_details { double subtitle_time_base; int subtitle_text_field; bool subtitle_logged; - int64_t subtitle_last_logged_hash; + int64_t subtitle_last_hash; + char* subtitle_cached_text; int stream_index; // match against this following av_read_frame() int sub_stream_index; // subtitle stream index, can be < 0 if no subtitles int audio_stream_index; // audio stream index, can be < 0 if no audio @@ -323,88 +314,6 @@ print_frame_summary(const AVCodecContext* cctx, const AVFrame* f){ f->quality); }*/ -static char* -deass(const char* ass, int text_field_index){ - if(!ass){ - return NULL; - } - const char* dialog = strstr(ass, "Dialogue:"); - const char* payload = dialog ? dialog + strlen("Dialogue:") : ass; - while(*payload == ' '){ - ++payload; - } - const char* text = payload; - int field = 0; - bool in_field_brace = false; - bool found_field = false; - for(const char* src = payload ; *src ; ++src){ - if(*src == '{'){ - in_field_brace = true; - }else if(*src == '}'){ - in_field_brace = false; - } - if(!in_field_brace && *src == ','){ - ++field; - if(field == text_field_index){ - text = src + 1; - found_field = true; - break; - } - } - } - if(!found_field){ - const char* lastcomma = strrchr(payload, ','); - if(lastcomma){ - text = lastcomma + 1; - } - } - while(*text == ' ' || *text == '\t'){ - ++text; - } - size_t textlen = strlen(text); - char* buf = malloc(textlen + 1); - if(!buf){ - return NULL; - } - char* dst = buf; - bool in_brace = false; - for(const char* src = text ; *src ; ++src){ - if(in_brace){ - if(*src == '}'){ - in_brace = false; - } - continue; - } - if(*src == '{'){ - in_brace = true; - continue; - } - if(*src == '\\'){ - ++src; - if(!*src){ - break; - } - if(*src == 'N' || *src == 'n'){ - *dst++ = '\n'; - } - continue; - } - *dst++ = *src; - } - *dst = '\0'; - char* trimmed = buf; - while(*trimmed && isspace((unsigned char)*trimmed)){ - ++trimmed; - } - size_t len = strlen(trimmed); - while(len > 0 && isspace((unsigned char)trimmed[len - 1])){ - trimmed[--len] = '\0'; - } - char* out = len ? strdup(trimmed) : NULL; - free(buf); - return out; -} - static struct ncplane* subtitle_plane_from_text(ncplane* parent, const char* text, bool* logged_flag){ if(parent == NULL || text == NULL){ @@ -525,26 +434,16 @@ struct ncplane* ffmpeg_subtitle(ncplane* parent, const ncvisual* ncv){ // but we only bother dealing with the first one we find FIXME? const AVSubtitleRect* rect = ncv->details->subtitle.rects[i]; if(rect->type == SUBTITLE_ASS){ - if(rect->ass){ - int64_t hash = ass_text_hash(rect->ass); - if(hash != ncv->details->subtitle_last_logged_hash){ - SUBLOG_DEBUG("raw ASS text: \"%s\"", rect->ass); - ncv->details->subtitle_last_logged_hash = hash; - } - } - char* ass = rect->ass ? deass(rect->ass, ncv->details->subtitle_text_field) : NULL; - if(!ass && rect->text){ - ass = strdup(rect->text); - } - if(!ass){ + const char* render_text = NULL; + render_text = rect->ass ? rect->ass : rect->text; + if(!render_text){ SUBLOG_DEBUG("ASS subtitle missing text, skipping"); return NULL; } - struct ncplane* n = subtitle_plane_from_text(parent, ass, &ncv->details->subtitle_logged); + struct ncplane* n = subtitle_plane_from_text(parent, render_text, &ncv->details->subtitle_logged); if(!n){ - SUBLOG_WARN("failed to render ASS subtitle: \"%s\"", ass); + SUBLOG_WARN("failed to render ASS subtitle: \"%s\"", render_text); } - free(ass); return n; }else if(rect->type == SUBTITLE_TEXT){ return subtitle_plane_from_text(parent, rect->text, &ncv->details->subtitle_logged); @@ -893,7 +792,8 @@ ffmpeg_details_init(void){ deets->subtitle_time_base = 0.0; deets->subtitle_text_field = 9; deets->subtitle_logged = false; - deets->subtitle_last_logged_hash = 0; + deets->subtitle_last_hash = 0; + deets->subtitle_cached_text = NULL; } return deets; } @@ -951,6 +851,9 @@ ffmpeg_from_file(const char* filename){ ncv->details->subtitle_text_field = ass_detect_text_field(ncv->details->subtcodecctx->subtitle_header, ncv->details->subtcodecctx->subtitle_header_size); + if(ncv->details->subtitle_text_field < 1){ + ncv->details->subtitle_text_field = 9; + } } if(avcodec_parameters_to_context(ncv->details->subtcodecctx, sst->codecpar) < 0){ goto err; @@ -1410,6 +1313,7 @@ ffmpeg_details_destroy(ncvisual_details* deets){ av_packet_free(&deets->packet); avformat_close_input(&deets->fmtctx); avsubtitle_free(&deets->subtitle); + free(deets->subtitle_cached_text); pthread_mutex_destroy(&deets->audio_packet_mutex); pthread_mutex_destroy(&deets->packet_mutex); free(deets); From bfb7d1791f3b0aca681187873d0281ae8b9d1c0d Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sun, 30 Nov 2025 11:49:34 -0600 Subject: [PATCH 12/23] fixup --- src/media/ffmpeg.c | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/src/media/ffmpeg.c b/src/media/ffmpeg.c index d95c3d9e4..cd980eddb 100644 --- a/src/media/ffmpeg.c +++ b/src/media/ffmpeg.c @@ -324,6 +324,17 @@ subtitle_plane_from_text(ncplane* parent, const char* text, bool* logged_flag){ return NULL; } char* trimmed = dup; + const char* raw = dup; + int commas = 0; + for(; *trimmed ; ++trimmed){ + if(*trimmed == ','){ + ++commas; + if(commas == 8){ + ++trimmed; + break; + } + } + } while(*trimmed && isspace((unsigned char)*trimmed)){ ++trimmed; } @@ -372,15 +383,8 @@ subtitle_plane_from_text(ncplane* parent, const char* text, bool* logged_flag){ free(dup); return NULL; } - int cols = maxwidth + 2; - if(cols > parent_cols){ - cols = parent_cols; - } - if(cols <= 0){ - free(dup); - return NULL; - } - int xpos = (parent_cols - cols) / 2; + int xpos = 0; + int cols = parent_cols; struct ncplane_options nopts = { .y = ncplane_dim_y(parent) - (linecount + 1), @@ -403,10 +407,11 @@ subtitle_plane_from_text(ncplane* parent, const char* text, bool* logged_flag){ ncplane_set_fg_rgb8(n, 0x88, 0x88, 0x88); ncplane_set_bg_alpha(n, NCALPHA_TRANSPARENT); if(logged_flag && !*logged_flag){ - SUBLOG_DEBUG("rendering subtitle text: \"%s\"", trimmed); + SUBLOG_DEBUG("rendering subtitle text (raw): \"%s\"", raw); + SUBLOG_DEBUG("rendering subtitle text (trimmed): \"%s\"", trimmed); *logged_flag = true; } - ncplane_puttext(n, 0, NCALIGN_LEFT, trimmed, NULL); + ncplane_puttext(n, 0, NCALIGN_CENTER, trimmed, NULL); free(dup); return n; } From 32575129e099dc4c93b6588e13314b9e8079fca4 Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sun, 30 Nov 2025 12:01:33 -0600 Subject: [PATCH 13/23] fixup (quad/sex around subtitles fixed) --- src/media/ffmpeg.c | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/src/media/ffmpeg.c b/src/media/ffmpeg.c index cd980eddb..60466aa6a 100644 --- a/src/media/ffmpeg.c +++ b/src/media/ffmpeg.c @@ -383,8 +383,11 @@ subtitle_plane_from_text(ncplane* parent, const char* text, bool* logged_flag){ free(dup); return NULL; } - int xpos = 0; - int cols = parent_cols; + int cols = maxwidth; + if(cols > parent_cols){ + cols = parent_cols; + } + int xpos = (parent_cols - cols) / 2; struct ncplane_options nopts = { .y = ncplane_dim_y(parent) - (linecount + 1), @@ -411,7 +414,20 @@ subtitle_plane_from_text(ncplane* parent, const char* text, bool* logged_flag){ SUBLOG_DEBUG("rendering subtitle text (trimmed): \"%s\"", trimmed); *logged_flag = true; } - ncplane_puttext(n, 0, NCALIGN_CENTER, trimmed, NULL); + char* linewalker = trimmed; + for(int line = 0 ; line < linecount ; ++line){ + char* next = strchr(linewalker, '\n'); + if(next){ + *next = '\0'; + } + ncplane_putstr_yx(n, line, 0, linewalker); + if(next){ + *next = '\n'; + linewalker = next + 1; + }else{ + break; + } + } free(dup); return n; } From 0f1d9a8407db0860d6b732523040be1fc6a419a5 Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sun, 30 Nov 2025 12:07:18 -0600 Subject: [PATCH 14/23] fixup (subtitles look good!) --- src/media/ffmpeg.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/media/ffmpeg.c b/src/media/ffmpeg.c index 60466aa6a..b85cd4679 100644 --- a/src/media/ffmpeg.c +++ b/src/media/ffmpeg.c @@ -402,13 +402,16 @@ subtitle_plane_from_text(ncplane* parent, const char* text, bool* logged_flag){ return NULL; } uint64_t channels = 0; - ncchannels_set_fg_alpha(&channels, NCALPHA_TRANSPARENT); - ncchannels_set_bg_alpha(&channels, NCALPHA_TRANSPARENT); + ncchannels_set_fg_rgb8(&channels, 0xff, 0xff, 0xff); + ncchannels_set_fg_alpha(&channels, NCALPHA_OPAQUE); + ncchannels_set_bg_rgb8(&channels, 0x20, 0x20, 0x20); + ncchannels_set_bg_alpha(&channels, NCALPHA_BLEND); ncplane_set_base(n, " ", 0, channels); - ncplane_set_fg_alpha(n, NCALPHA_HIGHCONTRAST); - ncplane_set_fg_rgb8(n, 0x88, 0x88, 0x88); - ncplane_set_bg_alpha(n, NCALPHA_TRANSPARENT); + ncplane_set_fg_rgb8(n, 0xff, 0xff, 0xff); + ncplane_set_fg_alpha(n, NCALPHA_OPAQUE); + ncplane_set_bg_rgb8(n, 0x20, 0x20, 0x20); + ncplane_set_bg_alpha(n, NCALPHA_BLEND); if(logged_flag && !*logged_flag){ SUBLOG_DEBUG("rendering subtitle text (raw): \"%s\"", raw); SUBLOG_DEBUG("rendering subtitle text (trimmed): \"%s\"", trimmed); From aeedae6642488dfbe521795c5d330c6d55085bbb Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sun, 30 Nov 2025 12:40:30 -0600 Subject: [PATCH 15/23] try to make window resizing possible --- src/player/play.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/player/play.cpp b/src/player/play.cpp index 049c66182..762d72019 100644 --- a/src/player/play.cpp +++ b/src/player/play.cpp @@ -185,7 +185,10 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, clock_nanosleep(CLOCK_MONOTONIC, 0, &sleep_ts, NULL); } - struct ncplane* subp = ncvisual_subtitle_plane(*stdn, ncv); + auto recreate_subtitle_plane = [&]() -> struct ncplane* { + return ncvisual_subtitle_plane(*stdn, ncv); + }; + struct ncplane* subp = recreate_subtitle_plane(); int64_t remaining = display_ns; const int64_t h = remaining / (60 * 60 * NANOSECS_IN_SEC); remaining -= h * (60 * 60 * NANOSECS_IN_SEC); @@ -241,7 +244,15 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, } // if we just hit a non-space character to unpause, ignore it if(keyp == NCKey::Resize){ - return 0; + if(!nc.refresh(&dimy, &dimx)){ + ncplane_destroy(subp); + return -1; + } + if(subp){ + ncplane_destroy(subp); + } + subp = recreate_subtitle_plane(); + continue; }else if(keyp == ' '){ // space for unpause continue; }else if(keyp == 'L' && ncinput_ctrl_p(&ni) && !ncinput_alt_p(&ni)){ From 2715de4206025bcc75f69f87120b47b7526403f8 Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sun, 30 Nov 2025 13:21:40 -0600 Subject: [PATCH 16/23] add seek absolute --- src/player/play.cpp | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/src/player/play.cpp b/src/player/play.cpp index 762d72019..97e87c4a0 100644 --- a/src/player/play.cpp +++ b/src/player/play.cpp @@ -15,6 +15,7 @@ #include #include #include +#include extern "C" { #include #include @@ -84,6 +85,7 @@ struct marshal { double current_drop_pct; PlaybackRequest request; double seek_delta; + bool seek_absolute; }; static constexpr double kNcplayerSeekSeconds = 5.0; @@ -248,11 +250,15 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, ncplane_destroy(subp); return -1; } - if(subp){ - ncplane_destroy(subp); + double resume = ffmpeg_get_video_position_seconds(ncv); + if(!std::isfinite(resume)){ + resume = static_cast(display_ns) / 1e9; } - subp = recreate_subtitle_plane(); - continue; + marsh->request = PlaybackRequest::Seek; + marsh->seek_delta = resume; + marsh->seek_absolute = true; + ncplane_destroy(subp); + return 2; }else if(keyp == ' '){ // space for unpause continue; }else if(keyp == 'L' && ncinput_ctrl_p(&ni) && !ncinput_alt_p(&ni)){ @@ -270,21 +276,25 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, }else if(keyp == NCKey::Up){ marsh->request = PlaybackRequest::Seek; marsh->seek_delta = -kNcplayerSeekMinutes; + marsh->seek_absolute = false; ncplane_destroy(subp); return 2; }else if(keyp == NCKey::Down){ marsh->request = PlaybackRequest::Seek; marsh->seek_delta = kNcplayerSeekMinutes; + marsh->seek_absolute = false; ncplane_destroy(subp); return 2; }else if(keyp == NCKey::Right){ marsh->request = PlaybackRequest::Seek; marsh->seek_delta = kNcplayerSeekSeconds; + marsh->seek_absolute = false; ncplane_destroy(subp); return 2; }else if(keyp == NCKey::Left){ marsh->request = PlaybackRequest::Seek; marsh->seek_delta = -kNcplayerSeekSeconds; + marsh->seek_absolute = false; ncplane_destroy(subp); return 2; }else if(keyp != 'q'){ @@ -645,6 +655,7 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, .current_drop_pct = 0.0, .request = PlaybackRequest::None, .seek_delta = 0.0, + .seek_absolute = false, }; r = ncv->stream(&vopts, timescale, perframe, &marsh); pending_request = marsh.request; @@ -669,9 +680,18 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, restart_stream = false; if(pending_request == PlaybackRequest::Seek){ double delta = marsh.seek_delta; + if(marsh.seek_absolute){ + double current = ffmpeg_get_video_position_seconds(*ncv); + if(!std::isfinite(current)){ + current = 0.0; + } + delta = marsh.seek_delta - current; + marsh.seek_absolute = false; + } if(ncvisual_seek(*ncv, delta) == 0){ pending_request = PlaybackRequest::None; restart_stream = true; + marsh.seek_delta = 0.0; }else{ pending_request = PlaybackRequest::None; } From 50bfd3c771deaa6cff90e394bdab11acb381b455 Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sun, 30 Nov 2025 14:02:31 -0600 Subject: [PATCH 17/23] partial resize works --- src/player/play.cpp | 93 +++++++++++++++++++++++++++++++-------------- 1 file changed, 64 insertions(+), 29 deletions(-) diff --git a/src/player/play.cpp b/src/player/play.cpp index 97e87c4a0..3d4e4c882 100644 --- a/src/player/play.cpp +++ b/src/player/play.cpp @@ -586,10 +586,6 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, for(auto i = 0 ; i < argc ; ++i){ std::unique_ptr ncv; ncv = std::make_unique(argv[i]); - if((n = ncplane_create(*stdn, &nopts)) == nullptr){ - return -1; - } - ncplane_move_bottom(n); struct ncvisual_options vopts{}; int r; if(noninterp){ @@ -599,35 +595,66 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, vopts.flags |= NCVISUAL_OPTION_ADDALPHA; } vopts.transcolor = transcolor & 0xffffffull; - vopts.n = n; vopts.scaling = scalemode; vopts.blitter = blitter; - if(!climode){ - vopts.flags |= NCVISUAL_OPTION_HORALIGNED | NCVISUAL_OPTION_VERALIGNED; - vopts.y = NCALIGN_CENTER; - vopts.x = NCALIGN_CENTER; - }else{ - ncvgeom geom; - if(ncvisual_geom(nc, *ncv, &vopts, &geom)){ - return -1; + + auto recreate_visual_planes = [&]() -> bool { + if(n){ + ncplane_destroy(n); + n = nullptr; } - struct ncplane_options cliopts{}; - cliopts.y = stdn->cursor_y(); - cliopts.x = stdn->cursor_x(); - cliopts.rows = geom.rcelly; - cliopts.cols = geom.rcellx; - clin = ncplane_create(n, &cliopts); - if(!clin){ - return -1; + if(clin){ + ncplane_destroy(clin); + clin = nullptr; + } + n = ncplane_create(*stdn, &nopts); + if(n == nullptr){ + return false; + } + ncplane_move_bottom(n); + vopts.n = n; + if(climode){ + ncvgeom geom; + if(ncvisual_geom(nc, *ncv, &vopts, &geom)){ + return false; + } + struct ncplane_options cliopts{}; + cliopts.y = stdn->cursor_y(); + cliopts.x = stdn->cursor_x(); + cliopts.rows = geom.rcelly; + cliopts.cols = geom.rcellx; + clin = ncplane_create(n, &cliopts); + if(!clin){ + return false; + } + vopts.n = clin; + ncplane_scrollup_child(*stdn, clin); + }else{ + vopts.flags |= NCVISUAL_OPTION_HORALIGNED | NCVISUAL_OPTION_VERALIGNED; + vopts.y = NCALIGN_CENTER; + vopts.x = NCALIGN_CENTER; + vopts.n = n; } - vopts.n = clin; - ncplane_scrollup_child(*stdn, clin); + ncplane_erase(n); + return true; + }; + + if(!recreate_visual_planes()){ + return -1; } - ncplane_erase(n); PlaybackRequest pending_request = PlaybackRequest::None; + bool needs_plane_recreate = false; + double pending_seek_value = 0.0; + bool pending_seek_absolute = false; while(true){ bool restart_stream = false; + if(needs_plane_recreate){ + if(!recreate_visual_planes()){ + goto err; + } + needs_plane_recreate = false; + } if(ffmpeg_has_audio(*ncv)){ int sample_rate = 44100; int channels = ffmpeg_get_audio_channels(*ncv); @@ -659,6 +686,10 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, }; r = ncv->stream(&vopts, timescale, perframe, &marsh); pending_request = marsh.request; + if(pending_request == PlaybackRequest::Seek){ + pending_seek_value = marsh.seek_delta; + pending_seek_absolute = marsh.seek_absolute; + } show_fps_overlay = marsh.show_fps; if(audio_thread){ audio_running = false; @@ -679,19 +710,23 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, stdn->set_userptr(nullptr); restart_stream = false; if(pending_request == PlaybackRequest::Seek){ - double delta = marsh.seek_delta; - if(marsh.seek_absolute){ + double delta = pending_seek_value; + bool was_absolute = pending_seek_absolute; + if(pending_seek_absolute){ double current = ffmpeg_get_video_position_seconds(*ncv); if(!std::isfinite(current)){ current = 0.0; } - delta = marsh.seek_delta - current; - marsh.seek_absolute = false; + delta = pending_seek_value - current; + pending_seek_absolute = false; } if(ncvisual_seek(*ncv, delta) == 0){ pending_request = PlaybackRequest::None; restart_stream = true; - marsh.seek_delta = 0.0; + pending_seek_value = 0.0; + if(was_absolute){ + needs_plane_recreate = true; + } }else{ pending_request = PlaybackRequest::None; } From dc1c8a0ec239bff23196486d7bd3a44ef67c9edd Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sun, 30 Nov 2025 15:38:22 -0600 Subject: [PATCH 18/23] resize works! --- src/player/play.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/player/play.cpp b/src/player/play.cpp index 3d4e4c882..23ea89235 100644 --- a/src/player/play.cpp +++ b/src/player/play.cpp @@ -27,6 +27,7 @@ extern "C" { #include #include "compat/compat.h" #include "media/audio-output.h" +#include "lib/logging.h" using namespace ncpp; @@ -250,6 +251,7 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, ncplane_destroy(subp); return -1; } + logdebug("[resize] resize event captured (%ux%u)", dimx, dimy); double resume = ffmpeg_get_video_position_seconds(ncv); if(!std::isfinite(resume)){ resume = static_cast(display_ns) / 1e9; @@ -598,7 +600,8 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, vopts.scaling = scalemode; vopts.blitter = blitter; - auto recreate_visual_planes = [&]() -> bool { + auto recreate_visual_planes = [&](const char* reason) -> bool { + logdebug("[resize] recreating planes due to %s", reason ? reason : "unknown"); if(n){ ncplane_destroy(n); n = nullptr; @@ -609,6 +612,7 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, } n = ncplane_create(*stdn, &nopts); if(n == nullptr){ + logerror("[resize] failed to create main plane"); return false; } ncplane_move_bottom(n); @@ -639,7 +643,7 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, return true; }; - if(!recreate_visual_planes()){ + if(!recreate_visual_planes("initial create")){ return -1; } @@ -650,11 +654,13 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, while(true){ bool restart_stream = false; if(needs_plane_recreate){ - if(!recreate_visual_planes()){ + logdebug("[resize] applying pending plane recreate"); + if(!recreate_visual_planes("pending resize")){ goto err; } needs_plane_recreate = false; } + /* capture needs_plane_recreate and apply before next stream */ if(ffmpeg_has_audio(*ncv)){ int sample_rate = 44100; int channels = ffmpeg_get_audio_channels(*ncv); From 9420183d0296de58437252146df54af9880eba49 Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sun, 30 Nov 2025 17:12:09 -0600 Subject: [PATCH 19/23] fail to fix resize issue --- src/media/ffmpeg.c | 2 + src/player/play.cpp | 211 +++++++++++++++++++++++++++++++++++++------- 2 files changed, 181 insertions(+), 32 deletions(-) diff --git a/src/media/ffmpeg.c b/src/media/ffmpeg.c index b85cd4679..38401de75 100644 --- a/src/media/ffmpeg.c +++ b/src/media/ffmpeg.c @@ -89,6 +89,7 @@ ass_detect_text_field(const uint8_t* header, size_t size){ if(q > t){ size_t toklen = q - t; if(!strncasecmp(t, "Text", toklen)){ + logdebug("[ass] format text field index=%d column=\"%.*s\"", field, (int)toklen, t); return field; } } @@ -100,6 +101,7 @@ ass_detect_text_field(const uint8_t* header, size_t size){ } data = linebreak ? linebreak + 1 : end; } + logerror("[ass] no [Events] Format line found"); return default_index; } diff --git a/src/player/play.cpp b/src/player/play.cpp index 23ea89235..e9216d5ef 100644 --- a/src/player/play.cpp +++ b/src/player/play.cpp @@ -74,6 +74,63 @@ enum class PlaybackRequest { Seek, }; +struct plane_runtime { + struct timespec start; + std::atomic resize_pending; +}; + +static plane_runtime* +init_plane_runtime(ncplane* n){ + if(n == nullptr){ + return nullptr; + } + auto runtime = static_cast(ncplane_userptr(n)); + if(runtime == nullptr){ + runtime = new plane_runtime{}; + ncplane_set_userptr(n, runtime); + } + clock_gettime(CLOCK_MONOTONIC, &runtime->start); + runtime->resize_pending.store(false, std::memory_order_relaxed); + return runtime; +} + +static plane_runtime* +get_plane_runtime(ncplane* n){ + if(n == nullptr){ + return nullptr; + } + auto runtime = static_cast(ncplane_userptr(n)); + if(runtime == nullptr){ + runtime = init_plane_runtime(n); + } + return runtime; +} + +static void +destroy_plane_runtime(ncplane* n){ + if(n == nullptr){ + return; + } + if(auto runtime = static_cast(ncplane_userptr(n))){ + delete runtime; + ncplane_set_userptr(n, nullptr); + } +} + +static int player_plane_resize_cb(struct ncplane* n){ + if(auto runtime = static_cast(ncplane_userptr(n))){ + runtime->resize_pending.store(true, std::memory_order_release); + } + return 0; +} + +static int player_cli_resize_cb(struct ncplane* n){ + if(auto runtime = static_cast(ncplane_userptr(n))){ + runtime->resize_pending.store(true, std::memory_order_release); + } + return ncplane_resize_marginalized(n); +} + struct marshal { int framecount; bool quiet; @@ -87,6 +144,7 @@ struct marshal { PlaybackRequest request; double seek_delta; bool seek_absolute; + bool resize_restart_pending; }; static constexpr double kNcplayerSeekSeconds = 5.0; @@ -96,12 +154,9 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, const struct timespec* abstime, void* vmarshal) -> int { struct marshal* marsh = static_cast(vmarshal); NotCurses &nc = NotCurses::get_instance(); - auto start = static_cast(ncplane_userptr(vopts->n)); - if(!start){ - // FIXME how do we get this free()d at the end? - start = static_cast(malloc(sizeof(struct timespec))); - clock_gettime(CLOCK_MONOTONIC, start); - ncplane_set_userptr(vopts->n, start); + auto* runtime = get_plane_runtime(vopts->n); + if(runtime == nullptr){ + return -1; } std::unique_ptr stdn(nc.get_stdplane()); static auto last_fps_sample = std::chrono::steady_clock::now(); @@ -135,7 +190,7 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, }else{ struct timespec now; clock_gettime(CLOCK_MONOTONIC, &now); - display_ns = timespec_to_ns(&now) - timespec_to_ns(start); + display_ns = timespec_to_ns(&now) - timespec_to_ns(&runtime->start); } if(display_ns < 0){ display_ns = 0; @@ -192,6 +247,36 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, return ncvisual_subtitle_plane(*stdn, ncv); }; struct ncplane* subp = recreate_subtitle_plane(); + auto destroy_subtitle_plane = [&](){ + if(subp){ + ncplane_destroy(subp); + subp = nullptr; + } + }; + auto pop_async_resize = [&]() -> bool { + if(runtime && runtime->resize_pending.load(std::memory_order_acquire)){ + runtime->resize_pending.store(false, std::memory_order_release); + return true; + } + return false; + }; + auto request_resize_restart = [&](const char* src) -> int { + if(marsh->resize_restart_pending){ + logdebug("[resize] restart already pending, ignoring %s", src ? src : "unknown"); + return 0; + } + marsh->resize_restart_pending = true; + double resume = ffmpeg_get_video_position_seconds(ncv); + if(!std::isfinite(resume)){ + resume = static_cast(display_ns) / 1e9; + } + marsh->request = PlaybackRequest::Seek; + marsh->seek_delta = resume; + marsh->seek_absolute = true; + destroy_subtitle_plane(); + logdebug("[resize] async restart due to %s", src ? src : "unknown"); + return 2; + }; int64_t remaining = display_ns; const int64_t h = remaining / (60 * 60 * NANOSECS_IN_SEC); remaining -= h * (60 * 60 * NANOSECS_IN_SEC); @@ -206,11 +291,24 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, if(!nc.render()){ return -1; } + if(pop_async_resize()){ + int restart = request_resize_restart("plane resize callback"); + if(restart != 0){ + return restart; + } + } unsigned dimx, dimy, oldx, oldy; nc.get_term_dim(&dimy, &dimx); ncplane_dim_yx(vopts->n, &oldy, &oldx); uint64_t absnow = timespec_to_ns(abstime); for( ; ; ){ + if(pop_async_resize()){ + int restart = request_resize_restart("pending resize"); + if(restart != 0){ + return restart; + } + continue; + } struct timespec interval; clock_gettime(CLOCK_MONOTONIC, &interval); uint64_t nsnow = timespec_to_ns(&interval); @@ -229,7 +327,7 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, // Check for 'q' key immediately to allow quitting if((keyp == 'q' || keyp == 'Q') && ni.evtype != EvType::Release){ marsh->request = PlaybackRequest::Quit; - ncplane_destroy(subp); + destroy_subtitle_plane(); return 1; } // we don't care about key release events, especially the enter @@ -240,27 +338,27 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, if(keyp == ' '){ do{ if((keyp = nc.get(true, &ni)) == (uint32_t)-1){ - ncplane_destroy(subp); + destroy_subtitle_plane(); return -1; } }while(ni.id != 'q' && (ni.evtype == EvType::Release || ni.id != ' ')); } // if we just hit a non-space character to unpause, ignore it if(keyp == NCKey::Resize){ + if(marsh->resize_restart_pending){ + logdebug("[resize] restart already pending, skipping key resize"); + continue; + } if(!nc.refresh(&dimy, &dimx)){ - ncplane_destroy(subp); + destroy_subtitle_plane(); return -1; } logdebug("[resize] resize event captured (%ux%u)", dimx, dimy); - double resume = ffmpeg_get_video_position_seconds(ncv); - if(!std::isfinite(resume)){ - resume = static_cast(display_ns) / 1e9; + int restart = request_resize_restart("key resize"); + if(restart != 0){ + return restart; } - marsh->request = PlaybackRequest::Seek; - marsh->seek_delta = resume; - marsh->seek_absolute = true; - ncplane_destroy(subp); - return 2; + continue; }else if(keyp == ' '){ // space for unpause continue; }else if(keyp == 'L' && ncinput_ctrl_p(&ni) && !ncinput_alt_p(&ni)){ @@ -279,33 +377,34 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, marsh->request = PlaybackRequest::Seek; marsh->seek_delta = -kNcplayerSeekMinutes; marsh->seek_absolute = false; - ncplane_destroy(subp); + destroy_subtitle_plane(); return 2; }else if(keyp == NCKey::Down){ marsh->request = PlaybackRequest::Seek; marsh->seek_delta = kNcplayerSeekMinutes; marsh->seek_absolute = false; - ncplane_destroy(subp); + destroy_subtitle_plane(); return 2; }else if(keyp == NCKey::Right){ marsh->request = PlaybackRequest::Seek; marsh->seek_delta = kNcplayerSeekSeconds; marsh->seek_absolute = false; - ncplane_destroy(subp); + destroy_subtitle_plane(); return 2; }else if(keyp == NCKey::Left){ marsh->request = PlaybackRequest::Seek; marsh->seek_delta = -kNcplayerSeekSeconds; marsh->seek_absolute = false; - ncplane_destroy(subp); + destroy_subtitle_plane(); return 2; }else if(keyp != 'q'){ continue; } - ncplane_destroy(subp); + destroy_subtitle_plane(); return 1; } - ncplane_destroy(subp); + destroy_subtitle_plane(); + destroy_subtitle_plane(); return 0; } @@ -572,8 +671,13 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, stdn->set_base("", 0, transchan); struct ncplane_options nopts{}; nopts.name = "play"; - nopts.resizecb = ncplane_resize_marginalized; - nopts.flags = NCPLANE_OPTION_MARGINALIZED; + if(climode){ + nopts.resizecb = player_cli_resize_cb; + nopts.flags = NCPLANE_OPTION_MARGINALIZED; + }else{ + nopts.resizecb = player_plane_resize_cb; + nopts.flags = NCPLANE_OPTION_FIXED; + } ncplane* n = nullptr; ncplane* clin = nullptr; @@ -602,11 +706,28 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, auto recreate_visual_planes = [&](const char* reason) -> bool { logdebug("[resize] recreating planes due to %s", reason ? reason : "unknown"); + unsigned stdrows, stdcols; + ncplane_dim_yx(*stdn, &stdrows, &stdcols); + if(stdrows == 0 || stdcols == 0){ + logerror("[resize] stdn has zero dimensions (%u x %u)", stdrows, stdcols); + return false; + } + nopts.y = 0; + nopts.x = 0; + if(climode){ + nopts.rows = 0; + nopts.cols = 0; + }else{ + nopts.rows = stdrows; + nopts.cols = stdcols; + } if(n){ + destroy_plane_runtime(n); ncplane_destroy(n); n = nullptr; } if(clin){ + destroy_plane_runtime(clin); ncplane_destroy(clin); clin = nullptr; } @@ -639,7 +760,9 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, vopts.x = NCALIGN_CENTER; vopts.n = n; } + init_plane_runtime(vopts.n); ncplane_erase(n); + logdebug("[resize] planes ready main=%ux%u", ncplane_dim_x(vopts.n), ncplane_dim_y(vopts.n)); return true; }; @@ -651,7 +774,10 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, bool needs_plane_recreate = false; double pending_seek_value = 0.0; bool pending_seek_absolute = false; + bool preserve_audio = false; + int stream_iteration = 0; while(true){ + logdebug("[stream] begin iteration %d (need_recreate=%d, audio=%d)", stream_iteration++, needs_plane_recreate ? 1 : 0, audio_thread ? 1 : 0); bool restart_stream = false; if(needs_plane_recreate){ logdebug("[resize] applying pending plane recreate"); @@ -661,7 +787,7 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, needs_plane_recreate = false; } /* capture needs_plane_recreate and apply before next stream */ - if(ffmpeg_has_audio(*ncv)){ + if(ffmpeg_has_audio(*ncv) && audio_thread == nullptr){ int sample_rate = 44100; int channels = ffmpeg_get_audio_channels(*ncv); if(channels > 2){ @@ -669,10 +795,13 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, } ao = audio_output_init(sample_rate, channels, AV_SAMPLE_FMT_S16); if(ao){ + logdebug("[audio] starting audio thread"); audio_running = true; audio_data = new audio_thread_data{*ncv, ao, &audio_running, &audio_mutex}; audio_thread = new std::thread(audio_thread_func, audio_data); audio_output_start(ao); + }else{ + logerror("[audio] failed to init audio output"); } } @@ -689,28 +818,36 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, .request = PlaybackRequest::None, .seek_delta = 0.0, .seek_absolute = false, + .resize_restart_pending = false, }; + logdebug("[stream] starting ncvisual_stream iteration with %ux%u plane", vopts.n ? ncplane_dim_x(vopts.n) : 0, vopts.n ? ncplane_dim_y(vopts.n) : 0); r = ncv->stream(&vopts, timescale, perframe, &marsh); + logdebug("[stream] ncvisual_stream returned %d", r); pending_request = marsh.request; if(pending_request == PlaybackRequest::Seek){ pending_seek_value = marsh.seek_delta; pending_seek_absolute = marsh.seek_absolute; } show_fps_overlay = marsh.show_fps; - if(audio_thread){ + if(audio_thread && !preserve_audio){ + logdebug("[audio] stopping audio thread"); audio_running = false; audio_thread->join(); delete audio_thread; audio_thread = nullptr; } - if(audio_data){ + if(audio_data && !preserve_audio){ delete audio_data; audio_data = nullptr; } - if(ao){ + if(ao && !preserve_audio){ audio_output_destroy(ao); ao = nullptr; } + if(preserve_audio){ + logdebug("[resize] preserving audio thread during resize restart"); + preserve_audio = false; + } free(stdn->get_userptr()); stdn->set_userptr(nullptr); @@ -726,14 +863,18 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, delta = pending_seek_value - current; pending_seek_absolute = false; } + logdebug("[seek] request delta=%f (absolute=%d)", delta, was_absolute ? 1 : 0); if(ncvisual_seek(*ncv, delta) == 0){ pending_request = PlaybackRequest::None; restart_stream = true; pending_seek_value = 0.0; + logdebug("[seek] success"); if(was_absolute){ needs_plane_recreate = true; + preserve_audio = true; } }else{ + logerror("[seek] ncvisual_seek failed"); pending_request = PlaybackRequest::None; } } @@ -792,6 +933,7 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, } } if(clin){ + destroy_plane_runtime(clin); ncplane_destroy(clin); clin = nullptr; } @@ -799,7 +941,7 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, std::cerr << "Error while playing " << argv[i] << std::endl; goto err; } - free(ncplane_userptr(n)); + destroy_plane_runtime(n); ncplane_destroy(n); if(pending_request == PlaybackRequest::Quit){ return 0; @@ -850,7 +992,12 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, audio_output_destroy(ao); ao = nullptr; } - free(ncplane_userptr(n)); + if(clin){ + destroy_plane_runtime(clin); + ncplane_destroy(clin); + clin = nullptr; + } + destroy_plane_runtime(n); ncplane_destroy(n); return -1; } From e97ac4e6ffdcfd3d867d7e54cb7eedc208944aa6 Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sun, 30 Nov 2025 17:38:30 -0600 Subject: [PATCH 20/23] try to fix resize --- src/player/play.cpp | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/src/player/play.cpp b/src/player/play.cpp index e9216d5ef..b31f280b8 100644 --- a/src/player/play.cpp +++ b/src/player/play.cpp @@ -117,6 +117,8 @@ destroy_plane_runtime(ncplane* n){ } } +static void attach_plane_runtime(ncplane* n); + static int player_plane_resize_cb(struct ncplane* n){ if(auto runtime = static_cast(ncplane_userptr(n))){ runtime->resize_pending.store(true, std::memory_order_release); @@ -131,6 +133,15 @@ static int player_cli_resize_cb(struct ncplane* n){ return ncplane_resize_marginalized(n); } +static void +attach_plane_runtime(ncplane* n){ + if(n == nullptr){ + return; + } + init_plane_runtime(n); + ncplane_set_resizecb(n, player_plane_resize_cb); +} + struct marshal { int framecount; bool quiet; @@ -288,7 +299,12 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, stdn->printf(0, NCAlign::Right, "%02" PRId64 ":%02" PRId64 ":%02" PRId64 ".%04" PRId64, h, m, s, remaining / 1000000); } + if(marsh->resize_restart_pending){ + destroy_subtitle_plane(); + return 2; + } if(!nc.render()){ + destroy_subtitle_plane(); return -1; } if(pop_async_resize()){ @@ -760,7 +776,7 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, vopts.x = NCALIGN_CENTER; vopts.n = n; } - init_plane_runtime(vopts.n); + attach_plane_runtime(vopts.n); ncplane_erase(n); logdebug("[resize] planes ready main=%ux%u", ncplane_dim_x(vopts.n), ncplane_dim_y(vopts.n)); return true; @@ -863,6 +879,10 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, delta = pending_seek_value - current; pending_seek_absolute = false; } + if(marsh.resize_restart_pending){ + logdebug("[resize] clearing pending restart"); + marsh.resize_restart_pending = false; + } logdebug("[seek] request delta=%f (absolute=%d)", delta, was_absolute ? 1 : 0); if(ncvisual_seek(*ncv, delta) == 0){ pending_request = PlaybackRequest::None; From 45b3fa701e38bf2c35a5748db7d1e12a7bbebf23 Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sun, 30 Nov 2025 18:00:54 -0600 Subject: [PATCH 21/23] Add volume control --- src/player/play.cpp | 178 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 139 insertions(+), 39 deletions(-) diff --git a/src/player/play.cpp b/src/player/play.cpp index b31f280b8..5564a8820 100644 --- a/src/player/play.cpp +++ b/src/player/play.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -48,7 +49,7 @@ static void usage(std::ostream& os, const char* name, int exitcode) __attribute__ ((noreturn)); void usage(std::ostream& o, const char* name, int exitcode){ - o << "usage: " << name << " [ -h ] [ -q ] [ -m margins ] [ -l loglevel ] [ -d mult ] [ -s scaletype ] [ -k ] [ -L ] [ -t seconds ] [ -n ] [ -a color ] files" << '\n'; + o << "usage: " << name << " [ -h ] [ -q ] [ -m margins ] [ -l loglevel ] [ -d mult ] [ -s scaletype ] [ -k ] [ -L ] [ -t seconds ] [ -n ] [ -a color ] [ -v volume ] files" << '\n'; o << " -h: display help and exit with success\n"; o << " -V: print program name and version\n"; o << " -q: be quiet (no frame/timing information along top of screen)\n"; @@ -60,6 +61,7 @@ void usage(std::ostream& o, const char* name, int exitcode){ o << " -b blitter: one of 'ascii', 'half', 'quad', 'sex', 'oct', 'braille', or 'pixel'\n"; o << " -m margins: margin, or 4 comma-separated margins\n"; o << " -a color: replace color with a transparent channel\n"; + o << " -v volume: initial audio volume percentage (0-100, default 100)\n"; o << " -n: force non-interpolative scaling\n"; o << " -d mult: non-negative floating point scale for frame time" << std::endl; exit(exitcode); @@ -156,10 +158,14 @@ struct marshal { double seek_delta; bool seek_absolute; bool resize_restart_pending; + std::atomic* volume_percent; + std::chrono::steady_clock::time_point volume_overlay_until; }; static constexpr double kNcplayerSeekSeconds = 5.0; static constexpr double kNcplayerSeekMinutes = 2.0 * 60.0; +static constexpr int kNcplayerVolumeOverlayMs = 1000; +static constexpr int kNcplayerVolumeStep = 5; // frame count is in the curry. original time is kept in n's userptr. auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, const struct timespec* abstime, void* vmarshal) -> int { @@ -210,6 +216,7 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, if(marsh->blitter == NCBLIT_DEFAULT){ marsh->blitter = ncvisual_media_defblitter(nc, vopts->scaling); } + const int current_volume = marsh->volume_percent ? marsh->volume_percent->load(std::memory_order_relaxed) : 100; if(!marsh->quiet){ // FIXME put this on its own plane if we're going to be erase()ing it stdn->erase(); @@ -221,6 +228,17 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, stdn->printf(0, NCAlign::Left, "frame %06d (%s)", marsh->framecount, notcurses_str_blitter(vopts->blitter)); } + bool show_volume_overlay = false; + if(marsh->volume_percent && marsh->volume_overlay_until.time_since_epoch().count() > 0){ + auto now_overlay = std::chrono::steady_clock::now(); + show_volume_overlay = now_overlay < marsh->volume_overlay_until; + } + if(show_volume_overlay){ + unsigned rows, cols; + ncplane_dim_yx(*stdn, &rows, &cols); + int center_row = rows > 0 ? static_cast(rows / 2) : 0; + stdn->printf(center_row, NCAlign::Center, "vol %3d%%", current_volume); + } } const uint64_t target_ns = timespec_to_ns(abstime); @@ -288,6 +306,19 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, logdebug("[resize] async restart due to %s", src ? src : "unknown"); return 2; }; + auto adjust_volume = [&](int delta){ + if(marsh->volume_percent == nullptr){ + return; + } + int original = marsh->volume_percent->load(std::memory_order_relaxed); + int updated = std::clamp(original + delta, 0, 100); + if(updated != original){ + marsh->volume_percent->store(updated, std::memory_order_relaxed); + logdebug("[audio] volume set to %d%%", updated); + } + marsh->volume_overlay_until = std::chrono::steady_clock::now() + + std::chrono::milliseconds(kNcplayerVolumeOverlayMs); + }; int64_t remaining = display_ns; const int64_t h = remaining / (60 * 60 * NANOSECS_IN_SEC); remaining -= h * (60 * 60 * NANOSECS_IN_SEC); @@ -360,6 +391,7 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, }while(ni.id != 'q' && (ni.evtype == EvType::Release || ni.id != ' ')); } // if we just hit a non-space character to unpause, ignore it + const bool shift_pressed = ncinput_shift_p(&ni); if(keyp == NCKey::Resize){ if(marsh->resize_restart_pending){ logdebug("[resize] restart already pending, skipping key resize"); @@ -390,26 +422,20 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, marsh->show_fps = !marsh->show_fps; continue; }else if(keyp == NCKey::Up){ - marsh->request = PlaybackRequest::Seek; - marsh->seek_delta = -kNcplayerSeekMinutes; - marsh->seek_absolute = false; - destroy_subtitle_plane(); - return 2; + adjust_volume(kNcplayerVolumeStep); + continue; }else if(keyp == NCKey::Down){ - marsh->request = PlaybackRequest::Seek; - marsh->seek_delta = kNcplayerSeekMinutes; - marsh->seek_absolute = false; - destroy_subtitle_plane(); - return 2; + adjust_volume(-kNcplayerVolumeStep); + continue; }else if(keyp == NCKey::Right){ marsh->request = PlaybackRequest::Seek; - marsh->seek_delta = kNcplayerSeekSeconds; + marsh->seek_delta = shift_pressed ? kNcplayerSeekMinutes : kNcplayerSeekSeconds; marsh->seek_absolute = false; destroy_subtitle_plane(); return 2; }else if(keyp == NCKey::Left){ marsh->request = PlaybackRequest::Seek; - marsh->seek_delta = -kNcplayerSeekSeconds; + marsh->seek_delta = shift_pressed ? -kNcplayerSeekMinutes : -kNcplayerSeekSeconds; marsh->seek_absolute = false; destroy_subtitle_plane(); return 2; @@ -428,13 +454,17 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, auto handle_opts(int argc, char** argv, notcurses_options& opts, bool* quiet, float* timescale, ncscale_e* scalemode, ncblitter_e* blitter, float* displaytime, bool* loop, bool* noninterp, - uint32_t* transcolor, bool* climode) + uint32_t* transcolor, bool* climode, int* volume_percent) -> int { *timescale = 1.0; *scalemode = NCSCALE_STRETCH; *displaytime = -1; + if(volume_percent){ + *volume_percent = std::clamp(*volume_percent, 0, 100); + } + bool volume_seen = false; int c; - while((c = getopt(argc, argv, "Vhql:d:s:b:t:m:kLa:n")) != -1){ + while((c = getopt(argc, argv, "Vhql:d:s:b:t:m:kLa:nv:")) != -1){ switch(c){ case 'h': usage(std::cout, argv[0], EXIT_SUCCESS); @@ -549,6 +579,25 @@ auto handle_opts(int argc, char** argv, notcurses_options& opts, bool* quiet, } opts.loglevel = static_cast(ll); break; + }case 'v':{ + if(volume_percent == nullptr){ + break; + } + if(volume_seen){ + std::cerr << "Provided -v twice!" << std::endl; + usage(std::cerr, argv[0], EXIT_FAILURE); + } + std::stringstream ss; + ss << optarg; + int vol; + ss >> vol; + if(ss.fail() || vol < 0 || vol > 100){ + std::cerr << "Invalid volume [" << optarg << "] (wanted [0..100])\n"; + usage(std::cerr, argv[0], EXIT_FAILURE); + } + *volume_percent = vol; + volume_seen = true; + break; }default: usage(std::cerr, argv[0], EXIT_FAILURE); break; @@ -570,6 +619,7 @@ struct audio_thread_data { audio_output* ao; std::atomic* running; std::mutex* mutex; + std::atomic* volume_percent; }; // Audio thread function - processes decoded frames (video decoder reads packets) @@ -577,6 +627,32 @@ static void audio_thread_func(audio_thread_data* data) { ncvisual* ncv = data->ncv; audio_output* ao = data->ao; std::atomic* running = data->running; + std::atomic* volume_percent = data->volume_percent; + bool ao_paused = false; + + auto apply_volume = [&](uint8_t* buffer, int byte_count){ + if(buffer == nullptr || byte_count <= 0 || volume_percent == nullptr){ + return; + } + int vol = volume_percent->load(std::memory_order_relaxed); + if(vol >= 100){ + return; + } + int16_t* samples = reinterpret_cast(buffer); + size_t sample_count = static_cast(byte_count) / sizeof(int16_t); + if(sample_count == 0){ + std::memset(buffer, 0, byte_count); + return; + } + if(vol <= 0){ + std::fill(samples, samples + sample_count, 0); + return; + } + for(size_t idx = 0 ; idx < sample_count ; ++idx){ + int scaled = (samples[idx] * vol) / 100; + samples[idx] = static_cast(std::clamp(scaled, -32768, 32767)); + } + }; if(!ffmpeg_has_audio(ncv) || !ao){ return; @@ -606,31 +682,49 @@ static void audio_thread_func(audio_thread_data* data) { int frames_since_log = 0; while(*running){ ffmpeg_audio_request_packets(ncv); + int current_volume = volume_percent ? volume_percent->load(std::memory_order_relaxed) : 100; + bool muted = current_volume <= 0; + if(muted && !ao_paused){ + audio_output_pause(ao); + audio_output_flush(ao); + ao_paused = true; + }else if(!muted && ao_paused){ + audio_output_resume(ao); + ao_paused = false; + } // Get decoded audio frame (packets are read by video decoder) // IMPORTANT: avcodec_receive_frame can only return each frame once // So we must process the frame immediately and not call get_decoded_audio_frame again // until we've finished processing int samples = ffmpeg_get_decoded_audio_frame(ncv); if(samples > 0){ - do{ - consecutive_eagain = 0; - uint8_t* out_data = nullptr; - int out_samples = 0; - int bytes = ffmpeg_resample_audio(ncv, &out_data, &out_samples); - if(bytes > 0 && out_data){ - audio_output_write(ao, out_data, bytes); - frame_count++; - frames_since_log++; - auto now = std::chrono::steady_clock::now(); - auto elapsed = std::chrono::duration_cast(now - last_log).count(); - if(elapsed >= 1000){ - frames_since_log = 0; - last_log = now; + if(muted){ + do{ + consecutive_eagain = 0; + samples = ffmpeg_get_decoded_audio_frame(ncv); + }while(samples > 0); + }else{ + do{ + consecutive_eagain = 0; + uint8_t* out_data = nullptr; + int out_samples = 0; + int bytes = ffmpeg_resample_audio(ncv, &out_data, &out_samples); + if(bytes > 0 && out_data){ + apply_volume(out_data, bytes); + audio_output_write(ao, out_data, bytes); + frame_count++; + frames_since_log++; + auto now = std::chrono::steady_clock::now(); + auto elapsed = std::chrono::duration_cast(now - last_log).count(); + if(elapsed >= 1000){ + frames_since_log = 0; + last_log = now; + } + free(out_data); } - free(out_data); - } - samples = ffmpeg_get_decoded_audio_frame(ncv); - }while(samples > 0 && audio_output_needs_data(ao)); + samples = ffmpeg_get_decoded_audio_frame(ncv); + }while(samples > 0 && audio_output_needs_data(ao)); + } if(!audio_output_needs_data(ao)){ usleep(1000); } @@ -660,6 +754,7 @@ static void audio_thread_func(audio_thread_data* data) { int out_samples = 0; int bytes = ffmpeg_resample_audio(ncv, &out_data, &out_samples); if(bytes > 0 && out_data){ + apply_volume(out_data, bytes); audio_output_write(ao, out_data, bytes); free(out_data); flush_count++; @@ -675,7 +770,7 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, bool quiet, bool loop, double timescale, double displaytime, bool noninterp, uint32_t transcolor, - bool climode){ + bool climode, int initial_volume){ unsigned dimy, dimx; std::unique_ptr stdn(nc.get_stdplane(&dimy, &dimx)); if(climode){ @@ -704,6 +799,8 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, std::atomic audio_running(false); std::mutex audio_mutex; audio_thread_data* audio_data = nullptr; + const int clamped_volume = std::clamp(initial_volume, 0, 100); + std::atomic volume_percent(clamped_volume); for(auto i = 0 ; i < argc ; ++i){ std::unique_ptr ncv; @@ -813,7 +910,7 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, if(ao){ logdebug("[audio] starting audio thread"); audio_running = true; - audio_data = new audio_thread_data{*ncv, ao, &audio_running, &audio_mutex}; + audio_data = new audio_thread_data{*ncv, ao, &audio_running, &audio_mutex, &volume_percent}; audio_thread = new std::thread(audio_thread_func, audio_data); audio_output_start(ao); }else{ @@ -835,6 +932,8 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, .seek_delta = 0.0, .seek_absolute = false, .resize_restart_pending = false, + .volume_percent = &volume_percent, + .volume_overlay_until = std::chrono::steady_clock::time_point::min(), }; logdebug("[stream] starting ncvisual_stream iteration with %ux%u plane", vopts.n ? ncplane_dim_x(vopts.n) : 0, vopts.n ? ncplane_dim_y(vopts.n) : 0); r = ncv->stream(&vopts, timescale, perframe, &marsh); @@ -1027,7 +1126,7 @@ int rendered_mode_player(int argc, char** argv, ncscale_e scalemode, bool quiet, bool loop, double timescale, double displaytime, bool noninterp, uint32_t transcolor, - bool climode){ + bool climode, int initial_volume){ // no -k, we're using full rendered mode (and the alternate screen). ncopts.flags |= NCOPTION_INHIBIT_SETLOCALE; if(quiet){ @@ -1043,7 +1142,7 @@ int rendered_mode_player(int argc, char** argv, ncscale_e scalemode, } r = rendered_mode_player_inner(nc, argc, argv, scalemode, blitter, quiet, loop, timescale, displaytime, - noninterp, transcolor, climode); + noninterp, transcolor, climode, initial_volume); if(!nc.stop()){ return -1; } @@ -1071,14 +1170,15 @@ auto main(int argc, char** argv) -> int { bool loop = false; bool noninterp = false; bool climode = false; + int initial_volume = 100; auto nonopt = handle_opts(argc, argv, ncopts, &quiet, ×cale, &scalemode, &blitter, &displaytime, &loop, &noninterp, &transcolor, - &climode); + &climode, &initial_volume); // if -k was provided, we use CLI mode rather than simply not using the // alternate screen, so that output is inline with the shell. if(rendered_mode_player(argc - nonopt, argv + nonopt, scalemode, blitter, ncopts, quiet, loop, timescale, displaytime, noninterp, - transcolor, climode)){ + transcolor, climode, initial_volume)){ return EXIT_FAILURE; } return EXIT_SUCCESS; From db487acbc56855bc9734ae865c66e208e470c797 Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sun, 30 Nov 2025 21:46:55 -0600 Subject: [PATCH 22/23] Don't even decode video streams when muted --- src/media/ffmpeg.c | 42 +++++++++++++++++++++++++ src/player/play.cpp | 77 +++++++++++++++++++++++++-------------------- 2 files changed, 85 insertions(+), 34 deletions(-) diff --git a/src/media/ffmpeg.c b/src/media/ffmpeg.c index 38401de75..d68c07d59 100644 --- a/src/media/ffmpeg.c +++ b/src/media/ffmpeg.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include "lib/visual-details.h" @@ -169,6 +170,7 @@ typedef struct ncvisual_details { int audio_stream_index; // audio stream index, can be < 0 if no audio bool packet_outstanding; bool audio_packet_outstanding; // whether we have an audio packet waiting to be decoded + atomic_bool audio_enabled; audio_packet_queue pending_audio_packets; // audio packets waiting to be sent int64_t last_audio_frame_pts; // PTS of last processed audio frame (to prevent reprocessing) int64_t last_video_frame_pts; // PTS of last displayed video frame @@ -702,6 +704,11 @@ ffmpeg_decode(ncvisual* n){ continue; } + if(!atomic_load_explicit(&n->details->audio_enabled, memory_order_relaxed)){ + av_packet_unref(n->details->packet); + continue; + } + static int audio_packet_count = 0; audio_packet_count++; @@ -792,6 +799,7 @@ ffmpeg_details_init(void){ deets->decoded_frames = 0; deets->audio_out_channels = 0; // Will be set when resampler is initialized audio_queue_init(&deets->pending_audio_packets); + atomic_init(&deets->audio_enabled, true); if(pthread_mutex_init(&deets->packet_mutex, NULL) != 0){ free(deets); return NULL; @@ -1390,10 +1398,44 @@ ffmpeg_audio_request_packets(ncvisual* ncv){ return; } pthread_mutex_lock(&ncv->details->audio_packet_mutex); + if(!atomic_load_explicit(&ncv->details->audio_enabled, memory_order_relaxed)){ + pthread_mutex_unlock(&ncv->details->audio_packet_mutex); + return; + } ffmpeg_drain_pending_audio_locked(ncv->details); pthread_mutex_unlock(&ncv->details->audio_packet_mutex); } +API void +ffmpeg_set_audio_enabled(ncvisual* ncv, bool enabled){ + if(!ncv || !ncv->details || ncv->details->audio_stream_index < 0){ + return; + } + ncvisual_details* deets = ncv->details; + bool current = atomic_load_explicit(&deets->audio_enabled, memory_order_relaxed); + if(current == enabled){ + return; + } + pthread_mutex_lock(&deets->audio_packet_mutex); + atomic_store_explicit(&deets->audio_enabled, enabled, memory_order_relaxed); + if(!enabled){ + audio_queue_clear(&deets->pending_audio_packets); + deets->audio_packet_outstanding = false; + if(deets->audiocodecctx){ + avcodec_flush_buffers(deets->audiocodecctx); + } + } + pthread_mutex_unlock(&deets->audio_packet_mutex); +} + +API bool +ffmpeg_is_audio_enabled(ncvisual* ncv){ + if(!ncv || !ncv->details){ + return false; + } + return atomic_load_explicit(&ncv->details->audio_enabled, memory_order_relaxed); +} + // Check if the visual has an audio stream API bool ffmpeg_has_audio(ncvisual* ncv){ diff --git a/src/player/play.cpp b/src/player/play.cpp index 5564a8820..3d52dffc0 100644 --- a/src/player/play.cpp +++ b/src/player/play.cpp @@ -42,6 +42,8 @@ int ffmpeg_init_audio_resampler(ncvisual* ncv, int out_sample_rate, int out_chan int ffmpeg_get_audio_sample_rate(ncvisual* ncv); int ffmpeg_get_audio_channels(ncvisual* ncv); void ffmpeg_audio_request_packets(ncvisual* ncv); +void ffmpeg_set_audio_enabled(ncvisual* ncv, bool enabled); +bool ffmpeg_is_audio_enabled(ncvisual* ncv); double ffmpeg_get_video_position_seconds(const ncvisual* ncv); } @@ -316,6 +318,11 @@ auto perframe(struct ncvisual* ncv, struct ncvisual_options* vopts, marsh->volume_percent->store(updated, std::memory_order_relaxed); logdebug("[audio] volume set to %d%%", updated); } + bool want_audio = marsh->volume_percent->load(std::memory_order_relaxed) > 0; + bool audio_enabled = ffmpeg_is_audio_enabled(ncv); + if(want_audio != audio_enabled){ + ffmpeg_set_audio_enabled(ncv, want_audio); + } marsh->volume_overlay_until = std::chrono::steady_clock::now() + std::chrono::milliseconds(kNcplayerVolumeOverlayMs); }; @@ -681,50 +688,51 @@ static void audio_thread_func(audio_thread_data* data) { auto last_log = std::chrono::steady_clock::now(); int frames_since_log = 0; while(*running){ - ffmpeg_audio_request_packets(ncv); - int current_volume = volume_percent ? volume_percent->load(std::memory_order_relaxed) : 100; - bool muted = current_volume <= 0; - if(muted && !ao_paused){ - audio_output_pause(ao); - audio_output_flush(ao); - ao_paused = true; - }else if(!muted && ao_paused){ + bool audio_enabled = ffmpeg_is_audio_enabled(ncv); + if(!audio_enabled){ + if(!ao_paused){ + audio_output_pause(ao); + audio_output_flush(ao); + ao_paused = true; + } + usleep(10000); + continue; + }else if(ao_paused){ audio_output_resume(ao); ao_paused = false; } + ffmpeg_audio_request_packets(ncv); + int current_volume = volume_percent ? volume_percent->load(std::memory_order_relaxed) : 100; + if(current_volume <= 0){ + ffmpeg_set_audio_enabled(ncv, false); + continue; + } // Get decoded audio frame (packets are read by video decoder) // IMPORTANT: avcodec_receive_frame can only return each frame once // So we must process the frame immediately and not call get_decoded_audio_frame again // until we've finished processing int samples = ffmpeg_get_decoded_audio_frame(ncv); if(samples > 0){ - if(muted){ - do{ - consecutive_eagain = 0; - samples = ffmpeg_get_decoded_audio_frame(ncv); - }while(samples > 0); - }else{ - do{ - consecutive_eagain = 0; - uint8_t* out_data = nullptr; - int out_samples = 0; - int bytes = ffmpeg_resample_audio(ncv, &out_data, &out_samples); - if(bytes > 0 && out_data){ - apply_volume(out_data, bytes); - audio_output_write(ao, out_data, bytes); - frame_count++; - frames_since_log++; - auto now = std::chrono::steady_clock::now(); - auto elapsed = std::chrono::duration_cast(now - last_log).count(); - if(elapsed >= 1000){ - frames_since_log = 0; - last_log = now; - } - free(out_data); + do{ + consecutive_eagain = 0; + uint8_t* out_data = nullptr; + int out_samples = 0; + int bytes = ffmpeg_resample_audio(ncv, &out_data, &out_samples); + if(bytes > 0 && out_data){ + apply_volume(out_data, bytes); + audio_output_write(ao, out_data, bytes); + frame_count++; + frames_since_log++; + auto now = std::chrono::steady_clock::now(); + auto elapsed = std::chrono::duration_cast(now - last_log).count(); + if(elapsed >= 1000){ + frames_since_log = 0; + last_log = now; } - samples = ffmpeg_get_decoded_audio_frame(ncv); - }while(samples > 0 && audio_output_needs_data(ao)); - } + free(out_data); + } + samples = ffmpeg_get_decoded_audio_frame(ncv); + }while(samples > 0 && audio_output_needs_data(ao)); if(!audio_output_needs_data(ao)){ usleep(1000); } @@ -805,6 +813,7 @@ int rendered_mode_player_inner(NotCurses& nc, int argc, char** argv, for(auto i = 0 ; i < argc ; ++i){ std::unique_ptr ncv; ncv = std::make_unique(argv[i]); + ffmpeg_set_audio_enabled(*ncv, clamped_volume > 0); struct ncvisual_options vopts{}; int r; if(noninterp){ From 18f75c7eaee39a93a71b4a15d665d0a8f4cf04cb Mon Sep 17 00:00:00 2001 From: R E Broadley Date: Sun, 30 Nov 2025 22:01:07 -0600 Subject: [PATCH 23/23] subtitle formatting --- src/media/ffmpeg.c | 156 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 143 insertions(+), 13 deletions(-) diff --git a/src/media/ffmpeg.c b/src/media/ffmpeg.c index d68c07d59..1479d9c27 100644 --- a/src/media/ffmpeg.c +++ b/src/media/ffmpeg.c @@ -318,6 +318,86 @@ print_frame_summary(const AVCodecContext* cctx, const AVFrame* f){ f->quality); }*/ +static void +ass_apply_tag_controls(const char* tag, size_t len, bool* italic){ + const char* cursor = tag; + const char* end = tag + len; + while(cursor < end){ + if(*cursor == '\\'){ + ++cursor; + } + if(cursor >= end){ + break; + } + if((*cursor == 'i' || *cursor == 'I') && cursor + 1 < end){ + char state = cursor[1]; + if(state == '0'){ + *italic = false; + }else if(state == '1'){ + *italic = true; + } + cursor += 2; + continue; + } + while(cursor < end && *cursor != '\\'){ + ++cursor; + } + } +} + +static bool +ass_extract_text_and_styles(char* text, unsigned char** styles_out, size_t* len_out){ + size_t rawlen = strlen(text); + unsigned char* styles = malloc(rawlen ? rawlen : 1); + if(!styles){ + return false; + } + size_t src = 0; + size_t dst = 0; + bool italic = false; + while(text[src]){ + if(text[src] == '\\'){ + char next = text[src + 1]; + if(next == 'N' || next == 'n'){ + text[dst] = '\n'; + styles[dst++] = italic; + src += 2; + continue; + }else if(next == 'h' || next == 'H'){ + text[dst] = ' '; + styles[dst++] = italic; + src += 2; + continue; + } + } + if(text[src] == '{'){ + char* closing = strchr(text + src, '}'); + if(closing){ + ass_apply_tag_controls(text + src + 1, (size_t)(closing - (text + src + 1)), &italic); + src = (closing - text) + 1; + continue; + } + } + if(text[src] == '\r'){ + ++src; + continue; + } + text[dst] = text[src]; + styles[dst] = italic; + ++dst; + ++src; + } + text[dst] = '\0'; + if(dst == 0){ + free(styles); + return false; + } + *len_out = dst; + unsigned char* resized = realloc(styles, dst); + *styles_out = resized ? resized : styles; + return true; +} + static struct ncplane* subtitle_plane_from_text(ncplane* parent, const char* text, bool* logged_flag){ if(parent == NULL || text == NULL){ @@ -350,11 +430,15 @@ subtitle_plane_from_text(ncplane* parent, const char* text, bool* logged_flag){ free(dup); return NULL; } + unsigned char* style_map = NULL; + size_t processed_len = 0; + if(!ass_extract_text_and_styles(trimmed, &style_map, &processed_len)){ + free(dup); + return NULL; + } + len = processed_len; int linecount = 1; for(size_t i = 0 ; i < len ; ++i){ - if(trimmed[i] == '\r'){ - trimmed[i] = '\n'; - } if(trimmed[i] == '\n'){ ++linecount; } @@ -362,15 +446,37 @@ subtitle_plane_from_text(ncplane* parent, const char* text, bool* logged_flag){ int parent_cols = ncplane_dim_x(parent); if(parent_cols <= 0){ + free(style_map); free(dup); return NULL; } int maxwidth = 0; char* walker = trimmed; + size_t offset = 0; + size_t* line_offsets = malloc(sizeof(*line_offsets) * linecount); + size_t* line_lengths = malloc(sizeof(*line_lengths) * linecount); + if(line_offsets == NULL || line_lengths == NULL){ + free(line_offsets); + free(line_lengths); + free(style_map); + free(dup); + return NULL; + } for(int line = 0 ; line < linecount ; ++line){ + line_offsets[line] = offset; char* next = strchr(walker, '\n'); if(next){ *next = '\0'; + line_lengths[line] = (size_t)(next - walker); + }else{ + line_lengths[line] = strlen(walker); + if(line != linecount - 1){ + free(line_offsets); + free(line_lengths); + free(style_map); + free(dup); + return NULL; + } } int w = ncstrwidth(walker, NULL, NULL); if(w > maxwidth){ @@ -379,11 +485,16 @@ subtitle_plane_from_text(ncplane* parent, const char* text, bool* logged_flag){ if(next){ *next = '\n'; walker = next + 1; + offset += line_lengths[line] + 1; }else{ + offset += line_lengths[line]; break; } } if(maxwidth <= 0){ + free(line_offsets); + free(line_lengths); + free(style_map); free(dup); return NULL; } @@ -402,6 +513,9 @@ subtitle_plane_from_text(ncplane* parent, const char* text, bool* logged_flag){ }; struct ncplane* n = ncplane_create(parent, &nopts); if(n == NULL){ + free(line_offsets); + free(line_lengths); + free(style_map); free(dup); return NULL; } @@ -421,20 +535,36 @@ subtitle_plane_from_text(ncplane* parent, const char* text, bool* logged_flag){ SUBLOG_DEBUG("rendering subtitle text (trimmed): \"%s\"", trimmed); *logged_flag = true; } - char* linewalker = trimmed; for(int line = 0 ; line < linecount ; ++line){ - char* next = strchr(linewalker, '\n'); - if(next){ - *next = '\0'; + size_t line_len = line_lengths[line]; + if(line_len == 0){ + continue; } - ncplane_putstr_yx(n, line, 0, linewalker); - if(next){ - *next = '\n'; - linewalker = next + 1; - }else{ - break; + size_t base = line_offsets[line]; + size_t cursor = 0; + int line_xpos = 0; + while(cursor < line_len){ + bool italic = style_map[base + cursor]; + size_t chunk_end = cursor + 1; + while(chunk_end < line_len && style_map[base + chunk_end] == italic){ + ++chunk_end; + } + char saved = trimmed[base + chunk_end]; + trimmed[base + chunk_end] = '\0'; + if(italic){ + ncplane_set_fg_rgb8(n, 0xff, 0xff, 0x00); + }else{ + ncplane_set_fg_rgb8(n, 0xff, 0xff, 0xff); + } + ncplane_putstr_yx(n, line, line_xpos, trimmed + base + cursor); + line_xpos += ncstrwidth(trimmed + base + cursor, NULL, NULL); + trimmed[base + chunk_end] = saved; + cursor = chunk_end; } } + free(line_offsets); + free(line_lengths); + free(style_map); free(dup); return n; }