diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index dc020db7e86..263b269bd43 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -15,6 +15,7 @@ set(ZM_BIN_SRC_FILES zm.cpp zm_db.cpp zm_decoder_thread.cpp + zm_second_stream_thread.cpp zm_group_permission.cpp zm_monitor_permission.cpp zm_logger.cpp diff --git a/src/zm_ffmpeg.cpp b/src/zm_ffmpeg.cpp index 20cc6b8b497..e2dca09ac59 100644 --- a/src/zm_ffmpeg.cpp +++ b/src/zm_ffmpeg.cpp @@ -415,6 +415,24 @@ void zm_dump_codecpar(const AVCodecParameters *par) { ); } +void zm_set_rtsp_transport_method(AVDictionary **opts, const std::string &method) { + int ret = 0; + if (method == "rtpMulti") { + ret = av_dict_set(opts, "rtsp_transport", "udp_multicast", 0); + } else if (method == "rtpRtsp") { + ret = av_dict_set(opts, "rtsp_transport", "tcp", 0); + } else if (method == "rtpRtspHttp") { + ret = av_dict_set(opts, "rtsp_transport", "http", 0); + } else if (method == "rtpUni") { + ret = av_dict_set(opts, "rtsp_transport", "udp", 0); + } else { + Warning("Unknown method (%s)", method.c_str()); + } + if (ret < 0) { + Warning("Could not set rtsp_transport method '%s'", method.c_str()); + } +} + void zm_dump_codec(const AVCodecContext *codec) { Debug(1, "Dumping codec_context codec_type %d %s codec_id %d %s tag %c%c%c%c width %d height %d timebase %d/%d format %s profile %d level %d " "gop_size %d has_b_frames %d max_b_frames %d me_cmp %d me_range %d qmin %d qmax %d bit_rate %" PRId64 " qcompress %f extradata:%d:%s", diff --git a/src/zm_ffmpeg.h b/src/zm_ffmpeg.h index 1879eea0492..f7d2eff2147 100644 --- a/src/zm_ffmpeg.h +++ b/src/zm_ffmpeg.h @@ -149,6 +149,12 @@ void zm_dump_stream_format(AVFormatContext *ic, int i, int index, int is_output) void zm_dump_codec(const AVCodecContext *codec); void zm_dump_codecpar(const AVCodecParameters *par); +// Map a monitor's Method setting (rtpMulti/rtpRtsp/rtpRtspHttp/rtpUni) to the +// ffmpeg rtsp demuxer's rtsp_transport option. Warns on an unknown method or +// a failure to set the option. Shared by the primary FfmpegCamera open path +// and the analysis-substream sidecar so both streams use the same transport. +void zm_set_rtsp_transport_method(AVDictionary **opts, const std::string &method); + #if LIBAVUTIL_VERSION_CHECK(57, 28, 100, 28, 0) #define zm_dump_frame(frame, text) Debug(1, "%s: format %d %s sample_rate %" PRIu32 " nb_samples %d" \ " layout %" PRIu64 " pts %" PRId64, \ diff --git a/src/zm_ffmpeg_camera.cpp b/src/zm_ffmpeg_camera.cpp index 31eb51bec3b..637eccc4614 100644 --- a/src/zm_ffmpeg_camera.cpp +++ b/src/zm_ffmpeg_camera.cpp @@ -127,6 +127,13 @@ FfmpegCamera::FfmpegCamera( FfmpegCamera::~FfmpegCamera() { Close(); + // mSecondFormatContext is a non-owning alias into mSecondInput's AVFormatContext + // (set in OpenFfmpeg from mSecondInput->get_format_context()). mSecondInput, a + // unique_ptr, closes and frees that context in its own destructor, + // so clear the alias here to stop the base Camera::~Camera() from freeing the same + // context a second time (double free -> crash in avformat_free_context/av_opt_free). + mSecondFormatContext = nullptr; + FFMPEGDeInit(); } @@ -477,21 +484,7 @@ int FfmpegCamera::OpenFfmpeg() { std::string protocol = mPath.substr(0, 4); protocol = StringToUpper(protocol); if ( protocol == "RTSP" ) { - const std::string method = Method(); - if ( method == "rtpMulti" ) { - ret = av_dict_set(&opts, "rtsp_transport", "udp_multicast", 0); - } else if ( method == "rtpRtsp" ) { - ret = av_dict_set(&opts, "rtsp_transport", "tcp", 0); - } else if ( method == "rtpRtspHttp" ) { - ret = av_dict_set(&opts, "rtsp_transport", "http", 0); - } else if ( method == "rtpUni" ) { - ret = av_dict_set(&opts, "rtsp_transport", "udp", 0); - } else { - Warning("Unknown method (%s)", method.c_str()); - } - if (ret < 0) { - Warning("Could not set rtsp_transport method '%s'", method.c_str()); - } + zm_set_rtsp_transport_method(&opts, Method()); } else if (protocol == "V4L2") { avdevice_register_all(); input_format = av_find_input_format("video4linux2"); diff --git a/src/zm_ffmpeg_input.cpp b/src/zm_ffmpeg_input.cpp index 51d6e7a37be..58fb7cdc8d8 100644 --- a/src/zm_ffmpeg_input.cpp +++ b/src/zm_ffmpeg_input.cpp @@ -38,11 +38,11 @@ int FFmpeg_Input::Open( return 1; } -int FFmpeg_Input::Open(const char *filepath) { +int FFmpeg_Input::Open(const char *filepath, AVDictionary **options) { int error; /** Open the input file to read from it. */ - error = avformat_open_input(&input_format_context, filepath, nullptr, nullptr); + error = avformat_open_input(&input_format_context, filepath, nullptr, options); if ( error < 0 ) { if (std::string(filepath).find("incomplete") != std::string::npos) { Warning("Could not open input file '%s' (error '%s')", @@ -97,6 +97,12 @@ int FFmpeg_Input::Open(const char *filepath) { std::listcodec_data = get_decoder_data(input_format_context->streams[i]->codecpar->codec_id, "auto"); for (auto it = codec_data.begin(); it != codec_data.end(); it ++) { const CodecData *chosen_codec_data = *it; +#if HAVE_LIBAVUTIL_HWCONTEXT_H && LIBAVCODEC_VERSION_CHECK(57, 107, 0, 107, 0) + if (no_hwaccel and (chosen_codec_data->hwdevice_type != AV_HWDEVICE_TYPE_NONE)) { + Debug(1, "Skipping hardware codec %s (software decoding forced)", chosen_codec_data->codec_name); + continue; + } +#endif Debug(1, "Found codec %s", chosen_codec_data->codec_name); streams[i].codec = avcodec_find_decoder_by_name(chosen_codec_data->codec_name); diff --git a/src/zm_ffmpeg_input.h b/src/zm_ffmpeg_input.h index c2091a5eda1..45b305c8779 100644 --- a/src/zm_ffmpeg_input.h +++ b/src/zm_ffmpeg_input.h @@ -16,7 +16,11 @@ class FFmpeg_Input { FFmpeg_Input(); ~FFmpeg_Input(); - int Open(const char *filename ); + int Open(const char *filename, AVDictionary **options = nullptr); + // Force software decoding (skip hardware decoders / hwaccel setup). Must be + // called before Open(). Used by callers that cannot handle hardware-format + // frames, e.g. the analysis substream sidecar. + void set_no_hwaccel(bool v) { no_hwaccel = v; } int Open( const AVStream *, const AVCodecContext *, @@ -55,6 +59,7 @@ class FFmpeg_Input { av_frame_ptr frame; int64_t last_seek_request; AVBufferRef *hw_device_ctx; + bool no_hwaccel = false; }; #endif diff --git a/src/zm_monitor.cpp b/src/zm_monitor.cpp index 5ae420cc020..cb62e24161d 100644 --- a/src/zm_monitor.cpp +++ b/src/zm_monitor.cpp @@ -30,6 +30,7 @@ #include "zm_remote_camera_http.h" #include "zm_remote_camera_nvsocket.h" #include "zm_remote_camera_rtsp.h" +#include "zm_secondary_sync.h" #include "zm_signal.h" #include "zm_time.h" #include "zm_uri.h" @@ -182,6 +183,7 @@ Monitor::Monitor() : analysing(ANALYSING_ALWAYS), recording(RECORDING_ALWAYS), decoding(DECODING_ALWAYS), + secondary_analysis(false), RTSP2Web_enabled(false), RTSP2Web_type(WEBRTC), Go2RTC_enabled(false), @@ -533,6 +535,14 @@ void Monitor::Load(MYSQL_ROW dbrow, bool load_zones=true, Purpose p = QUERY) { col++; width = (orientation==ROTATE_90||orientation==ROTATE_270) ? camera_height : camera_width; height = (orientation==ROTATE_90||orientation==ROTATE_270) ? camera_width : camera_height; + // Motion detection runs at full frame size by default; secondary-substream + // analysis lowers this to the substream's native size on its first frame. + // If that size was already discovered (and we are still in secondary mode), + // keep it across reload so we don't reset the analysis resolution - and force a + // redundant zone rebuild / reference-image reset - on the next substream frame. + bool keep_substream_res = (analysis_source == ANALYSIS_SECONDARY) && substream_width; + analysis_width = keep_substream_res ? substream_width : width; + analysis_height = keep_substream_res ? substream_height : height; deinterlacing = atoi(dbrow[col]); col++; deinterlacing_value = deinterlacing & 0xff; @@ -2218,7 +2228,10 @@ bool Monitor::Analyse() { } else { event->addNote(SIGNAL_CAUSE, "Reacquired"); } - if (shared_data->analysing != ANALYSING_NONE) { + // In secondary-analysis mode the reference image is seeded and blended + // from the substream at the analysis resolution, so don't reseed it here + // from the full-res primary (which is only decoded on demand anyway). + if ((shared_data->analysing != ANALYSING_NONE) && !secondary_analysis) { if (analysis_image == ANALYSISIMAGE_YCHANNEL) { Image *y_image = packet->get_y_image(); if (y_image) { @@ -2292,7 +2305,13 @@ bool Monitor::Analyse() { #endif if (packet->codec_type == AVMEDIA_TYPE_VIDEO) { /* try to stay behind the decoder. */ - if (decoding != DECODING_NONE) { + // In secondary-analysis mode the motion source is the substream sidecar + // image, which is decoded independently of the primary. The primary is + // only decoded on demand (for live viewers), so waiting for the primary + // packet to decode here would stall motion detection entirely whenever + // nobody is watching. Only gate on the primary decode when the primary + // is actually the analysis source. + if ((decoding != DECODING_NONE) && !secondary_analysis) { if (!packet->decoded) { // We no longer wait because we need to be checking the triggers and other inputs. // Also the logic is too hairy. capture process can delete the packet that we have here. @@ -2314,42 +2333,31 @@ bool Monitor::Analyse() { Event::StringSet zoneSet; - if (packet->image) { - // decoder may not have been able to provide an image + // Resolve the motion-detection source. In secondary-analysis mode + // this is the substream sidecar image at its native substream + // resolution (zones are rebuilt at that size), independent of whether + // the primary was decoded. do_score is false when there is no + // fresh/valid substream frame to score this round. + bool do_score = true; + Image *motion_image = getMotionSourceImage(packet, do_score); + + if (motion_image) { + // decoder / sidecar may not have been able to provide an image if (!ref_image.Buffer()) { Debug(1, "Assigning instead of Detecting"); - - if (analysis_image == ANALYSISIMAGE_YCHANNEL) { - // If not decoding, y_image can be null - Image *y_image = packet->get_y_image(); - if (y_image) ref_image.Assign(*y_image); - } else if (packet->image) { - ref_image.Assign(*(packet->image)); - } else { - Debug(1, "No image to ref yet"); - } - WriteAlarmImage(*(packet->image)); + ref_image.Assign(*motion_image); + if (packet->image) WriteAlarmImage(*(packet->image)); } else { // didn't assign, do motion detection maybe and blending definitely - if (!(analysis_image_count % (motion_frame_skip+1))) { + if (do_score && !(analysis_image_count % (motion_frame_skip+1))) { motion_score = 0; - // Get new score. - if (analysis_image == ANALYSISIMAGE_YCHANNEL) { - Image *y_image = packet->get_y_image(); - Debug(1, "Detecting motion on image %d, y_image %p", packet->image_index, y_image); - if (y_image) { - motion_score += DetectMotion(*y_image, zoneSet); - } else { - // y_image unavailable (e.g., LocalCamera without in_frame) - skip motion detection - Debug(1, "y_image unavailable, skipping motion detection"); - } - } else { - Debug(1, "Detecting motion on image %d, image %p", packet->image_index, packet->image); - motion_score += DetectMotion(*(packet->image), zoneSet); - } + Debug(1, "Detecting motion on image %d, motion_image %p", packet->image_index, motion_image); + motion_score += DetectMotion(*motion_image, zoneSet); - // Instead of showing a greyscale image, let's use the full colour - if (!packet->analysis_image) + // Instead of showing a greyscale image, let's use the full colour. + // In secondary mode the primary may not be decoded, so packet->image + // can be null; the analysis image is then simply unavailable. + if (packet->image && !packet->analysis_image) packet->analysis_image = new Image(*(packet->image)); // lets construct alarm cause. It will contain cause + names of zones alarmed @@ -2361,7 +2369,12 @@ bool Monitor::Analyse() { if (zone.Alarmed()) { if (!packet->alarm_cause.empty()) packet->alarm_cause += ","; packet->alarm_cause += zone.Label(); - if (zone.AlarmImage()) + // The zone alarm image is at the analysis resolution, which + // differs from the full-res analysis_image when detecting on + // the substream; only overlay when the sizes match. + if (zone.AlarmImage() && packet->analysis_image + && zone.AlarmImage()->Width() == packet->analysis_image->Width() + && zone.AlarmImage()->Height() == packet->analysis_image->Height()) packet->analysis_image->Overlay(*(zone.AlarmImage())); } Debug(4, "Setting score for zone %d to %d", zone_index, zone.Score()); @@ -2386,35 +2399,21 @@ bool Monitor::Analyse() { //score += last_motion_score; } - if (hasAnalysisViewers()) { + if (hasAnalysisViewers() && (packet->analysis_image || packet->image)) { // These extra copies are expensive, so only do it if we have viewers. WriteAlarmImage(*(packet->analysis_image ? packet->analysis_image : packet->image)); } - if (analysis_image == ANALYSISIMAGE_YCHANNEL) { - Debug(1, "Blending from y-channel"); - Image *y_image = packet->get_y_image(); - if (y_image) { - ref_image.Blend(*y_image, ( state==ALARM ? alarm_ref_blend_perc : ref_blend_perc )); - } else { - Debug(1, "y_image unavailable, skipping blend"); - } - } else if (packet->image) { - Debug(1, "Blending full colour image because analysis_image = %d, in_frame=%p and format %d != %d, %d", - analysis_image, packet->in_frame.get(), - (packet->in_frame ? packet->in_frame->format : -1), - AV_PIX_FMT_YUV420P, - AV_PIX_FMT_YUVJ420P - ); - ref_image.Blend(*(packet->image), ( state==ALARM ? alarm_ref_blend_perc : ref_blend_perc )); - Debug(1, "Done Blending"); - } else { - Debug(1, "Not able to blend"); + // Only blend when we actually scored a fresh frame; re-blending a + // stale substream frame every primary packet would just churn. + if (do_score) { + Debug(1, "Blending analysis image"); + ref_image.Blend(*motion_image, ( state==ALARM ? alarm_ref_blend_perc : ref_blend_perc )); } } // end if had ref_image_buffer or not } else { - Debug(1, "no image so skipping motion detection"); - } // end if has image + Debug(1, "no motion image so skipping motion detection"); + } // end if has motion image } else { Debug(1, "Not analysing %d", shared_data->analysing); } // end if active and doing motion detection @@ -2864,7 +2863,10 @@ int Monitor::Capture() { std::shared_ptr packet = std::make_shared(); packet->image_index = shared_data->image_count; + // Stamp system and steady clocks back-to-back: the steady stamp pairs with + // the substream sidecar's steady frame times for NTP-immune wallclock sync. packet->timestamp = std::chrono::system_clock::now(); + packet->timestamp_steady = std::chrono::steady_clock::now(); shared_data->heartbeat_time = std::chrono::system_clock::to_time_t(packet->timestamp); int captureResult = camera->Capture(packet); Debug(4, "Back from capture result=%d image count %d timestamp %" PRId64, captureResult, shared_data->image_count, @@ -3299,6 +3301,14 @@ bool Monitor::Decode() { // between keyframe decodes. if (packet->codec_type == AVMEDIA_TYPE_VIDEO) { shared_data->last_write_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); + // In secondary-analysis mode the primary is only decoded on demand (for + // live viewers), so the normal decoding_image_count bump in the decode -> + // WriteShmFrame path never runs while nobody is watching. Without this, + // Ready() (decoding_image_count > ready_count) would stay false and motion + // detection on the substream would never start until a viewer bootstrapped + // the decoder - and would then latch on permanently. Advance the warmup + // counter here so substream analysis runs continuously, viewer or not. + if (secondary_analysis) decoding_image_count++; } } @@ -3644,6 +3654,110 @@ void Monitor::closeEvent() { if (shared_data) video_store_data->recording = {}; } // end bool Monitor::closeEvent() +// How old the newest substream frame may be before we treat it as "no motion +// data" (substream dropped): don't score, don't force a state change. +static constexpr Seconds kSecondaryStaleThreshold = Seconds(10); + +Image *Monitor::getMotionSourceImage(const std::shared_ptr &packet, bool &do_score) { + if (secondary_analysis and second_stream) { + uint64_t sequence = 0; + TimePoint frame_steady = TimePoint::min(); + // Peek at the frame metadata first (cheap, no pixel copy) so we only pay for + // the mailbox copy + upscale when the frame will actually be used. + if (!second_stream->PeekLatest(sequence, frame_steady)) { + // Sidecar has not produced a frame yet. + do_score = false; + return nullptr; + } + + bool fresh = (sequence != last_secondary_sequence); + // Wallclock sync on the steady clock (immune to NTP steps): the substream + // is considered stalled when this packet was captured more than the + // threshold AFTER the newest substream frame, i.e. the sidecar has stopped + // producing. The test is one-sided on purpose: a frame newer than the + // packet just means the analysis thread lags capture (packetqueue backlog) + // while the substream is healthy, and analysis pairs with the freshest + // frame as it always has. + bool stale = SecondaryFrameStalled(packet->timestamp_steady, frame_steady, + kSecondaryStaleThreshold); + if (stale) { + Debug(1, "Monitor %d: substream stalled: packet was captured %.1fs after the newest substream frame, no motion data", + id, FPSeconds(packet->timestamp_steady - frame_steady).count()); + } + + // Only pay for the copy + upscale when the result will actually be used: to + // seed the reference image, or to score a fresh, non-stale frame. Re-scoring + // a frame that has not advanced would just burn CPU at the primary rate. + const bool need_image = (!ref_image.Buffer()) or (fresh and !stale); + if (!need_image) { + do_score = false; + return nullptr; + } + + if (!second_stream->GetLatestImage(secondary_image_native, sequence, frame_steady)) { + do_score = false; + return nullptr; + } + // The mailbox may have advanced between PeekLatest and GetLatestImage, so + // recompute the verdict from the metadata GetLatestImage returned: the + // scored image and the fresh/stale decision must refer to the same frame. + // (A newer frame can only decrease the skew, so stale cannot newly appear.) + fresh = (sequence != last_secondary_sequence); + stale = SecondaryFrameStalled(packet->timestamp_steady, frame_steady, + kSecondaryStaleThreshold); + last_secondary_sequence = sequence; + do_score = fresh and !stale; + + // Run motion detection at the substream's NATIVE resolution rather than + // upscaling it to the full camera size. Zones are percentage-based, so + // rebuilding them at the substream size scales their geometry and pixel + // thresholds automatically - this keeps the per-frame Delta/Blend/zone work + // proportional to the (small) substream, instead of paying full-res cost. + // The substream size is only known once it decodes a frame, so discover it + // here and, on change, rebuild the zones and drop the reference image so it + // reseeds at the new resolution. + const unsigned int native_w = secondary_image_native.Width(); + const unsigned int native_h = secondary_image_native.Height(); + if (native_w and native_h and + ((native_w != analysis_width) or (native_h != analysis_height))) { + const unsigned int prev_w = analysis_width; + const unsigned int prev_h = analysis_height; + // Zone::Load builds zones at AnalysisWidth()/Height(), so switch the analysis + // resolution BEFORE loading. If the load comes back empty while we still have + // usable zones (e.g. a transient DB error), revert and keep detecting at the + // current resolution, retrying next frame rather than dropping every zone - + // which, since the switch would not fire again, would silently disable motion + // detection. A monitor that genuinely has no zones has zones.empty() already + // and falls through to switch. + analysis_width = native_w; + analysis_height = native_h; + std::vector new_zones = Zone::Load(shared_from_this()); + if (new_zones.empty() and !zones.empty()) { + analysis_width = prev_w; + analysis_height = prev_h; + Warning("Monitor %d: could not load zones at substream resolution %ux%u; retrying", + id, native_w, native_h); + do_score = false; + return nullptr; + } + Info("Monitor %d: motion detection resolution set to substream native %ux%u (was %ux%u); rebuilding zones", + id, native_w, native_h, prev_w, prev_h); + substream_width = native_w; + substream_height = native_h; + zones = std::move(new_zones); + ref_image = Image(); // force reseed at the new resolution + } + return &secondary_image_native; + } + + // Primary analysis: motion source is the decoded primary packet image. + do_score = true; + if (analysis_image == ANALYSISIMAGE_YCHANNEL) { + return packet->get_y_image(); // may be null if not decoded + } + return packet->image; +} + unsigned int Monitor::DetectMotion(const Image &comp_image, Event::StringSet &zoneSet) { bool alarm = false; unsigned int score = 0; @@ -3806,7 +3920,9 @@ unsigned int Monitor::AnalyseFrame(const Image &frame_image, Event::StringSet &z if (analysis_image && score) { analysis_image->Assign(frame_image); for (const Zone &zone : zones) { - if (zone.Alarmed() && zone.AlarmImage()) { + if (zone.Alarmed() && zone.AlarmImage() + && zone.AlarmImage()->Width() == analysis_image->Width() + && zone.AlarmImage()->Height() == analysis_image->Height()) { analysis_image->Overlay(*(zone.AlarmImage())); } } @@ -3923,6 +4039,22 @@ int Monitor::PrimeCapture() { packetqueue.notify_all(); // wake the thread if it's blocked on wait_for decoder->Join(); } + if (second_stream) { + second_stream->Stop(); + second_stream->Join(); + } + + // AnalysisSource=Secondary: analyse the low-res substream instead of the + // full-res primary. This does not change the primary Decoding setting: the + // user's choice is honoured (Encode recording, for instance, still needs the + // primary decoded). The analysis thread runs off the substream regardless of + // whether the primary is decoded (see the analysis-thread gate). + secondary_analysis = (analysis_source == ANALYSIS_SECONDARY) and !second_path.empty(); + if (secondary_analysis and (decoding == DECODING_ALWAYS)) { + Info("Monitor %d: secondary analysis active with Decoding=Always; the primary " + "stream is still decoded continuously. Set Decoding=Ondemand to save CPU " + "when no one is viewing (Encode recording still requires decoding).", id); + } int ret = camera->PrimeCapture(); if (ret <= 0) return ret; @@ -3988,6 +4120,17 @@ int Monitor::PrimeCapture() { } Debug(1, "Done restarting decoder"); + + if (secondary_analysis) { + if (!second_stream) { + Debug(1, "Creating secondary analysis stream thread for monitor %d", id); + second_stream = zm::make_unique(this); + } else { + Debug(1, "Restarting secondary analysis stream thread for monitor %d", id); + second_stream->Start(); + } + } + if (!analysis_it) { Debug(1, "getting analysis_it"); analysis_it = packetqueue.get_video_it(false); @@ -4011,6 +4154,9 @@ int Monitor::Pause() { // Because the stream indexes may change we have to clear out the packetqueue if (decoder) decoder->Stop(); + // Stop the substream sidecar early; it owns its own connection so it is + // independent of the packetqueue, but it must be joined before camera teardown. + if (second_stream) second_stream->Stop(); if (analysis_thread) { analysis_thread->Stop(); @@ -4036,6 +4182,10 @@ int Monitor::Pause() { Debug(1, "Joining analysis"); analysis_thread->Join(); } + if (second_stream) { + Debug(1, "Joining secondary analysis stream"); + second_stream->Join(); + } // Must close event before closing camera because it uses in_streams if (close_event_thread.joinable()) { diff --git a/src/zm_monitor.h b/src/zm_monitor.h index dd9cb05e3ed..818397ca2cb 100644 --- a/src/zm_monitor.h +++ b/src/zm_monitor.h @@ -25,6 +25,7 @@ #include "zm_analysis_thread.h" #include "zm_poll_thread.h" #include "zm_decoder_thread.h" +#include "zm_second_stream_thread.h" #include "zm_event.h" #include "zm_fifo.h" #include "zm_image.h" @@ -60,6 +61,7 @@ class Monitor : public std::enable_shared_from_this { friend class MonitorStream; friend class MonitorLinkExpression; friend class ONVIF; + friend class SecondStreamThread; public: typedef enum { @@ -524,6 +526,7 @@ class Monitor : public std::enable_shared_from_this { RecordingSourceOption recording_source; // Primary, Secondary, Both DecodingOption decoding; // Whether the monitor will decode h264/h265 packets + bool secondary_analysis; // AnalysisSource=Secondary and a SecondPath is configured: analyse the substream bool RTSP2Web_enabled; // Whether we set the h264/h265 stream up on RTSP2Web int RTSP2Web_type; // Whether we set the h264/h265 stream up on RTSP2Web StreamChannelOption stream_channel; // Which stream source to use: Restream, CameraDirectPrimary, or CameraDirectSecondary @@ -563,6 +566,15 @@ class Monitor : public std::enable_shared_from_this { int camera_height; unsigned int width; // Normally the same as the camera, but not if partly rotated unsigned int height; // Normally the same as the camera, but not if partly rotated + unsigned int analysis_width; // Resolution motion detection runs at. Same as width/height + unsigned int analysis_height; // except in AnalysisSource=Secondary, where it is the substream's + // native size so zones/detection run cheaply at the substream res. + // Substream native size once discovered from its first decoded frame (0 until + // then). Persisted across Load()/Reload() so a reload does not reset the + // analysis resolution to the full frame and force a redundant zone rebuild + + // reference-image reset. + unsigned int substream_width = 0; + unsigned int substream_height = 0; bool v4l_multi_buffer; unsigned int v4l_captures_per_frame; Orientation orientation; // Whether the image has to be rotated at all @@ -702,6 +714,9 @@ class Monitor : public std::enable_shared_from_this { std::unique_ptr analysis_thread; packetqueue_iterator *decoder_it; std::unique_ptr decoder; + std::unique_ptr second_stream; // decodes the substream for analysis (AnalysisSource=Secondary) + Image secondary_image_native; // latest substream image at its native (low) resolution, copied from the sidecar mailbox + uint64_t last_secondary_sequence = 0; // sidecar frame counter last scored, to avoid re-scoring a stale frame SwsContext *convert_context; std::thread close_event_thread; @@ -915,6 +930,10 @@ class Monitor : public std::enable_shared_from_this { unsigned int Width() const { return width; } unsigned int Height() const { return height; } + // Resolution motion detection / zones run at. Equals Width()/Height() except in + // AnalysisSource=Secondary, where it tracks the substream's native size. + unsigned int AnalysisWidth() const { return analysis_width; } + unsigned int AnalysisHeight() const { return analysis_height; } unsigned int Colours() const; unsigned int SubpixelOrder() const; @@ -1023,6 +1042,12 @@ class Monitor : public std::enable_shared_from_this { bool CheckSignal( const Image *image ); bool Analyse(); bool setupConvertContext(const AVFrame *input_frame, const Image *image); + // Resolve the image to run motion detection / ref-blending against for this + // packet. In secondary-analysis mode this comes from the substream sidecar + // (independent of whether the primary was decoded); otherwise it is the + // primary packet's Y-channel or colour image. do_score is set false when the + // caller should skip scoring this round (no fresh/valid substream frame). + Image *getMotionSourceImage(const std::shared_ptr &packet, bool &do_score); // Write capture_image into image_buffer[index] without conversion and // record its AVPixelFormat in image_pixelformats[index] so reading // processes can adopt that format via ReadShmFrame. diff --git a/src/zm_packet.cpp b/src/zm_packet.cpp index 0934d9c383a..a9eaccb4876 100644 --- a/src/zm_packet.cpp +++ b/src/zm_packet.cpp @@ -33,6 +33,7 @@ ZMPacket::ZMPacket() : locked(false), keyframe(0), stream(nullptr), + timestamp_steady(std::chrono::steady_clock::now()), image(nullptr), y_image(nullptr), analysis_image(nullptr), @@ -52,6 +53,7 @@ ZMPacket::ZMPacket(Image *i, SystemTimePoint tv) : keyframe(0), stream(nullptr), timestamp(tv), + timestamp_steady(std::chrono::steady_clock::now()), image(i), y_image(nullptr), analysis_image(nullptr), @@ -71,6 +73,7 @@ ZMPacket::ZMPacket(ZMPacket &p) : keyframe(p.keyframe), stream(p.stream), timestamp(p.timestamp), + timestamp_steady(p.timestamp_steady), image(p.image), y_image(p.y_image), analysis_image(p.analysis_image), @@ -314,7 +317,10 @@ AVPacket *ZMPacket::set_packet(AVPacket *p) { Error("error refing packet"); } + // Stamp system and steady clocks back-to-back: the steady stamp pairs with + // the substream sidecar's steady frame times for NTP-immune wallclock sync. timestamp = std::chrono::system_clock::now(); + timestamp_steady = std::chrono::steady_clock::now(); keyframe = p->flags & AV_PKT_FLAG_KEY; return packet.get(); } diff --git a/src/zm_packet.h b/src/zm_packet.h index 67aca0a1507..f307e51f617 100644 --- a/src/zm_packet.h +++ b/src/zm_packet.h @@ -55,6 +55,12 @@ class ZMPacket { av_frame_ptr out_frame; // output image, Only filled if needed. av_frame_ptr hw_frame; // output image, Only filled if needed. SystemTimePoint timestamp; + // steady_clock capture time, stamped back-to-back with timestamp. Used for + // wallclock sync against the substream sidecar (which stamps on the steady + // clock), so the comparison is immune to system-clock steps even after + // capture start. Always valid: every constructor stamps it, and the capture + // paths re-stamp it together with timestamp. + TimePoint timestamp_steady; Image *image; Image *y_image; Image *analysis_image; diff --git a/src/zm_second_stream_thread.cpp b/src/zm_second_stream_thread.cpp new file mode 100644 index 00000000000..04f36a3261a --- /dev/null +++ b/src/zm_second_stream_thread.cpp @@ -0,0 +1,289 @@ +#include "zm_second_stream_thread.h" + +#include "zm_ffmpeg.h" +#include "zm_ffmpeg_input.h" +#include "zm_monitor.h" +#include "zm_signal.h" +#include "zm_utils.h" +#include "url.hpp" + +#include + +namespace { +constexpr int kMinBackoffSeconds = 1; +constexpr int kMaxBackoffSeconds = 30; +} // namespace + +SecondStreamThread::SecondStreamThread(Monitor *monitor) : + monitor_(monitor), + terminate_(false), + input_(nullptr), + convert_context_(nullptr), + have_image_(false), + sequence_(0) { + thread_ = std::thread(&SecondStreamThread::Run, this); +} + +SecondStreamThread::~SecondStreamThread() { + Stop(); + if (thread_.joinable()) thread_.join(); + CloseInput(); +} + +void SecondStreamThread::Start() { + Stop(); // Signal any running thread to terminate first + if (thread_.joinable()) thread_.join(); + // Drop the pre-restart frame so the mailbox does not serve a stale image from + // the previous connection (e.g. after a reconnect or SecondPath change). Keep + // sequence_ monotonic across restarts: consumers hold last_secondary_sequence + // and a reset-to-equal value would read as "no fresh frame". + { + std::lock_guard lck(mutex_); + have_image_ = false; + } + terminate_ = false; + thread_ = std::thread(&SecondStreamThread::Run, this); +} + +void SecondStreamThread::Stop() { + terminate_ = true; +} + +void SecondStreamThread::Join() { + if (thread_.joinable()) thread_.join(); +} + +bool SecondStreamThread::PeekLatest(uint64_t &sequence, TimePoint &capture_steady) { + std::lock_guard lck(mutex_); + if (!have_image_) return false; + sequence = sequence_; + capture_steady = capture_time_; + return true; +} + +bool SecondStreamThread::GetLatestImage(Image &dest, uint64_t &sequence, TimePoint &capture_steady) { + std::lock_guard lck(mutex_); + if (!have_image_) return false; + dest.Assign(latest_image_); + sequence = sequence_; + capture_steady = capture_time_; + return true; +} + +// Build the credential-injected substream URL, mirroring the primary open in +// FfmpegCamera so the sidecar authenticates the same way. +static std::string BuildSecondUrl(const std::string &second_path, + const std::string &user, + const std::string &pass) { + std::string url_string = second_path; + if (!user.empty()) { + try { + Url url(second_path); + if (url.user_info().empty()) { + url.user_info(user + ":" + pass); + url_string = url.str(); + } + } catch (const Url::parse_error &e) { + Debug(1, "Could not parse secondary path as URL: %s", e.what()); + } + } + return url_string; +} + +bool SecondStreamThread::OpenInput() { + const std::string url = BuildSecondUrl(monitor_->second_path, monitor_->user, monitor_->pass); + + // Replicate the primary stream's ffmpeg input options so, in particular, an + // rtsp_transport=tcp primary does not end up pulling the substream over UDP. + AVDictionary *opts = nullptr; + if (!monitor_->options.empty()) { + if (av_dict_parse_string(&opts, monitor_->options.c_str(), "=", ",", 0) < 0) { + Warning("Monitor %d: could not parse ffmpeg options for substream '%s'", + monitor_->id, monitor_->options.c_str()); + } + } + // General avio read/write timeout so a dead non-RTSP network substream is + // detected and the thread can be joined on shutdown instead of blocking + // forever in av_read_frame. The rtsp demuxer ignores rw_timeout (it uses its + // own timeout option, set below), but other avio-based protocols honour it. + av_dict_set(&opts, "rw_timeout", "5000000", AV_DICT_DONT_OVERWRITE); // microseconds + if (StringToUpper(url.substr(0, 4)) == "RTSP") { + // Bound socket reads so a dead substream is detected (and the thread can be + // joined on shutdown) instead of blocking forever in av_read_frame. The + // RTSP option was renamed stimeout -> timeout in ffmpeg 5.0; set both so the + // bound applies regardless of the linked ffmpeg version. + av_dict_set(&opts, "timeout", "5000000", AV_DICT_DONT_OVERWRITE); // ffmpeg >= 5.0, microseconds + av_dict_set(&opts, "stimeout", "5000000", AV_DICT_DONT_OVERWRITE); // ffmpeg < 5.0, microseconds + zm_set_rtsp_transport_method(&opts, monitor_->method); + } + + input_ = new FFmpeg_Input(); + // The D1 substream is cheap to decode on the CPU, and we cannot consume + // hardware-format frames here. Forcing software decode also avoids contending + // with the primary stream over the VAAPI device (/dev/dri/renderD128). + input_->set_no_hwaccel(true); + int ret = input_->Open(url.c_str(), &opts); + av_dict_free(&opts); + + if (ret <= 0) { + Warning("Monitor %d: failed to open secondary analysis stream", monitor_->id); + CloseInput(); + return false; + } + if (input_->get_video_stream_id() < 0) { + Warning("Monitor %d: no video stream in secondary analysis input", monitor_->id); + CloseInput(); + return false; + } + Debug(1, "Monitor %d: opened secondary analysis stream (video stream %d)", + monitor_->id, input_->get_video_stream_id()); + return true; +} + +void SecondStreamThread::CloseInput() { + if (input_) { + delete input_; // FFmpeg_Input dtor closes and frees everything it allocated + input_ = nullptr; + } + if (convert_context_) { + sws_freeContext(convert_context_); + convert_context_ = nullptr; + } +} + +bool SecondStreamThread::ProduceImage(AVFrame *frame) { + // We decode the substream in software (set_no_hwaccel), so frames are CPU + // buffers. Guard anyway: a hardware-format frame has a GPU surface in data[0], + // not pixels, and must never be treated as a Y plane / swscale input. + if (frame->hw_frames_ctx) { + Error("Monitor %d: unexpected hardware substream frame (format %d); skipping", + monitor_->id, frame->format); + return false; + } + + // The mailbox stores the image at the substream's NATIVE resolution. The + // consumer (Monitor::getMotionSourceImage) upscales to camera/zone dimensions + // only when it actually scores a frame, so the expensive upscale runs at the + // analysis rate (AnalysisFPSLimit) rather than the substream decode rate. + Image produced; + + if (monitor_->analysis_image == Monitor::ANALYSISIMAGE_YCHANNEL) { + // Mirror ZMPacket::get_y_image(): Y lives in data[0] of a planar YUV frame. + const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(static_cast(frame->format)); + if (!desc) { + Error("Monitor %d: no pixel format descriptor for substream format %d", + monitor_->id, frame->format); + return false; + } + if (desc->flags & AV_PIX_FMT_FLAG_RGB) { + Error("Monitor %d: cannot get Y image from RGB substream format %s", monitor_->id, desc->name); + return false; + } + if (!(desc->flags & AV_PIX_FMT_FLAG_PLANAR)) { + Error("Monitor %d: cannot get Y image from non-planar substream format %s", monitor_->id, desc->name); + return false; + } + + // Copy the Y plane into a single-channel image. Use the align-32 Image + // constructor (buffer=nullptr, allocation=0) because ZM's Image::Scale and + // all swscale paths assume an align-32 row layout; a width-packed image + // (linesize == width) is misread at FFALIGN(width,32) when width is not a + // multiple of 32 (e.g. 720), shearing the image into bands. av_image_copy_plane + // bridges the decoder's stride (frame->linesize[0]) to the image's stride. + produced = Image(frame->width, frame->height, 1, ZM_SUBPIX_ORDER_NONE, nullptr, 0UL, 0); + av_image_copy_plane( + produced.Buffer(), produced.LineSize(), + frame->data[0], frame->linesize[0], + frame->width, frame->height); + } else { + // Full colour: convert pixel format only (still at native substream dims). + // Align-32 layout (see the YChannel branch) so the downstream Scale is not + // misread for widths that are not a multiple of 32. + produced = Image(frame->width, frame->height, + monitor_->camera->Colours(), monitor_->camera->SubpixelOrder(), + nullptr, 0UL, 0); + + if (!convert_context_) { + AVPixelFormat input_format; + const int *coefs = nullptr; + int src_range = 0; + switch (frame->format) { + case AV_PIX_FMT_YUVJ420P: input_format = AV_PIX_FMT_YUV420P; src_range = 1; break; + case AV_PIX_FMT_YUVJ422P: input_format = AV_PIX_FMT_YUV422P; src_range = 1; break; + case AV_PIX_FMT_YUVJ444P: input_format = AV_PIX_FMT_YUV444P; src_range = 1; break; + case AV_PIX_FMT_YUVJ440P: input_format = AV_PIX_FMT_YUV440P; src_range = 1; break; + default: input_format = static_cast(frame->format); + } + convert_context_ = sws_getContext( + frame->width, frame->height, input_format, + frame->width, frame->height, produced.AVPixFormat(), + SWS_BICUBIC, nullptr, nullptr, nullptr); + if (!convert_context_) { + Error("Monitor %d: unable to create substream conversion context", monitor_->id); + return false; + } + if (src_range) { + // Mark the source as full-range (yuvj) so levels are not crushed. + int *inv_table = nullptr; + int *table = nullptr; + int in_range, out_range, brightness, contrast, saturation; + sws_getColorspaceDetails(convert_context_, &inv_table, &in_range, + &table, &out_range, + &brightness, &contrast, &saturation); + coefs = sws_getCoefficients(SWS_CS_DEFAULT); + sws_setColorspaceDetails(convert_context_, coefs, 1, coefs, out_range, + brightness, contrast, saturation); + } + } + if (!produced.Assign(frame, convert_context_)) { + Error("Monitor %d: failed to convert substream frame", monitor_->id); + return false; + } + } + + { + std::lock_guard lck(mutex_); + latest_image_.Assign(produced); + have_image_ = true; + sequence_++; + capture_time_ = std::chrono::steady_clock::now(); + } + return true; +} + +void SecondStreamThread::Run() { + Debug(2, "SecondStreamThread::Run() for monitor %d", monitor_->id); + + int backoff = kMinBackoffSeconds; + + // Interruptible sleep: wake promptly on Stop()/zm_terminate. + auto backoff_sleep = [this, &backoff]() { + for (int i = 0; i < backoff * 10 && !(terminate_ or zm_terminate); i++) { + std::this_thread::sleep_for(Milliseconds(100)); + } + backoff = std::min(backoff * 2, kMaxBackoffSeconds); + }; + + while (!(terminate_ or zm_terminate)) { + if (!input_) { + if (!OpenInput()) { + backoff_sleep(); + continue; + } + } + + AVFrame *frame = input_->get_frame(input_->get_video_stream_id()); + if (!frame) { + Warning("Monitor %d: secondary analysis stream read failed, reconnecting", monitor_->id); + CloseInput(); + backoff_sleep(); + continue; + } + + ProduceImage(frame); + backoff = kMinBackoffSeconds; // healthy stream, reset backoff + } + + CloseInput(); + Debug(2, "SecondStreamThread::Run() exiting for monitor %d", monitor_->id); +} diff --git a/src/zm_second_stream_thread.h b/src/zm_second_stream_thread.h new file mode 100644 index 00000000000..643f91cd364 --- /dev/null +++ b/src/zm_second_stream_thread.h @@ -0,0 +1,77 @@ +#ifndef ZM_SECOND_STREAM_THREAD_H +#define ZM_SECOND_STREAM_THREAD_H + +#include "zm_image.h" +#include "zm_time.h" + +#include +#include +#include +#include + +class Monitor; +class FFmpeg_Input; +struct AVFrame; +struct SwsContext; + +// Decodes a monitor's low-res analysis substream (SecondPath) in its own thread +// and publishes the latest decoded frame as an analysis-ready Image at the +// substream's NATIVE resolution (no scaling to the primary camera dimensions). +// Depending on the monitor's analysis_image setting the mailbox holds either the +// raw Y channel (ANALYSISIMAGE_YCHANNEL) or a full-colour, pixel-format-converted +// image; either way it stays at the substream's own width/height and the consumer +// (Monitor::getMotionSourceImage) rebuilds zones at that size. Used when +// AnalysisSource=Secondary so that motion detection does not require +// software-decoding the full-resolution primary stream while nobody is watching +// live. +// +// The thread owns its own FFmpeg_Input outright (it must free nothing it did not +// allocate) and never emits packets into the monitor packetqueue. On read +// failure it closes the input and reconnects with exponential backoff, which +// also keeps the substream RTSP session alive (an unread session gets FIN'd by +// the camera and left stuck in CLOSE-WAIT). +class SecondStreamThread { + public: + explicit SecondStreamThread(Monitor *monitor); + ~SecondStreamThread(); + SecondStreamThread(const SecondStreamThread &) = delete; + SecondStreamThread(SecondStreamThread &&) = delete; + + void Start(); + void Stop(); + void Join(); + + // Cheap peek at the newest frame's metadata without copying pixels. Returns + // false if no frame has ever been produced. On success sets sequence (a + // monotonically increasing frame counter used to detect fresh frames) and + // capture_steady (the steady_clock time the frame was captured, for wallclock + // sync against a primary packet's steady stamp). Callers use this to decide + // whether a copy is worthwhile before paying for GetLatestImage. + bool PeekLatest(uint64_t &sequence, TimePoint &capture_steady); + + // Copy the latest published analysis image into dest. Returns false if no + // frame has ever been produced. On success sets sequence and capture_steady + // as above. + bool GetLatestImage(Image &dest, uint64_t &sequence, TimePoint &capture_steady); + + private: + void Run(); + bool OpenInput(); + void CloseInput(); + bool ProduceImage(AVFrame *frame); + + Monitor *monitor_; + std::atomic terminate_; + std::thread thread_; + + FFmpeg_Input *input_; + SwsContext *convert_context_; + + std::mutex mutex_; + Image latest_image_; + bool have_image_; + uint64_t sequence_; + TimePoint capture_time_; // steady_clock time the latest frame was captured +}; + +#endif // ZM_SECOND_STREAM_THREAD_H diff --git a/src/zm_secondary_sync.h b/src/zm_secondary_sync.h new file mode 100644 index 00000000000..e53a5fb41a3 --- /dev/null +++ b/src/zm_secondary_sync.h @@ -0,0 +1,30 @@ +#ifndef ZM_SECONDARY_SYNC_H +#define ZM_SECONDARY_SYNC_H + +#include "zm_time.h" + +#include + +// Wallclock sync helper for AnalysisSource=Secondary. +// +// The primary capture stamps each packet with a steady_clock timestamp +// (ZMPacket::timestamp_steady) taken back-to-back with its system timestamp, +// and the substream sidecar stamps each decoded frame with a steady_clock time +// (SecondStreamThread::capture_time_). Comparing the two steady stamps never +// touches the system clock, so the check is immune to NTP steps at any time. + +// True when the substream has stalled or died: the packet under analysis was +// captured more than max_skew AFTER the newest substream frame, i.e. the +// sidecar has stopped producing frames. The test is deliberately one-sided: a +// frame NEWER than the packet just means the analysis thread lags capture +// (packetqueue backlog) while the substream is healthy, and analysis pairs +// with the freshest frame as it always has, so such frames must still be +// scored. +inline bool SecondaryFrameStalled(TimePoint packet_steady_ts, + TimePoint frame_steady_ts, + Seconds max_skew) { + return (packet_steady_ts - frame_steady_ts) > + std::chrono::duration_cast(max_skew); +} + +#endif // ZM_SECONDARY_SYNC_H diff --git a/src/zm_zone.cpp b/src/zm_zone.cpp index edace6d09be..a274e493db0 100644 --- a/src/zm_zone.cpp +++ b/src/zm_zone.cpp @@ -73,18 +73,21 @@ void Zone::Setup( overload_count = 0; extend_alarm_count = 0; - pg_image = new Image(monitor->Width(), monitor->Height(), 1, ZM_SUBPIX_ORDER_NONE); + // Build the polygon mask/ranges at the analysis resolution (== camera size for + // primary analysis, but the substream's native size for AnalysisSource=Secondary) + // so the zone matches the image DetectMotion actually operates on. + pg_image = new Image(monitor->AnalysisWidth(), monitor->AnalysisHeight(), 1, ZM_SUBPIX_ORDER_NONE); pg_image->Clear(); pg_image->Fill(0xff, polygon); pg_image->Outline(0xff, polygon); - ranges = new Range[monitor->Height()]; - for ( unsigned int y = 0; y < monitor->Height(); y++ ) { + ranges = new Range[monitor->AnalysisHeight()]; + for ( unsigned int y = 0; y < monitor->AnalysisHeight(); y++ ) { ranges[y].lo_x = -1; ranges[y].hi_x = 0; ranges[y].off_x = 0; const uint8_t *ppoly = pg_image->Buffer( 0, y ); - for ( unsigned int x = 0; x < monitor->Width(); x++, ppoly++ ) { + for ( unsigned int x = 0; x < monitor->AnalysisWidth(); x++, ppoly++ ) { if ( *ppoly ) { if ( ranges[y].lo_x == -1 ) { ranges[y].lo_x = x; @@ -997,7 +1000,7 @@ std::vector Zone::Load(const std::shared_ptr &monitor) { Polygon polygon; if (strchr(Coords, '.')) { // Decimal values present — treat as percentages regardless of Units field - if (!ParsePercentagePolygon(Coords, monitor->Width(), monitor->Height(), polygon)) { + if (!ParsePercentagePolygon(Coords, monitor->AnalysisWidth(), monitor->AnalysisHeight(), polygon)) { Error("Unable to parse polygon string '%s' for zone %d/%s for monitor %s, ignoring", Coords, Id, Name, monitor->Name()); continue; @@ -1192,8 +1195,10 @@ Zone::Zone(const Zone &z) : diag_path(z.diag_path) { std::copy(z.blob_stats, z.blob_stats+256, blob_stats); pg_image = z.pg_image ? new Image(*z.pg_image) : nullptr; - ranges = new Range[monitor->Height()]; - std::copy(z.ranges, z.ranges+monitor->Height(), ranges); + // ranges is sized to the analysis resolution (see Setup), which is what the + // source zone was built at - copying monitor->Height() here would over-read. + ranges = new Range[monitor->AnalysisHeight()]; + std::copy(z.ranges, z.ranges+monitor->AnalysisHeight(), ranges); image = z.image ? new Image(*z.image) : nullptr; //z.stats.debug("Copy Source"); stats.DumpToLog("Copy dest"); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9007f279d47..34651caf21f 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -25,6 +25,7 @@ set(TEST_SOURCES zm_pixformat.cpp zm_swscale_range.cpp zm_poly.cpp + zm_secondary_sync.cpp zm_time.cpp zm_utils.cpp zm_vector2.cpp diff --git a/tests/zm_secondary_sync.cpp b/tests/zm_secondary_sync.cpp new file mode 100644 index 00000000000..5eb9964bef3 --- /dev/null +++ b/tests/zm_secondary_sync.cpp @@ -0,0 +1,67 @@ +/* + * This file is part of the ZoneMinder Project. See AUTHORS file for Copyright information + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "zm_catch2.h" + +#include "zm_secondary_sync.h" +#include "zm_time.h" + +#include + +// An arbitrary steady_clock origin for the fabricated capture times below. +namespace { +const TimePoint kBase = TimePoint() + Seconds(1000); +const Seconds kThreshold = Seconds(10); +} // namespace + +TEST_CASE("SecondaryFrameStalled: frame captured with the packet is not stalled") { + REQUIRE_FALSE(SecondaryFrameStalled(kBase, kBase, kThreshold)); + // Frame slightly older than the packet, within the threshold. + REQUIRE_FALSE(SecondaryFrameStalled(kBase + Seconds(5), kBase, kThreshold)); +} + +TEST_CASE("SecondaryFrameStalled: analysis backlog (frame newer than packet) must still score") { + // The analysis thread lags capture: the packet under analysis is old, but the + // substream is healthy and its newest frame is 15s NEWER than the packet. + // The one-sided test must not report a stall, no matter how large the lag. + TimePoint packet = kBase; + REQUIRE_FALSE(SecondaryFrameStalled(packet, kBase + Seconds(15), kThreshold)); + REQUIRE_FALSE(SecondaryFrameStalled(packet, kBase + Minutes(10), kThreshold)); +} + +TEST_CASE("SecondaryFrameStalled: dead substream (packet far ahead of frozen frame) is stalled") { + // The sidecar stopped producing: the newest frame is frozen 15s behind the + // packet under analysis. + TimePoint frozen_frame = kBase; + REQUIRE(SecondaryFrameStalled(kBase + Seconds(15), frozen_frame, kThreshold)); + REQUIRE(SecondaryFrameStalled(kBase + Minutes(10), frozen_frame, kThreshold)); +} + +TEST_CASE("SecondaryFrameStalled: threshold boundary on the stalled side") { + // Packet exactly the threshold ahead of the frame: not yet stalled (strict >). + REQUIRE_FALSE(SecondaryFrameStalled(kBase + Seconds(10), kBase, kThreshold)); + // One millisecond past the threshold: stalled. + REQUIRE(SecondaryFrameStalled(kBase + Seconds(10) + Milliseconds(1), kBase, kThreshold)); +} + +TEST_CASE("SecondaryFrameStalled: threshold boundary on the backlog side") { + // Any frame newer than the packet is never a stall, including exactly at and + // beyond the threshold distance. + TimePoint packet = kBase; + REQUIRE_FALSE(SecondaryFrameStalled(packet, kBase + Seconds(10), kThreshold)); + REQUIRE_FALSE(SecondaryFrameStalled(packet, kBase + Seconds(10) + Milliseconds(1), kThreshold)); +} diff --git a/web/lang/en_gb.php b/web/lang/en_gb.php index 57fd734200b..6ce3ebce9d0 100644 --- a/web/lang/en_gb.php +++ b/web/lang/en_gb.php @@ -1099,6 +1099,7 @@ function zmVlang($langVarArray, $count) { OnDemand: only do decoding when someone is watching.~~~~ KeyFrames: Only keyframes will be decoded, so viewing frame rate will be very low, depending on the keyframe interval set in the camera.~~~~ None: No frames will be decoded, live view and thumbnails will not be available~~~~ +When AnalysisSource is set to Secondary, motion detection runs off the low-res substream and no longer requires decoding the primary stream. Pair Secondary analysis with Decoding=OnDemand to get the CPU saving: the primary is then only decoded while someone is watching. This setting is always honoured, so Encode recording (VideoWriter=Encode) still requires decoding as before.~~~~ ' ), 'FUNCTION_RTSP2WEB_ENABLED' => array(