From 1a224c77def00d72376383f64d58e674d27f5e6b Mon Sep 17 00:00:00 2001 From: chenhaibo <495810242@qq.com> Date: Sun, 16 Feb 2025 09:19:48 +0800 Subject: [PATCH 01/17] rtmp2rtc: support hevc --- trunk/src/app/srs_app_rtc_conn.cpp | 166 +++++++--- trunk/src/app/srs_app_rtc_sdp.cpp | 39 ++- trunk/src/app/srs_app_rtc_sdp.hpp | 10 +- trunk/src/app/srs_app_rtc_source.cpp | 260 +++++++++++----- trunk/src/app/srs_app_rtc_source.hpp | 3 + trunk/src/app/srs_app_stream_bridge.cpp | 25 +- trunk/src/app/srs_app_stream_bridge.hpp | 1 + trunk/src/kernel/srs_kernel_codec.cpp | 84 +++++- trunk/src/kernel/srs_kernel_codec.hpp | 11 + trunk/src/kernel/srs_kernel_error.hpp | 2 +- trunk/src/kernel/srs_kernel_rtc_rtp.cpp | 383 +++++++++++++++++++++++- trunk/src/kernel/srs_kernel_rtc_rtp.hpp | 76 ++++- 12 files changed, 927 insertions(+), 133 deletions(-) diff --git a/trunk/src/app/srs_app_rtc_conn.cpp b/trunk/src/app/srs_app_rtc_conn.cpp index af2fbcfdcc..410db41851 100644 --- a/trunk/src/app/srs_app_rtc_conn.cpp +++ b/trunk/src/app/srs_app_rtc_conn.cpp @@ -2598,6 +2598,51 @@ bool srs_sdp_has_h264_profile(const SrsSdp& sdp, const string& profile) return false; } +bool srs_sdp_has_h265_profile(const SrsMediaPayloadType& payload_type, const string& profile) +{ + srs_error_t err = srs_success; + + if (payload_type.format_specific_param_.empty()) { + return false; + } + + H265SpecificParam h265_param; + if ((err = srs_parse_h265_fmtp(payload_type.format_specific_param_, h265_param)) != srs_success) { + srs_error_reset(err); + return false; + } + + if (h265_param.profile_id == profile) { + return true; + } + + return false; +} + +bool srs_sdp_has_h265_profile(const SrsSdp& sdp, const string& profile) +{ + for (size_t i = 0; i < sdp.media_descs_.size(); ++i) { + const SrsMediaDesc& desc = sdp.media_descs_[i]; + if (!desc.is_video()) { + continue; + } + + std::vector payloads = desc.find_media_with_encoding_name("H265"); + if (payloads.empty()) { + continue; + } + + for (std::vector::iterator it = payloads.begin(); it != payloads.end(); ++it) { + const SrsMediaPayloadType& payload_type = *it; + if (srs_sdp_has_h265_profile(payload_type, profile)) { + return true; + } + } + } + + return false; +} + srs_error_t SrsRtcConnection::negotiate_publish_capability(SrsRtcUserConfig* ruc, SrsRtcSourceDescription* stream_desc) { srs_error_t err = srs_success; @@ -3041,8 +3086,6 @@ srs_error_t SrsRtcConnection::negotiate_play_capability(SrsRtcUserConfig* ruc, s bool nack_enabled = _srs_config->get_rtc_nack_enabled(req->vhost); bool twcc_enabled = _srs_config->get_rtc_twcc_enabled(req->vhost); - // TODO: FIME: Should check packetization-mode=1 also. - bool has_42e01f = srs_sdp_has_h264_profile(remote_sdp, "42e01f"); SrsSharedPtr source; if ((err = _srs_rtc_sources->fetch_or_create(req, source)) != srs_success) { @@ -3083,56 +3126,87 @@ srs_error_t SrsRtcConnection::negotiate_play_capability(SrsRtcUserConfig* ruc, s remote_payload = payloads.at(0); track_descs = source->get_track_desc("audio", "opus"); - } else if (remote_media_desc.is_video() && ruc->codec_ == "av1") { - std::vector payloads = remote_media_desc.find_media_with_encoding_name("AV1"); - if (payloads.empty()) { - // Be compatible with the Chrome M96, still check the AV1X encoding name - // @see https://bugs.chromium.org/p/webrtc/issues/detail?id=13166 - payloads = remote_media_desc.find_media_with_encoding_name("AV1X"); - } - if (payloads.empty()) { - return srs_error_new(ERROR_RTC_SDP_EXCHANGE, "no found valid AV1 payload type"); + } else if (remote_media_desc.is_video()) { + std::string prefer_codec = ruc->codec_; + if (prefer_codec.empty()) { + // Get the source codec if not specified. + std::vector track_descs = source->get_track_desc("video", ""); + if (!track_descs.empty()) { + std::string codec_name = track_descs.at(0)->media_->name_; + std::transform(codec_name.begin(), codec_name.end(), codec_name.begin(), ::tolower); + if (codec_name == "h265") { + prefer_codec = "hevc"; + } else { + prefer_codec = codec_name; + } + } else { + return srs_error_new(ERROR_RTC_SDP_EXCHANGE, "no video track in source"); + } } - remote_payload = payloads.at(0); - track_descs = source->get_track_desc("video", "AV1"); - if (track_descs.empty()) { - // Be compatible with the Chrome M96, still check the AV1X encoding name - // @see https://bugs.chromium.org/p/webrtc/issues/detail?id=13166 - track_descs = source->get_track_desc("video", "AV1X"); - } - } else if (remote_media_desc.is_video() && ruc->codec_ == "hevc") { - std::vector payloads = remote_media_desc.find_media_with_encoding_name("H265"); - if (payloads.empty()) { - return srs_error_new(ERROR_RTC_SDP_EXCHANGE, "no valid found h265 payload type"); - } + if (prefer_codec == "av1") { + std::vector payloads = remote_media_desc.find_media_with_encoding_name("AV1"); + if (payloads.empty()) { + // Be compatible with the Chrome M96, still check the AV1X encoding name + // @see https://bugs.chromium.org/p/webrtc/issues/detail?id=13166 + payloads = remote_media_desc.find_media_with_encoding_name("AV1X"); + } + if (payloads.empty()) { + return srs_error_new(ERROR_RTC_SDP_EXCHANGE, "no found valid AV1 payload type"); + } - remote_payload = payloads.at(0); + remote_payload = payloads.at(0); + track_descs = source->get_track_desc("video", "AV1"); + if (track_descs.empty()) { + // Be compatible with the Chrome M96, still check the AV1X encoding name + // @see https://bugs.chromium.org/p/webrtc/issues/detail?id=13166 + track_descs = source->get_track_desc("video", "AV1X"); + } + } else if (prefer_codec == "hevc") { + std::vector payloads = remote_media_desc.find_media_with_encoding_name("H265"); + if (payloads.empty()) { + return srs_error_new(ERROR_RTC_SDP_EXCHANGE, "no valid found h265 payload type"); + } + + // @see https://www.rfc-editor.org/rfc/rfc7798#section-7.2.1 + bool has_main_profile = srs_sdp_has_h265_profile(remote_sdp, "1"); + remote_payload = payloads.at(0); + + for (int j = 0; j < (int)payloads.size(); j++) { + const SrsMediaPayloadType& payload = payloads.at(j); + + // For H.265, we only check if profile-id=1 (Main Profile) + // Format example: level-id=180;profile-id=1;tier-flag=0;tx-mode=SRST + if (!has_main_profile || srs_sdp_has_h265_profile(payload, "1")) { + remote_payload = payload; + break; + } + } - // TODO: FIXME: pick up a profile for HEVC. - // @see https://www.rfc-editor.org/rfc/rfc7798#section-7.2.1 + track_descs = source->get_track_desc("video", "H265"); + } else { + vector payloads = remote_media_desc.find_media_with_encoding_name("H264"); + if (payloads.empty()) { + return srs_error_new(ERROR_RTC_SDP_EXCHANGE, "no valid found h264 payload type"); + } - track_descs = source->get_track_desc("video", "H265"); - } else if (remote_media_desc.is_video()) { - // TODO: check opus format specific param - vector payloads = remote_media_desc.find_media_with_encoding_name("H264"); - if (payloads.empty()) { - return srs_error_new(ERROR_RTC_SDP_EXCHANGE, "no valid found h264 payload type"); - } + // TODO: FIME: Should check packetization-mode=1 also. + bool has_42e01f = srs_sdp_has_h264_profile(remote_sdp, "42e01f"); - remote_payload = payloads.at(0); - for (int j = 0; j < (int)payloads.size(); j++) { - const SrsMediaPayloadType& payload = payloads.at(j); + remote_payload = payloads.at(0); + for (int j = 0; j < (int)payloads.size(); j++) { + const SrsMediaPayloadType& payload = payloads.at(j); - // If exists 42e01f profile, choose it; otherwise, use the first payload. - // TODO: FIME: Should check packetization-mode=1 also. - if (!has_42e01f || srs_sdp_has_h264_profile(payload, "42e01f")) { - remote_payload = payload; - break; + // If exists 42e01f profile, choose it; otherwise, use the first payload. + // TODO: FIME: Should check packetization-mode=1 also. + if (!has_42e01f || srs_sdp_has_h264_profile(payload, "42e01f")) { + remote_payload = payload; + break; + } } - } - track_descs = source->get_track_desc("video", "H264"); + track_descs = source->get_track_desc("video", "H264"); + } } for (int j = 0; j < (int)track_descs.size(); ++j) { @@ -3238,7 +3312,11 @@ void video_track_generate_play_offer(SrsRtcTrackDescription* track, string mid, SrsVideoPayload* payload = (SrsVideoPayload*)track->media_; - local_media_desc.payload_types_.push_back(payload->generate_media_payload_type()); + if (payload->name_ == "H265") { + local_media_desc.payload_types_.push_back(payload->generate_media_payload_type_h265()); + } else { + local_media_desc.payload_types_.push_back(payload->generate_media_payload_type()); + } if (track->red_) { SrsRedPayload* red_payload = (SrsRedPayload*)track->red_; diff --git a/trunk/src/app/srs_app_rtc_sdp.cpp b/trunk/src/app/srs_app_rtc_sdp.cpp index e6f298a4ec..8f5e49a835 100644 --- a/trunk/src/app/srs_app_rtc_sdp.cpp +++ b/trunk/src/app/srs_app_rtc_sdp.cpp @@ -92,6 +92,42 @@ srs_error_t srs_parse_h264_fmtp(const std::string& fmtp, H264SpecificParam& h264 return err; } +srs_error_t srs_parse_h265_fmtp(const std::string& fmtp, H265SpecificParam& h265_param) +{ + srs_error_t err = srs_success; + + std::vector vec = srs_string_split(fmtp, ";"); + for (size_t i = 0; i < vec.size(); ++i) { + std::vector kv = srs_string_split(vec[i], "="); + if (kv.size() != 2) continue; + + if (kv[0] == "level-id") { + h265_param.level_id = kv[1]; + } else if (kv[0] == "profile-id") { + h265_param.profile_id = kv[1]; + } else if (kv[0] == "tier-flag") { + h265_param.tier_flag = kv[1]; + } else if (kv[0] == "tx-mode") { + h265_param.tx_mode = kv[1]; + } + } + + if (h265_param.level_id.empty()) { + return srs_error_new(ERROR_RTC_SDP_DECODE, "no h265 param: level-id"); + } + if (h265_param.profile_id.empty()) { + return srs_error_new(ERROR_RTC_SDP_DECODE, "no h265 param: profile-id"); + } + if (h265_param.tier_flag.empty()) { + return srs_error_new(ERROR_RTC_SDP_DECODE, "no h265 param: tier-flag"); + } + if (h265_param.tx_mode.empty()) { + return srs_error_new(ERROR_RTC_SDP_DECODE, "no h265 param: tx-mode"); + } + + return err; +} + SrsSessionInfo::SrsSessionInfo() { } @@ -1216,5 +1252,4 @@ srs_error_t SrsSdp::update_msid(string id) } return err; -} - +} \ No newline at end of file diff --git a/trunk/src/app/srs_app_rtc_sdp.hpp b/trunk/src/app/srs_app_rtc_sdp.hpp index d2aba8a489..39dbb8f0d8 100644 --- a/trunk/src/app/srs_app_rtc_sdp.hpp +++ b/trunk/src/app/srs_app_rtc_sdp.hpp @@ -97,8 +97,16 @@ struct H264SpecificParam std::string level_asymmerty_allow; }; -extern srs_error_t srs_parse_h264_fmtp(const std::string& fmtp, H264SpecificParam& h264_param); +struct H265SpecificParam +{ + std::string level_id; + std::string profile_id; + std::string tier_flag; + std::string tx_mode; +}; +extern srs_error_t srs_parse_h264_fmtp(const std::string& fmtp, H264SpecificParam& h264_param); +extern srs_error_t srs_parse_h265_fmtp(const std::string& fmtp, H265SpecificParam& h265_param); class SrsMediaPayloadType { public: diff --git a/trunk/src/app/srs_app_rtc_source.cpp b/trunk/src/app/srs_app_rtc_source.cpp index 43fde7845a..4d1d6390a3 100644 --- a/trunk/src/app/srs_app_rtc_source.cpp +++ b/trunk/src/app/srs_app_rtc_source.cpp @@ -785,7 +785,15 @@ std::vector SrsRtcSource::get_track_desc(std::string ty if (type == "video") { std::vector::iterator it = stream_desc_->video_track_descs_.begin(); while (it != stream_desc_->video_track_descs_.end() ){ - track_descs.push_back(*it); + if (media_name.empty()) { + track_descs.push_back(*it); + } else { + string name = (*it)->media_->name_; + std::transform(name.begin(), name.end(), name.begin(), static_cast(std::toupper)); + if (name == media_name) { + track_descs.push_back(*it); + } + } ++it; } } @@ -1065,13 +1073,6 @@ srs_error_t SrsRtcRtpBuilder::on_video(SrsSharedPtrMessage* msg) return err; } - // WebRTC does NOT support HEVC. -#ifdef SRS_H265 - if (format->vcodec->id == SrsVideoCodecIdHEVC) { - return err; - } -#endif - bool has_idr = false; vector samples; if ((err = filter(msg, format, has_idr, samples)) != srs_success) { @@ -1083,6 +1084,10 @@ srs_error_t SrsRtcRtpBuilder::on_video(SrsSharedPtrMessage* msg) if (has_idr) { SrsUniquePtr pkt(new SrsRtpPacket()); + if ((err = bridge_->update_codec(format->vcodec->id)) != srs_success) { + return srs_error_wrap(err, "update codec"); + } + if ((err = package_stap_a(msg, pkt.get())) != srs_success) { return srs_error_wrap(err, "package stap-a"); } @@ -1149,10 +1154,16 @@ srs_error_t SrsRtcRtpBuilder::filter(SrsSharedPtrMessage* msg, SrsFormat* format // Because RTC does not support B-frame, so we will drop them. // TODO: Drop B-frame in better way, which not cause picture corruption. - if (!keep_bframe && format->vcodec->id == SrsVideoCodecIdAVC) { - bool is_b_frame; - if ((err = SrsVideoFrame::parse_avc_b_frame(sample, is_b_frame)) != srs_success) { - return srs_error_wrap(err, "parse bframe"); + if (!keep_bframe) { + bool is_b_frame = false; + if (format->vcodec->id == SrsVideoCodecIdAVC) { + if ((err = SrsVideoFrame::parse_avc_b_frame(sample, is_b_frame)) != srs_success) { + return srs_error_wrap(err, "parse bframe"); + } + } else if (format->vcodec->id == SrsVideoCodecIdHEVC) { + if ((err = SrsVideoFrame::parse_hevc_b_frame(sample, format, is_b_frame)) != srs_success) { + return srs_error_wrap(err, "parse bframe"); + } } if (is_b_frame) { continue; @@ -1174,53 +1185,59 @@ srs_error_t SrsRtcRtpBuilder::package_stap_a(SrsSharedPtrMessage* msg, SrsRtpPac return err; } - // Note that the sps/pps may change, so we should copy it. - const vector& sps = format->vcodec->sequenceParameterSetNALUnit; - const vector& pps = format->vcodec->pictureParameterSetNALUnit; - if (sps.empty() || pps.empty()) { - return srs_error_new(ERROR_RTC_RTP_MUXER, "sps/pps empty"); - } - pkt->header.set_payload_type(video_payload_type_); pkt->header.set_ssrc(video_ssrc_); pkt->frame_type = SrsFrameTypeVideo; - pkt->nalu_type = (SrsAvcNaluType)kStapA; pkt->header.set_marker(false); pkt->header.set_sequence(video_sequence++); pkt->header.set_timestamp(msg->timestamp * 90); - SrsRtpSTAPPayload* stap = new SrsRtpSTAPPayload(); - pkt->set_payload(stap, SrsRtspPacketPayloadTypeSTAP); + ISrsRtpPayloader* stap = NULL; + vector*> params; + int size = 0; + if (format->vcodec->id == SrsVideoCodecIdHEVC) { + for (int i = 0; i < format->vcodec->hevc_dec_conf_record_.nalu_vec.size(); i++) { + if (format->vcodec->hevc_dec_conf_record_.nalu_vec[i].nal_unit_type == SrsHevcNaluType_VPS + || format->vcodec->hevc_dec_conf_record_.nalu_vec[i].nal_unit_type == SrsHevcNaluType_SPS + || format->vcodec->hevc_dec_conf_record_.nalu_vec[i].nal_unit_type == SrsHevcNaluType_PPS) { + vector& nalu = (vector&)format->vcodec->hevc_dec_conf_record_.nalu_vec[i].nal_data_vec[0].nal_unit_data; + params.push_back(&nalu); + size += format->vcodec->hevc_dec_conf_record_.nalu_vec[i].nal_data_vec[0].nal_unit_length; + } + } - uint8_t header = sps[0]; - stap->nri = (SrsAvcNaluType)header; + stap = new SrsRtpSTAPPayloadHevc(); + pkt->set_payload(stap, SrsRtspPacketPayloadTypeSTAPHevc); + pkt->nalu_type = kStapHevc; + } else if (format->vcodec->id == SrsVideoCodecIdAVC) { + params.push_back(&format->vcodec->sequenceParameterSetNALUnit); + params.push_back(&format->vcodec->pictureParameterSetNALUnit); + size = format->vcodec->sequenceParameterSetNALUnit.size() + format->vcodec->pictureParameterSetNALUnit.size(); - // Copy the SPS/PPS bytes, because it may change. - int size = (int)(sps.size() + pps.size()); - char* payload = pkt->wrap(size); - - if (true) { - SrsSample* sample = new SrsSample(); - sample->bytes = payload; - sample->size = (int)sps.size(); - stap->nalus.push_back(sample); + stap = new SrsRtpSTAPPayload(); + pkt->set_payload(stap, SrsRtspPacketPayloadTypeSTAP); + pkt->nalu_type = kStapA; + } - memcpy(payload, (char*)&sps[0], sps.size()); - payload += (int)sps.size(); + if (size == 0) { + return srs_error_new(ERROR_RTC_RTP_MUXER, "vps/sps/pps empty"); } + char* payload = pkt->wrap(size); - if (true) { + for (vector* param : params) { SrsSample* sample = new SrsSample(); sample->bytes = payload; - sample->size = (int)pps.size(); - stap->nalus.push_back(sample); + sample->size = param->size(); + if (format->vcodec->id == SrsVideoCodecIdHEVC) { + static_cast(stap)->nalus.push_back(sample); + } else { + static_cast(stap)->nalus.push_back(sample); + } - memcpy(payload, (char*)&pps[0], pps.size()); - payload += (int)pps.size(); + memcpy(payload, (char*)param->data(), param->size()); + payload += (int)param->size(); } - srs_info("RTC STAP-A seq=%u, sps %d, pps %d bytes", pkt->header.get_sequence(), sps.size(), pps.size()); - return err; } @@ -1260,12 +1277,22 @@ srs_error_t SrsRtcRtpBuilder::package_nalus(SrsSharedPtrMessage* msg, const vect pkt->header.set_payload_type(video_payload_type_); pkt->header.set_ssrc(video_ssrc_); pkt->frame_type = SrsFrameTypeVideo; - pkt->nalu_type = (SrsAvcNaluType)first_nalu_type; + pkt->nalu_type = first_nalu_type; pkt->header.set_sequence(video_sequence++); pkt->header.set_timestamp(msg->timestamp * 90); pkt->set_payload(raw_raw, SrsRtspPacketPayloadTypeNALU); pkt->wrap(msg); } else { + SrsFormat* format = meta->vsh_format(); + if (!format || !format->vcodec) { + return err; + } + + bool is_hevc = format->vcodec->id == SrsVideoCodecIdHEVC; + // H264 FU-A header size is 1 @see: https://datatracker.ietf.org/doc/html/rfc6184#section-5.8 + // H265 FU-A header size is 2 @see: https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.3 + int header_size = is_hevc ? 2 : 1; + // We must free it, should never use RTP packets to free it, // because more than one RTP packet will refer to it. SrsUniquePtr raw(raw_raw); @@ -1274,36 +1301,48 @@ srs_error_t SrsRtcRtpBuilder::package_nalus(SrsSharedPtrMessage* msg, const vect int fu_payload_size = kRtpMaxPayloadSize; // The first byte is store in FU-A header. - uint8_t header = raw->skip_first_byte(); - uint8_t nal_type = header & kNalTypeMask; - int nb_left = nn_bytes - 1; + uint8_t header = raw->skip_bytes(header_size); + + int nb_left = nn_bytes - header_size; int num_of_packet = 1 + (nn_bytes - 1) / fu_payload_size; for (int i = 0; i < num_of_packet; ++i) { int packet_size = srs_min(nb_left, fu_payload_size); - SrsRtpFUAPayload* fua = new SrsRtpFUAPayload(); - if ((err = raw->read_samples(fua->nalus, packet_size)) != srs_success) { - srs_freep(fua); - return srs_error_wrap(err, "read samples %d bytes, left %d, total %d", packet_size, nb_left, nn_bytes); - } - SrsRtpPacket* pkt = new SrsRtpPacket(); pkts.push_back(pkt); pkt->header.set_payload_type(video_payload_type_); pkt->header.set_ssrc(video_ssrc_); pkt->frame_type = SrsFrameTypeVideo; - pkt->nalu_type = (SrsAvcNaluType)kFuA; + pkt->nalu_type = kFuA; pkt->header.set_sequence(video_sequence++); pkt->header.set_timestamp(msg->timestamp * 90); - fua->nri = (SrsAvcNaluType)header; - fua->nalu_type = (SrsAvcNaluType)nal_type; - fua->start = bool(i == 0); - fua->end = bool(i == num_of_packet - 1); + if (is_hevc) { + SrsRtpFUAPayloadHevc* fua = new SrsRtpFUAPayloadHevc(); + if ((err = raw->read_samples(fua->nalus, packet_size)) != srs_success) { + srs_freep(fua); + return srs_error_wrap(err, "read samples %d bytes, left %d, total %d", packet_size, nb_left, nn_bytes); + } + fua->nalu_type = SrsHevcNaluTypeParse(header); + fua->start = bool(i == 0); + fua->end = bool(i == num_of_packet - 1); + + pkt->set_payload(fua, SrsRtspPacketPayloadTypeFUAHevc); + } else { + SrsRtpFUAPayload* fua = new SrsRtpFUAPayload(); + if ((err = raw->read_samples(fua->nalus, packet_size)) != srs_success) { + srs_freep(fua); + return srs_error_wrap(err, "read samples %d bytes, left %d, total %d", packet_size, nb_left, nn_bytes); + } + fua->nalu_type = (SrsAvcNaluType)(header & kNalTypeMask); + fua->start = bool(i == 0); + fua->end = bool(i == num_of_packet - 1); + + pkt->set_payload(fua, SrsRtspPacketPayloadTypeFUA); + } - pkt->set_payload(fua, SrsRtspPacketPayloadTypeFUA); pkt->wrap(msg); nb_left -= packet_size; @@ -1342,11 +1381,19 @@ srs_error_t SrsRtcRtpBuilder::package_fu_a(SrsSharedPtrMessage* msg, SrsSample* { srs_error_t err = srs_success; - char* p = sample->bytes + 1; - int nb_left = sample->size - 1; - uint8_t header = sample->bytes[0]; - uint8_t nal_type = header & kNalTypeMask; + SrsFormat* format = meta->vsh_format(); + if (!format || !format->vcodec) { + return err; + } + bool is_hevc = format->vcodec->id == SrsVideoCodecIdHEVC; + // H264 FU-A header size is 1 @see: https://datatracker.ietf.org/doc/html/rfc6184#section-5.8 + // H265 FU-A header size is 2 @see: https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.3 + int header_size = is_hevc ? 2 : 1; + char* p = sample->bytes + header_size; + int nb_left = sample->size - header_size; + uint8_t header = sample->bytes[0]; + int num_of_packet = 1 + (nb_left - 1) / fu_payload_size; for (int i = 0; i < num_of_packet; ++i) { int packet_size = srs_min(nb_left, fu_payload_size); @@ -1359,17 +1406,34 @@ srs_error_t SrsRtcRtpBuilder::package_fu_a(SrsSharedPtrMessage* msg, SrsSample* pkt->frame_type = SrsFrameTypeVideo; pkt->header.set_sequence(video_sequence++); pkt->header.set_timestamp(msg->timestamp * 90); + pkt->nalu_type = is_hevc ? kFuHevc : kFuA; - SrsRtpFUAPayload2* fua = new SrsRtpFUAPayload2(); - pkt->set_payload(fua, SrsRtspPacketPayloadTypeFUA2); + if (is_hevc) { + uint8_t nal_type = SrsHevcNaluTypeParse(header); + // H265 FU-A header + SrsRtpFUAPayloadHevc2* fua = new SrsRtpFUAPayloadHevc2(); + pkt->set_payload(fua, SrsRtspPacketPayloadTypeFUAHevc); - fua->nri = (SrsAvcNaluType)header; - fua->nalu_type = (SrsAvcNaluType)nal_type; - fua->start = bool(i == 0); - fua->end = bool(i == num_of_packet - 1); + fua->nalu_type = (SrsHevcNaluType)nal_type; + fua->start = bool(i == 0); + fua->end = bool(i == num_of_packet - 1); - fua->payload = p; - fua->size = packet_size; + fua->payload = p; + fua->size = packet_size; + } else { + uint8_t nal_type = header & kNalTypeMask; + // H264 FU-A header + SrsRtpFUAPayload2* fua = new SrsRtpFUAPayload2(); + pkt->set_payload(fua, SrsRtspPacketPayloadTypeFUA2); + + fua->nri = (SrsAvcNaluType)header; + fua->nalu_type = (SrsAvcNaluType)nal_type; + fua->start = bool(i == 0); + fua->end = bool(i == num_of_packet - 1); + + fua->payload = p; + fua->size = packet_size; + } pkt->wrap(msg); @@ -1568,6 +1632,7 @@ srs_error_t SrsRtcFrameBuilder::packet_video(SrsRtpPacket* src) SrsRtpPacket* pkt = src->copy(); if (pkt->is_keyframe()) { + // TODO: 处理H265 return packet_video_key_frame(pkt); } @@ -2042,6 +2107,7 @@ SrsVideoPayload* SrsVideoPayload::copy() cp->sample_ = sample_; cp->rtcp_fbs_ = rtcp_fbs_; cp->h264_param_ = h264_param_; + cp->h265_param_ = h265_param_; return cp; } @@ -2070,6 +2136,33 @@ SrsMediaPayloadType SrsVideoPayload::generate_media_payload_type() return media_payload_type; } +SrsMediaPayloadType SrsVideoPayload::generate_media_payload_type_h265() +{ + SrsMediaPayloadType media_payload_type(pt_); + + media_payload_type.encoding_name_ = name_; + media_payload_type.clock_rate_ = sample_; + media_payload_type.rtcp_fb_ = rtcp_fbs_; + + std::ostringstream format_specific_param; + if (!h265_param_.level_id.empty()) { + format_specific_param << "level-id=" << h265_param_.level_id; + } + if (!h265_param_.profile_id.empty()) { + format_specific_param << ";profile-id=" << h265_param_.profile_id; + } + if (!h265_param_.tier_flag.empty()) { + format_specific_param << ";tier-flag=" << h265_param_.tier_flag; + } + if (!h265_param_.tx_mode.empty()) { + format_specific_param << ";tx-mode=" << h265_param_.tx_mode; + } + + media_payload_type.format_specific_param_ = format_specific_param.str(); + + return media_payload_type; +} + srs_error_t SrsVideoPayload::set_h264_param_desc(std::string fmtp) { srs_error_t err = srs_success; @@ -2107,6 +2200,31 @@ srs_error_t SrsVideoPayload::set_h264_param_desc(std::string fmtp) return err; } +// level-id=180;profile-id=1;tier-flag=0;tx-mode=SRST +srs_error_t SrsVideoPayload::set_h265_param_desc(std::string fmtp) +{ + std::vector attributes = split_str(fmtp, ";"); + for (size_t i = 0; i < attributes.size(); ++i) { + std::string attribute = attributes.at(i); + std::vector kv = split_str(attribute, "="); + if (kv.size() != 2) { + return srs_error_new(ERROR_RTC_SDP_DECODE, "invalid h265 param=%s", attribute.c_str()); + } + if (kv[0] == "level-id") { + h265_param_.level_id = kv[1]; + } else if (kv[0] == "profile-id") { + h265_param_.profile_id = kv[1]; + } else if (kv[0] == "tier-flag") { + h265_param_.tier_flag = kv[1]; + } else if (kv[0] == "tx-mode") { + h265_param_.tx_mode = kv[1]; + } else { + return srs_error_new(ERROR_RTC_SDP_DECODE, "invalid h265 param=%s", kv[0].c_str()); + } + } + return srs_success; +} + SrsAudioPayload::SrsAudioPayload() { channel_ = 0; @@ -2699,7 +2817,7 @@ void SrsRtcVideoRecvTrack::on_before_decode_payload(SrsRtpPacket* pkt, SrsBuffer } uint8_t v = (uint8_t)(buf->head()[0] & kNalTypeMask); - pkt->nalu_type = SrsAvcNaluType(v); + pkt->nalu_type = v; if (v == kStapA) { *ppayload = new SrsRtpSTAPPayload(); diff --git a/trunk/src/app/srs_app_rtc_source.hpp b/trunk/src/app/srs_app_rtc_source.hpp index 7c813a97a3..2272c0a977 100644 --- a/trunk/src/app/srs_app_rtc_source.hpp +++ b/trunk/src/app/srs_app_rtc_source.hpp @@ -394,6 +394,7 @@ class SrsVideoPayload : public SrsCodecPayload { public: H264SpecificParam h264_param_; + H265SpecificParam h265_param_; public: SrsVideoPayload(); @@ -402,8 +403,10 @@ class SrsVideoPayload : public SrsCodecPayload public: virtual SrsVideoPayload* copy(); virtual SrsMediaPayloadType generate_media_payload_type(); + virtual SrsMediaPayloadType generate_media_payload_type_h265(); public: srs_error_t set_h264_param_desc(std::string fmtp); + srs_error_t set_h265_param_desc(std::string fmtp); }; // TODO: FIXME: Rename it. diff --git a/trunk/src/app/srs_app_stream_bridge.cpp b/trunk/src/app/srs_app_stream_bridge.cpp index de2915d924..b10b08b2fe 100644 --- a/trunk/src/app/srs_app_stream_bridge.cpp +++ b/trunk/src/app/srs_app_stream_bridge.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include using namespace std; @@ -85,7 +86,7 @@ SrsFrameToRtcBridge::SrsFrameToRtcBridge(SrsSharedPtr source) // video track ssrc if (true) { - std::vector descs = source->get_track_desc("video", "H264"); + std::vector descs = source->get_track_desc("video", ""); if (!descs.empty()) { video_ssrc = descs.at(0)->ssrc_; } @@ -144,7 +145,7 @@ void SrsFrameToRtcBridge::on_unpublish() srs_error_t SrsFrameToRtcBridge::on_frame(SrsSharedPtrMessage* frame) { -#ifdef SRS_FFMPEG_FIT +#ifdef SRS_FFMPEG_FIT return rtp_builder_->on_frame(frame); #else return srs_success; @@ -155,6 +156,26 @@ srs_error_t SrsFrameToRtcBridge::on_rtp(SrsRtpPacket* pkt) { return source_->on_rtp(pkt); } + +srs_error_t SrsFrameToRtcBridge::update_codec(SrsVideoCodecId id) +{ + // init with H264 default, so we need check if it's H265 only. + if (id == SrsVideoCodecIdHEVC) { + if (source_->get_track_desc("video", "H265").empty()) { + std::vector video_track_descs = source_->get_track_desc("video", "H264"); + if (!video_track_descs.empty()) { + SrsRtcTrackDescription* video_track_desc = video_track_descs.at(0); + SrsVideoPayload* video_payload = (SrsVideoPayload*)video_track_desc->media_; + video_payload->name_ = "H265"; + video_payload->set_h265_param_desc("level-id=180;profile-id=1;tier-flag=0;tx-mode=SRST"); + srs_trace("RTC: Switch video codec %d(%s) to %d(%s)", SrsVideoCodecIdAVC, srs_video_codec_id2str(SrsVideoCodecIdAVC).c_str(), + id, srs_video_codec_id2str(id).c_str()); + } + } + } + return srs_success; +} + #endif SrsCompositeBridge::SrsCompositeBridge() diff --git a/trunk/src/app/srs_app_stream_bridge.hpp b/trunk/src/app/srs_app_stream_bridge.hpp index dfa4857c86..aad045823b 100644 --- a/trunk/src/app/srs_app_stream_bridge.hpp +++ b/trunk/src/app/srs_app_stream_bridge.hpp @@ -74,6 +74,7 @@ class SrsFrameToRtcBridge : public ISrsStreamBridge virtual void on_unpublish(); virtual srs_error_t on_frame(SrsSharedPtrMessage* frame); srs_error_t on_rtp(SrsRtpPacket* pkt); + srs_error_t update_codec(SrsVideoCodecId id); }; #endif diff --git a/trunk/src/kernel/srs_kernel_codec.cpp b/trunk/src/kernel/srs_kernel_codec.cpp index ba02573824..4dfc55bf7d 100644 --- a/trunk/src/kernel/srs_kernel_codec.cpp +++ b/trunk/src/kernel/srs_kernel_codec.cpp @@ -703,7 +703,7 @@ srs_error_t SrsVideoFrame::parse_avc_nalu_type(const SrsSample* sample, SrsAvcNa srs_error_t err = srs_success; if (sample == NULL || sample->size < 1) { - return srs_error_new(ERROR_AVC_NALU_EMPTY, "empty nalu"); + return srs_error_new(ERROR_NALU_EMPTY, "empty nalu"); } uint8_t header = sample->bytes[0]; @@ -716,10 +716,6 @@ srs_error_t SrsVideoFrame::parse_avc_b_frame(const SrsSample* sample, bool& is_b { srs_error_t err = srs_success; - if (sample == NULL || sample->size < 1) { - return srs_error_new(ERROR_AVC_NALU_EMPTY, "empty nalu"); - } - SrsAvcNaluType nalu_type; if ((err = parse_avc_nalu_type(sample, nalu_type)) != srs_success) { return srs_error_wrap(err, "parse avc nalu type error"); @@ -755,6 +751,84 @@ srs_error_t SrsVideoFrame::parse_avc_b_frame(const SrsSample* sample, bool& is_b return err; } +srs_error_t SrsVideoFrame::parse_hevc_nalu_type(const SrsSample *sample, SrsHevcNaluType &hevc_nalu_type) +{ + srs_error_t err = srs_success; + + if (sample == NULL || sample->size < 1) { + return srs_error_new(ERROR_NALU_EMPTY, "empty nalu"); + } + + uint8_t header = sample->bytes[0]; + hevc_nalu_type = (SrsHevcNaluType)((header >> 1) & 0x3f); + + return err; +} + +srs_error_t SrsVideoFrame::parse_hevc_b_frame(const SrsSample *sample, SrsFormat *format, bool &is_b_frame) +{ + srs_error_t err = srs_success; + + SrsHevcNaluType nalu_type; + if ((err = parse_hevc_nalu_type(sample, nalu_type)) != srs_success) { + return srs_error_wrap(err, "parse hevc nalu type error"); + } + + SrsBuffer stream(sample->bytes, sample->size); + stream.skip(2); + + // @see 7.3.6.1 General slice segment header syntax + // @doc ITU-T-H.265-2021.pdf, page 66. + SrsBitBuffer bs(&stream); + uint8_t first_slice_segment_in_pic_flag = bs.read_bit(); + if (nalu_type > SrsHevcNaluType_CODED_SLICE_BLA && nalu_type < SrsHevcNaluType_RESERVED_23) { + bs.skip_bits(1); + is_b_frame = false; + return err; + } + + uint32_t slice_pic_parameter_set_id; + if ((err = bs.read_bits_ue(slice_pic_parameter_set_id)) != srs_success) { + return srs_error_wrap(err, "read slice pic parameter set id"); + } + + if (slice_pic_parameter_set_id >= SrsHevcMax_PPS_COUNT) { + return srs_error_new(ERROR_HEVC_DECODE_ERROR, "slice pic parameter set id out of range: %d", slice_pic_parameter_set_id); + } + + SrsHevcRbspPps *pps = &(format->vcodec->hevc_dec_conf_record_.pps_table[slice_pic_parameter_set_id]); + + uint8_t dependent_slice_segment_flag; + if (!first_slice_segment_in_pic_flag) { + if (pps->dependent_slice_segments_enabled_flag) { + dependent_slice_segment_flag = bs.read_bit(); + } else { + dependent_slice_segment_flag = 0; + } + } else { + dependent_slice_segment_flag = 0; + } + + if (dependent_slice_segment_flag) { + return srs_error_new(ERROR_HEVC_DECODE_ERROR, "dependent slice segment flag is not supported"); + } + + for (int i = 0; i < pps->num_extra_slice_header_bits; i++) { + bs.skip_bits(1); + } + + uint32_t slice_type; + if ((err = bs.read_bits_ue(slice_type)) != srs_success) { + return srs_error_wrap(err, "read slice type"); + } + + is_b_frame = (slice_type == SrsHevcSliceTypeB) ? true : false; + + // no need to evaluate the rest + + return err; +} + SrsFormat::SrsFormat() { acodec = NULL; diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp index 9899271b6e..8a17586674 100644 --- a/trunk/src/kernel/srs_kernel_codec.hpp +++ b/trunk/src/kernel/srs_kernel_codec.hpp @@ -14,6 +14,7 @@ class SrsBuffer; class SrsBitBuffer; +class SrsFormat; /** * The video codec id. @@ -496,8 +497,15 @@ enum SrsHevcNaluType { SrsHevcNaluType_UNSPECIFIED_63, SrsHevcNaluType_INVALID, }; +// @see https://datatracker.ietf.org/doc/html/rfc7798#section-1.1.4 #define SrsHevcNaluTypeParse(code) (SrsHevcNaluType)((code & 0x7E) >> 1) +enum SrsHevcSliceType { + SrsHevcSliceTypeB = 0, + SrsHevcSliceTypeP = 1, + SrsHevcSliceTypeI = 2, +}; + struct SrsHevcNalData { uint16_t nal_unit_length; std::vector nal_unit_data; @@ -1321,6 +1329,9 @@ class SrsVideoFrame : public SrsFrame public: static srs_error_t parse_avc_nalu_type(const SrsSample* sample, SrsAvcNaluType& avc_nalu_type); static srs_error_t parse_avc_b_frame(const SrsSample* sample, bool& is_b_frame); + + static srs_error_t parse_hevc_nalu_type(const SrsSample* sample, SrsHevcNaluType& hevc_nalu_type); + static srs_error_t parse_hevc_b_frame(const SrsSample* sample, SrsFormat* format, bool& is_b_frame); }; /** diff --git a/trunk/src/kernel/srs_kernel_error.hpp b/trunk/src/kernel/srs_kernel_error.hpp index 69a14a5907..afc7f904f1 100644 --- a/trunk/src/kernel/srs_kernel_error.hpp +++ b/trunk/src/kernel/srs_kernel_error.hpp @@ -279,7 +279,7 @@ XX(ERROR_HEVC_DECODE_ERROR , 3099, "HevcDecode", "HEVC decode av stream failed") \ XX(ERROR_MP4_HVCC_CHANGE , 3100, "Mp4HvcCChange", "MP4 does not support video HvcC change") \ XX(ERROR_HEVC_API_NO_PREFIXED , 3101, "HevcAnnexbPrefix", "No annexb prefix for HEVC decoder") \ - XX(ERROR_AVC_NALU_EMPTY , 3102, "AvcNaluEmpty", "AVC NALU is empty") + XX(ERROR_NALU_EMPTY , 3102, "NaluEmpty", "NALU is empty") /**************************************************/ /* HTTP/StreamConverter protocol error. */ diff --git a/trunk/src/kernel/srs_kernel_rtc_rtp.cpp b/trunk/src/kernel/srs_kernel_rtc_rtp.cpp index f5d5978131..5bcb3bbd54 100644 --- a/trunk/src/kernel/srs_kernel_rtc_rtp.cpp +++ b/trunk/src/kernel/srs_kernel_rtc_rtp.cpp @@ -758,7 +758,7 @@ SrsRtpPacket::SrsRtpPacket() shared_buffer_ = NULL; actual_buffer_size_ = 0; - nalu_type = SrsAvcNaluTypeReserved; + nalu_type = 0; frame_type = SrsFrameTypeReserved; cached_payload_size = 0; decode_handler = NULL; @@ -961,6 +961,23 @@ bool SrsRtpPacket::is_keyframe() if((SrsAvcNaluTypeIDR == nalu_type) || (SrsAvcNaluTypeSPS == nalu_type) || (SrsAvcNaluTypePPS == nalu_type)) { return true; } +#ifdef SRS_H265 + if(nalu_type == kStapHevc) { + SrsRtpSTAPPayloadHevc* stap_payload = dynamic_cast(payload_); + if(NULL != stap_payload->get_vps() || NULL != stap_payload->get_sps() || NULL != stap_payload->get_pps()) { + return true; + } + } else if(nalu_type == kFuHevc) { + SrsRtpFUAPayloadHevc2* fua_payload = dynamic_cast(payload_); + if(fua_payload->nalu_type >= SrsHevcNaluType_CODED_SLICE_BLA && fua_payload->nalu_type <= SrsHevcNaluType_RESERVED_23) { + return true; + } + } else { + if((SrsHevcNaluType_VPS == nalu_type) || (SrsHevcNaluType_SPS == nalu_type) || (SrsHevcNaluType_PPS == nalu_type)) { + return true; + } + } +#endif } return false; @@ -1061,10 +1078,10 @@ void SrsRtpRawNALUs::push_back(SrsSample* sample) nalus.push_back(sample); } -uint8_t SrsRtpRawNALUs::skip_first_byte() +uint8_t SrsRtpRawNALUs::skip_bytes(int count) { - srs_assert (cursor >= 0 && nn_bytes > 0 && cursor < nn_bytes); - cursor++; + srs_assert (cursor >= 0 && nn_bytes > 0 && cursor + count < nn_bytes); + cursor += count; return uint8_t(nalus[0]->bytes[0]); } @@ -1522,3 +1539,361 @@ ISrsRtpPayloader* SrsRtpFUAPayload2::copy() return cp; } + +SrsRtpSTAPPayloadHevc::SrsRtpSTAPPayloadHevc() +{ + ++_srs_pps_objs_rothers->sugar; +} + +SrsRtpSTAPPayloadHevc::~SrsRtpSTAPPayloadHevc() +{ + int nn_nalus = (int)nalus.size(); + for (int i = 0; i < nn_nalus; i++) { + SrsSample* p = nalus[i]; + srs_freep(p); + } +} + +SrsSample* SrsRtpSTAPPayloadHevc::get_vps() +{ + int nn_nalus = (int)nalus.size(); + for (int i = 0; i < nn_nalus; i++) { + SrsSample* p = nalus[i]; + if (!p || !p->size) { + continue; + } + + SrsHevcNaluType nalu_type = SrsHevcNaluTypeParse(p->bytes[0]); + if (nalu_type == SrsHevcNaluType_VPS) { + return p; + } + } + + return NULL; +} + +SrsSample* SrsRtpSTAPPayloadHevc::get_sps() +{ + int nn_nalus = (int)nalus.size(); + for (int i = 0; i < nn_nalus; i++) { + SrsSample* p = nalus[i]; + if (!p || !p->size) { + continue; + } + + SrsHevcNaluType nalu_type = SrsHevcNaluTypeParse(p->bytes[0]); + if (nalu_type == SrsHevcNaluType_SPS) { + return p; + } + } + + return NULL; +} + +SrsSample* SrsRtpSTAPPayloadHevc::get_pps() +{ + int nn_nalus = (int)nalus.size(); + for (int i = 0; i < nn_nalus; i++) { + SrsSample* p = nalus[i]; + if (!p || !p->size) { + continue; + } + + SrsHevcNaluType nalu_type = SrsHevcNaluTypeParse(p->bytes[0]); + if (nalu_type == SrsHevcNaluType_PPS) { + return p; + } + } + + return NULL; +} + +uint64_t SrsRtpSTAPPayloadHevc::nb_bytes() +{ + int size = 2; + + int nn_nalus = (int)nalus.size(); + for (int i = 0; i < nn_nalus; i++) { + SrsSample* p = nalus[i]; + size += 2 + p->size; + } + + return size; +} + +srs_error_t SrsRtpSTAPPayloadHevc::encode(SrsBuffer* buf) +{ + if (!buf->require(2)) { + return srs_error_new(ERROR_RTC_RTP_MUXER, "requires %d bytes", 2); + } + + // STAP header, RTP payload format for aggregation packets + // @see https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.2 + buf->write_1bytes(kStapHevc << 1); + buf->write_1bytes(1); + + // NALUs. + int nn_nalus = (int)nalus.size(); + for (int i = 0; i < nn_nalus; i++) { + SrsSample* p = nalus[i]; + + if (!buf->require(2 + p->size)) { + return srs_error_new(ERROR_RTC_RTP_MUXER, "requires %d bytes", 2 + p->size); + } + + buf->write_2bytes(p->size); + buf->write_bytes(p->bytes, p->size); + } + + return srs_success; +} + +srs_error_t SrsRtpSTAPPayloadHevc::decode(SrsBuffer* buf) +{ + if (!buf->require(2)) { + return srs_error_new(ERROR_RTC_RTP_MUXER, "requires %d bytes", 2); + } + + // STAP header, RTP payload format for aggregation packets + // @see https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.2 + uint8_t v = buf->read_1bytes(); + + // forbidden_zero_bit shoul be zero. + // @see https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.2 + uint8_t f = (v & 0x80); + if (f == 0x80) { + return srs_error_new(ERROR_RTC_RTP_MUXER, "forbidden_zero_bit should be zero"); + } + + // NALUs. + while (!buf->empty()) { + if (!buf->require(2)) { + return srs_error_new(ERROR_RTC_RTP_MUXER, "requires %d bytes", 2); + } + + int size = buf->read_2bytes(); + if (!buf->require(size)) { + return srs_error_new(ERROR_RTC_RTP_MUXER, "requires %d bytes", size); + } + + SrsSample* sample = new SrsSample(); + sample->bytes = buf->head(); + sample->size = size; + buf->skip(size); + + nalus.push_back(sample); + } + + return srs_success; +} + +ISrsRtpPayloader* SrsRtpSTAPPayloadHevc::copy() +{ + SrsRtpSTAPPayloadHevc* cp = new SrsRtpSTAPPayloadHevc(); + + int nn_nalus = (int)nalus.size(); + for (int i = 0; i < nn_nalus; i++) { + SrsSample* p = nalus[i]; + cp->nalus.push_back(p->copy()); + } + + return cp; +} + +SrsRtpFUAPayloadHevc::SrsRtpFUAPayloadHevc() +{ + start = end = false; + nalu_type = (SrsHevcNaluType)0; + + ++_srs_pps_objs_rothers->sugar; +} + +SrsRtpFUAPayloadHevc::~SrsRtpFUAPayloadHevc() +{ + int nn_nalus = (int)nalus.size(); + for (int i = 0; i < nn_nalus; i++) { + SrsSample* p = nalus[i]; + srs_freep(p); + } +} + +uint64_t SrsRtpFUAPayloadHevc::nb_bytes() +{ + int size = 3; + + int nn_nalus = (int)nalus.size(); + for (int i = 0; i < nn_nalus; i++) { + SrsSample* p = nalus[i]; + size += p->size; + } + + return size; +} + +srs_error_t SrsRtpFUAPayloadHevc::encode(SrsBuffer* buf) +{ + if (!buf->require(3)) { + return srs_error_new(ERROR_RTC_RTP_MUXER, "requires %d bytes", 3); + } + + // PayloadHdr, @see: https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.3 + buf->write_1bytes(kFuHevc << 1); + buf->write_1bytes(1); + + // FU header, @see https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.3 + uint8_t fu_header = (start ? kStart : 0) | (end ? kEnd : 0); + fu_header |= nalu_type; + buf->write_1bytes(fu_header); + + int nn_nalus = (int)nalus.size(); + for (int i = 0; i < nn_nalus; i++) { + SrsSample* p = nalus[i]; + + if (!buf->require(p->size)) { + return srs_error_new(ERROR_RTC_RTP_MUXER, "requires %d bytes", p->size); + } + + buf->write_bytes(p->bytes, p->size); + } + + return srs_success; +} + +srs_error_t SrsRtpFUAPayloadHevc::decode(SrsBuffer* buf) +{ + if (!buf->require(3)) { + return srs_error_new(ERROR_RTC_RTP_MUXER, "requires %d bytes", 3); + } + + uint8_t payload_hdr1 = buf->read_1bytes(); + uint8_t payload_hdr2 = buf->read_1bytes(); + + uint8_t fu_header = buf->read_1bytes(); + start = fu_header & kStart; + end = fu_header & kEnd; + nalu_type = SrsHevcNaluType(fu_header & 0x3F); + if (!buf->require(1)) { + return srs_error_new(ERROR_RTC_RTP_MUXER, "requires %d bytes", 1); + } + + SrsSample* sample = new SrsSample(); + sample->bytes = buf->head(); + sample->size = buf->left(); + buf->skip(sample->size); + + nalus.push_back(sample); + + return srs_success; +} + +ISrsRtpPayloader* SrsRtpFUAPayloadHevc::copy() +{ + SrsRtpFUAPayloadHevc* cp = new SrsRtpFUAPayloadHevc(); + + cp->start = start; + cp->end = end; + cp->nalu_type = nalu_type; + + int nn_nalus = (int)nalus.size(); + for (int i = 0; i < nn_nalus; i++) { + SrsSample* p = nalus[i]; + cp->nalus.push_back(p->copy()); + } + + return cp; +} + +SrsRtpFUAPayloadHevc2::SrsRtpFUAPayloadHevc2() +{ + start = end = false; + nalu_type = (SrsHevcNaluType)0; + + payload = NULL; + size = 0; + + ++_srs_pps_objs_rfua->sugar; +} + +SrsRtpFUAPayloadHevc2::~SrsRtpFUAPayloadHevc2() +{ +} + +uint64_t SrsRtpFUAPayloadHevc2::nb_bytes() +{ + // PayloadHdr(2) + FU header(1) + return 3 + size; +} + +srs_error_t SrsRtpFUAPayloadHevc2::encode(SrsBuffer* buf) +{ + if (!buf->require(3 + size)) { + return srs_error_new(ERROR_RTC_RTP_MUXER, "requires %d bytes", 3 + size); + } + + // Fast encoding. + char* p = buf->head(); + + // PayloadHdr, @see: https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.3 + /* + * create the HEVC payload header and transmit the buffer as fragmentation units (FU) + * + * 0 1 + * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * |F| Type | LayerId | TID | + * +-------------+-----------------+ + * + * F = 0 + * Type = 49 (fragmentation unit (FU)) + * LayerId = 0 + * TID = 1 + */ + *p++ = kFuHevc << 1; + *p++ = 1; + + // FU header, @see https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.3 + uint8_t fu_header = (start ? kStart : 0) | (end ? kEnd : 0); + fu_header |= nalu_type; + *p++ = fu_header; + + memcpy(p, payload, size); + + // Consume bytes. + buf->skip(3 + size); + + return srs_success; +} + +srs_error_t SrsRtpFUAPayloadHevc2::decode(SrsBuffer* buf) +{ + if (!buf->require(3)) { + return srs_error_new(ERROR_RTC_RTP_MUXER, "requires 3 bytes"); + } + + uint8_t payload_hdr1 = buf->read_1bytes(); + uint8_t payload_hdr2 = buf->read_1bytes(); + + uint8_t fu_header = buf->read_1bytes(); + start = fu_header & kStart; + end = fu_header & kEnd; + nalu_type = SrsHevcNaluType(fu_header & 0x3F); + + payload = buf->head(); + size = buf->left(); + buf->skip(size); + + return srs_success; +} + +ISrsRtpPayloader* SrsRtpFUAPayloadHevc2::copy() +{ + SrsRtpFUAPayloadHevc2* cp = new SrsRtpFUAPayloadHevc2(); + + cp->start = start; + cp->end = end; + cp->nalu_type = nalu_type; + cp->payload = payload; + cp->size = size; + + return cp; +} diff --git a/trunk/src/kernel/srs_kernel_rtc_rtp.hpp b/trunk/src/kernel/srs_kernel_rtc_rtp.hpp index b417208fbc..7f84dc5eea 100644 --- a/trunk/src/kernel/srs_kernel_rtc_rtp.hpp +++ b/trunk/src/kernel/srs_kernel_rtc_rtp.hpp @@ -29,10 +29,14 @@ const uint8_t kNalTypeMask = 0x1F; // @see: https://tools.ietf.org/html/rfc6184#section-5.2 const uint8_t kStapA = 24; - // @see: https://tools.ietf.org/html/rfc6184#section-5.2 const uint8_t kFuA = 28; +// @see: https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.2 +const uint8_t kStapHevc = 48; +// @see: https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.3 +const uint8_t kFuHevc = 49; + // @see: https://tools.ietf.org/html/rfc6184#section-5.8 const uint8_t kStart = 0x80; // Fu-header start bit const uint8_t kEnd = 0x40; // Fu-header end bit @@ -254,8 +258,10 @@ enum SrsRtspPacketPayloadType SrsRtspPacketPayloadTypeRaw, SrsRtspPacketPayloadTypeFUA2, SrsRtspPacketPayloadTypeFUA, + SrsRtspPacketPayloadTypeFUAHevc, SrsRtspPacketPayloadTypeNALU, SrsRtspPacketPayloadTypeSTAP, + SrsRtspPacketPayloadTypeSTAPHevc, SrsRtspPacketPayloadTypeUnknown, }; @@ -289,7 +295,7 @@ class SrsRtpPacket // Helper fields. public: // The first byte as nalu type, for video decoder only. - SrsAvcNaluType nalu_type; + uint8_t nalu_type; // The frame type, for RTMP bridge or SFU source. SrsFrameType frame_type; // Fast cache for performance. @@ -376,7 +382,7 @@ class SrsRtpRawNALUs : public ISrsRtpPayloader public: void push_back(SrsSample* sample); public: - uint8_t skip_first_byte(); + uint8_t skip_bytes(int count); // We will manage the returned samples, if user want to manage it, please copy it. srs_error_t read_samples(std::vector& samples, int packet_size); // interface ISrsRtpPayloader @@ -460,4 +466,68 @@ class SrsRtpFUAPayload2 : public ISrsRtpPayloader virtual ISrsRtpPayloader* copy(); }; +class SrsRtpSTAPPayloadHevc : public ISrsRtpPayloader +{ +public: + // The NALU samples, we will manage the samples. + // @remark We only refer to the memory, user must free its bytes. + std::vector nalus; +public: + SrsRtpSTAPPayloadHevc(); + virtual ~SrsRtpSTAPPayloadHevc(); +public: + SrsSample* get_vps(); + SrsSample* get_sps(); + SrsSample* get_pps(); +// interface ISrsRtpPayloader +public: + virtual uint64_t nb_bytes(); + virtual srs_error_t encode(SrsBuffer* buf); + virtual srs_error_t decode(SrsBuffer* buf); + virtual ISrsRtpPayloader* copy(); +}; + +// FU, for one NALU with multiple fragments. +// With more than one payload. For HEVC. +class SrsRtpFUAPayloadHevc : public ISrsRtpPayloader +{ +public: + // The FUA header. + bool start; + bool end; + SrsHevcNaluType nalu_type; + // The NALU samples, we manage the samples. + // @remark We only refer to the memory, user must free its bytes. + std::vector nalus; +public: + SrsRtpFUAPayloadHevc(); + virtual ~SrsRtpFUAPayloadHevc(); +// interface ISrsRtpPayloader +public: + virtual uint64_t nb_bytes(); + virtual srs_error_t encode(SrsBuffer* buf); + virtual srs_error_t decode(SrsBuffer* buf); + virtual ISrsRtpPayloader* copy(); +}; + +// FU, for one NALU with multiple fragments. +// With only one payload. For HEVC. +class SrsRtpFUAPayloadHevc2 : public ISrsRtpPayloader +{ +public: + bool start; + bool end; + SrsHevcNaluType nalu_type; + char* payload; + int size; +public: + SrsRtpFUAPayloadHevc2(); + virtual ~SrsRtpFUAPayloadHevc2(); +public: + virtual uint64_t nb_bytes(); + virtual srs_error_t encode(SrsBuffer* buf); + virtual srs_error_t decode(SrsBuffer* buf); + virtual ISrsRtpPayloader* copy(); +}; + #endif From b3c508534d7a543331eea7f534928f796717f1e6 Mon Sep 17 00:00:00 2001 From: Haibo Chen <495810242@qq.com> Date: Wed, 19 Feb 2025 15:40:05 +0800 Subject: [PATCH 02/17] remove unused code --- trunk/src/kernel/srs_kernel_rtc_rtp.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/trunk/src/kernel/srs_kernel_rtc_rtp.cpp b/trunk/src/kernel/srs_kernel_rtc_rtp.cpp index 5bcb3bbd54..b547f233e5 100644 --- a/trunk/src/kernel/srs_kernel_rtc_rtp.cpp +++ b/trunk/src/kernel/srs_kernel_rtc_rtp.cpp @@ -1765,8 +1765,8 @@ srs_error_t SrsRtpFUAPayloadHevc::decode(SrsBuffer* buf) return srs_error_new(ERROR_RTC_RTP_MUXER, "requires %d bytes", 3); } - uint8_t payload_hdr1 = buf->read_1bytes(); - uint8_t payload_hdr2 = buf->read_1bytes(); + // skip PayloadHdr, 2 bytes + buf->skip(2); uint8_t fu_header = buf->read_1bytes(); start = fu_header & kStart; @@ -1870,8 +1870,8 @@ srs_error_t SrsRtpFUAPayloadHevc2::decode(SrsBuffer* buf) return srs_error_new(ERROR_RTC_RTP_MUXER, "requires 3 bytes"); } - uint8_t payload_hdr1 = buf->read_1bytes(); - uint8_t payload_hdr2 = buf->read_1bytes(); + // skip PayloadHdr, 2 bytes + buf->skip(2); uint8_t fu_header = buf->read_1bytes(); start = fu_header & kStart; From 83d577a49a0d6dabcb8c3236915150612f878396 Mon Sep 17 00:00:00 2001 From: Haibo Chen <495810242@qq.com> Date: Wed, 19 Feb 2025 16:33:24 +0800 Subject: [PATCH 03/17] define SrsAvcNaluTypeParse --- trunk/src/app/srs_app_rtc_source.cpp | 10 ++++------ trunk/src/kernel/srs_kernel_codec.cpp | 4 ++-- trunk/src/kernel/srs_kernel_codec.hpp | 2 ++ trunk/src/kernel/srs_kernel_rtc_rtp.cpp | 8 ++++---- trunk/src/utest/srs_utest_rtc.cpp | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/trunk/src/app/srs_app_rtc_source.cpp b/trunk/src/app/srs_app_rtc_source.cpp index 4d1d6390a3..ebfd088cbe 100644 --- a/trunk/src/app/srs_app_rtc_source.cpp +++ b/trunk/src/app/srs_app_rtc_source.cpp @@ -1256,7 +1256,7 @@ srs_error_t SrsRtcRtpBuilder::package_nalus(SrsSharedPtrMessage* msg, const vect } if (first_nalu_type == SrsAvcNaluTypeReserved) { - first_nalu_type = SrsAvcNaluType((uint8_t)(sample->bytes[0] & kNalTypeMask)); + first_nalu_type = SrsAvcNaluTypeParse(sample->bytes[0]); } raw_raw->push_back(sample->copy()); @@ -1336,7 +1336,7 @@ srs_error_t SrsRtcRtpBuilder::package_nalus(SrsSharedPtrMessage* msg, const vect srs_freep(fua); return srs_error_wrap(err, "read samples %d bytes, left %d, total %d", packet_size, nb_left, nn_bytes); } - fua->nalu_type = (SrsAvcNaluType)(header & kNalTypeMask); + fua->nalu_type = SrsAvcNaluTypeParse(header); fua->start = bool(i == 0); fua->end = bool(i == num_of_packet - 1); @@ -1409,25 +1409,23 @@ srs_error_t SrsRtcRtpBuilder::package_fu_a(SrsSharedPtrMessage* msg, SrsSample* pkt->nalu_type = is_hevc ? kFuHevc : kFuA; if (is_hevc) { - uint8_t nal_type = SrsHevcNaluTypeParse(header); // H265 FU-A header SrsRtpFUAPayloadHevc2* fua = new SrsRtpFUAPayloadHevc2(); pkt->set_payload(fua, SrsRtspPacketPayloadTypeFUAHevc); - fua->nalu_type = (SrsHevcNaluType)nal_type; + fua->nalu_type = SrsHevcNaluTypeParse(header); fua->start = bool(i == 0); fua->end = bool(i == num_of_packet - 1); fua->payload = p; fua->size = packet_size; } else { - uint8_t nal_type = header & kNalTypeMask; // H264 FU-A header SrsRtpFUAPayload2* fua = new SrsRtpFUAPayload2(); pkt->set_payload(fua, SrsRtspPacketPayloadTypeFUA2); fua->nri = (SrsAvcNaluType)header; - fua->nalu_type = (SrsAvcNaluType)nal_type; + fua->nalu_type = SrsAvcNaluTypeParse(header); fua->start = bool(i == 0); fua->end = bool(i == num_of_packet - 1); diff --git a/trunk/src/kernel/srs_kernel_codec.cpp b/trunk/src/kernel/srs_kernel_codec.cpp index 4dfc55bf7d..38e68211a7 100644 --- a/trunk/src/kernel/srs_kernel_codec.cpp +++ b/trunk/src/kernel/srs_kernel_codec.cpp @@ -707,7 +707,7 @@ srs_error_t SrsVideoFrame::parse_avc_nalu_type(const SrsSample* sample, SrsAvcNa } uint8_t header = sample->bytes[0]; - avc_nalu_type = (SrsAvcNaluType)(header & kNalTypeMask); + avc_nalu_type = SrsAvcNaluTypeParse(header); return err; } @@ -760,7 +760,7 @@ srs_error_t SrsVideoFrame::parse_hevc_nalu_type(const SrsSample *sample, SrsHevc } uint8_t header = sample->bytes[0]; - hevc_nalu_type = (SrsHevcNaluType)((header >> 1) & 0x3f); + hevc_nalu_type = SrsHevcNaluTypeParse(header); return err; } diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp index 8a17586674..e461b07903 100644 --- a/trunk/src/kernel/srs_kernel_codec.hpp +++ b/trunk/src/kernel/srs_kernel_codec.hpp @@ -422,6 +422,8 @@ enum SrsAvcNaluType // Coded slice extension slice_layer_extension_rbsp( ) SrsAvcNaluTypeCodedSliceExt = 20, }; +// @see https://datatracker.ietf.org/doc/html/rfc6184#section-1.3 +#define SrsAvcNaluTypeParse(code) (SrsAvcNaluType)(code & 0x1F) std::string srs_avc_nalu2str(SrsAvcNaluType nalu_type); #ifdef SRS_H265 diff --git a/trunk/src/kernel/srs_kernel_rtc_rtp.cpp b/trunk/src/kernel/srs_kernel_rtc_rtp.cpp index b547f233e5..5ba2204aba 100644 --- a/trunk/src/kernel/srs_kernel_rtc_rtp.cpp +++ b/trunk/src/kernel/srs_kernel_rtc_rtp.cpp @@ -1208,7 +1208,7 @@ SrsSample* SrsRtpSTAPPayload::get_sps() continue; } - SrsAvcNaluType nalu_type = (SrsAvcNaluType)(p->bytes[0] & kNalTypeMask); + SrsAvcNaluType nalu_type = SrsAvcNaluTypeParse(p->bytes[0]); if (nalu_type == SrsAvcNaluTypeSPS) { return p; } @@ -1226,7 +1226,7 @@ SrsSample* SrsRtpSTAPPayload::get_pps() continue; } - SrsAvcNaluType nalu_type = (SrsAvcNaluType)(p->bytes[0] & kNalTypeMask); + SrsAvcNaluType nalu_type = SrsAvcNaluTypeParse(p->bytes[0]); if (nalu_type == SrsAvcNaluTypePPS) { return p; } @@ -1412,7 +1412,7 @@ srs_error_t SrsRtpFUAPayload::decode(SrsBuffer* buf) v = buf->read_1bytes(); start = v & kStart; end = v & kEnd; - nalu_type = SrsAvcNaluType(v & kNalTypeMask); + nalu_type = SrsAvcNaluTypeParse(v); if (!buf->require(1)) { return srs_error_new(ERROR_RTC_RTP_MUXER, "requires %d bytes", 1); @@ -1513,7 +1513,7 @@ srs_error_t SrsRtpFUAPayload2::decode(SrsBuffer* buf) v = buf->read_1bytes(); start = v & kStart; end = v & kEnd; - nalu_type = SrsAvcNaluType(v & kNalTypeMask); + nalu_type = SrsAvcNaluTypeParse(v); if (!buf->require(1)) { return srs_error_new(ERROR_RTC_RTP_MUXER, "requires %d bytes", 1); diff --git a/trunk/src/utest/srs_utest_rtc.cpp b/trunk/src/utest/srs_utest_rtc.cpp index b406790080..9603c512c8 100644 --- a/trunk/src/utest/srs_utest_rtc.cpp +++ b/trunk/src/utest/srs_utest_rtc.cpp @@ -62,7 +62,7 @@ VOID TEST(KernelRTCTest, RtpSTAPPayloadException) SrsAvcNaluType nalu_type = SrsAvcNaluTypeReserved; // Try to parse the NALU type for video decoder. if (!buf.empty()) { - nalu_type = SrsAvcNaluType((uint8_t)(buf.head()[0] & kNalTypeMask)); + nalu_type = SrsAvcNaluTypeParse(buf.head()[0]); } EXPECT_TRUE(nalu_type == kStapA); From 8953dbfba10f460bc918aa2d1bbf8a5f34f2d73f Mon Sep 17 00:00:00 2001 From: chundonglinlin Date: Fri, 21 Feb 2025 23:05:39 +0800 Subject: [PATCH 04/17] Optimize init h265 track and format. --- trunk/src/app/srs_app_rtc_sdp.cpp | 3 +- trunk/src/app/srs_app_rtc_source.cpp | 26 ++++++++++++----- trunk/src/app/srs_app_stream_bridge.cpp | 22 +------------- trunk/src/app/srs_app_stream_bridge.hpp | 1 - trunk/src/kernel/srs_kernel_codec.cpp | 38 ++++++++++++------------- trunk/src/kernel/srs_kernel_codec.hpp | 4 +++ trunk/src/kernel/srs_kernel_rtc_rtp.hpp | 8 +++--- 7 files changed, 49 insertions(+), 53 deletions(-) diff --git a/trunk/src/app/srs_app_rtc_sdp.cpp b/trunk/src/app/srs_app_rtc_sdp.cpp index 8f5e49a835..d87c5e0970 100644 --- a/trunk/src/app/srs_app_rtc_sdp.cpp +++ b/trunk/src/app/srs_app_rtc_sdp.cpp @@ -1252,4 +1252,5 @@ srs_error_t SrsSdp::update_msid(string id) } return err; -} \ No newline at end of file +} + diff --git a/trunk/src/app/srs_app_rtc_source.cpp b/trunk/src/app/srs_app_rtc_source.cpp index ebfd088cbe..676766e0bb 100644 --- a/trunk/src/app/srs_app_rtc_source.cpp +++ b/trunk/src/app/srs_app_rtc_source.cpp @@ -483,6 +483,23 @@ void SrsRtcSource::init_for_play_before_publishing() video_track_desc->media_ = video_payload; video_payload->set_h264_param_desc("level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f"); + +#ifdef SRS_H265 + // default h265 video track description + SrsRtcTrackDescription* h265_video_track_desc = new SrsRtcTrackDescription(); + stream_desc->video_track_descs_.push_back(h265_video_track_desc); + + h265_video_track_desc->type_ = "video"; + h265_video_track_desc->id_ = "video-" + srs_random_str(8); + h265_video_track_desc->ssrc_ = video_ssrc; + h265_video_track_desc->direction_ = "recvonly"; + + SrsVideoPayload* h265_video_payload = new SrsVideoPayload(kVideoPayloadType, "H265", kVideoSamplerate); + h265_video_track_desc->media_ = h265_video_payload; + + h265_video_payload->set_h265_param_desc("level-id=180;profile-id=1;tier-flag=0;tx-mode=SRST"); +#endif + } set_stream_desc(stream_desc.get()); @@ -1084,10 +1101,6 @@ srs_error_t SrsRtcRtpBuilder::on_video(SrsSharedPtrMessage* msg) if (has_idr) { SrsUniquePtr pkt(new SrsRtpPacket()); - if ((err = bridge_->update_codec(format->vcodec->id)) != srs_success) { - return srs_error_wrap(err, "update codec"); - } - if ((err = package_stap_a(msg, pkt.get())) != srs_success) { return srs_error_wrap(err, "package stap-a"); } @@ -1235,7 +1248,7 @@ srs_error_t SrsRtcRtpBuilder::package_stap_a(SrsSharedPtrMessage* msg, SrsRtpPac } memcpy(payload, (char*)param->data(), param->size()); - payload += (int)param->size(); + payload += (int)param->size(); } return err; @@ -1323,7 +1336,7 @@ srs_error_t SrsRtcRtpBuilder::package_nalus(SrsSharedPtrMessage* msg, const vect SrsRtpFUAPayloadHevc* fua = new SrsRtpFUAPayloadHevc(); if ((err = raw->read_samples(fua->nalus, packet_size)) != srs_success) { srs_freep(fua); - return srs_error_wrap(err, "read samples %d bytes, left %d, total %d", packet_size, nb_left, nn_bytes); + return srs_error_wrap(err, "read hevc samples %d bytes, left %d, total %d", packet_size, nb_left, nn_bytes); } fua->nalu_type = SrsHevcNaluTypeParse(header); fua->start = bool(i == 0); @@ -1630,7 +1643,6 @@ srs_error_t SrsRtcFrameBuilder::packet_video(SrsRtpPacket* src) SrsRtpPacket* pkt = src->copy(); if (pkt->is_keyframe()) { - // TODO: 处理H265 return packet_video_key_frame(pkt); } diff --git a/trunk/src/app/srs_app_stream_bridge.cpp b/trunk/src/app/srs_app_stream_bridge.cpp index b10b08b2fe..b80c49a690 100644 --- a/trunk/src/app/srs_app_stream_bridge.cpp +++ b/trunk/src/app/srs_app_stream_bridge.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include using namespace std; @@ -145,7 +144,7 @@ void SrsFrameToRtcBridge::on_unpublish() srs_error_t SrsFrameToRtcBridge::on_frame(SrsSharedPtrMessage* frame) { -#ifdef SRS_FFMPEG_FIT +#ifdef SRS_FFMPEG_FIT return rtp_builder_->on_frame(frame); #else return srs_success; @@ -157,25 +156,6 @@ srs_error_t SrsFrameToRtcBridge::on_rtp(SrsRtpPacket* pkt) return source_->on_rtp(pkt); } -srs_error_t SrsFrameToRtcBridge::update_codec(SrsVideoCodecId id) -{ - // init with H264 default, so we need check if it's H265 only. - if (id == SrsVideoCodecIdHEVC) { - if (source_->get_track_desc("video", "H265").empty()) { - std::vector video_track_descs = source_->get_track_desc("video", "H264"); - if (!video_track_descs.empty()) { - SrsRtcTrackDescription* video_track_desc = video_track_descs.at(0); - SrsVideoPayload* video_payload = (SrsVideoPayload*)video_track_desc->media_; - video_payload->name_ = "H265"; - video_payload->set_h265_param_desc("level-id=180;profile-id=1;tier-flag=0;tx-mode=SRST"); - srs_trace("RTC: Switch video codec %d(%s) to %d(%s)", SrsVideoCodecIdAVC, srs_video_codec_id2str(SrsVideoCodecIdAVC).c_str(), - id, srs_video_codec_id2str(id).c_str()); - } - } - } - return srs_success; -} - #endif SrsCompositeBridge::SrsCompositeBridge() diff --git a/trunk/src/app/srs_app_stream_bridge.hpp b/trunk/src/app/srs_app_stream_bridge.hpp index aad045823b..dfa4857c86 100644 --- a/trunk/src/app/srs_app_stream_bridge.hpp +++ b/trunk/src/app/srs_app_stream_bridge.hpp @@ -74,7 +74,6 @@ class SrsFrameToRtcBridge : public ISrsStreamBridge virtual void on_unpublish(); virtual srs_error_t on_frame(SrsSharedPtrMessage* frame); srs_error_t on_rtp(SrsRtpPacket* pkt); - srs_error_t update_codec(SrsVideoCodecId id); }; #endif diff --git a/trunk/src/kernel/srs_kernel_codec.cpp b/trunk/src/kernel/srs_kernel_codec.cpp index 38e68211a7..c929950247 100644 --- a/trunk/src/kernel/srs_kernel_codec.cpp +++ b/trunk/src/kernel/srs_kernel_codec.cpp @@ -676,7 +676,7 @@ srs_error_t SrsVideoFrame::add_sample(char* bytes, int size) // By default, use AVC(H.264) to parse NALU. // For video, parse the nalu type, set the IDR flag. - SrsAvcNaluType nal_unit_type = (SrsAvcNaluType)(bytes[0] & 0x1f); + SrsAvcNaluType nal_unit_type = SrsAvcNaluTypeParse(bytes[0]); if (nal_unit_type == SrsAvcNaluTypeIDR) { has_idr = true; @@ -751,12 +751,12 @@ srs_error_t SrsVideoFrame::parse_avc_b_frame(const SrsSample* sample, bool& is_b return err; } -srs_error_t SrsVideoFrame::parse_hevc_nalu_type(const SrsSample *sample, SrsHevcNaluType &hevc_nalu_type) +srs_error_t SrsVideoFrame::parse_hevc_nalu_type(const SrsSample* sample, SrsHevcNaluType& hevc_nalu_type) { srs_error_t err = srs_success; if (sample == NULL || sample->size < 1) { - return srs_error_new(ERROR_NALU_EMPTY, "empty nalu"); + return srs_error_new(ERROR_NALU_EMPTY, "empty hevc nalu"); } uint8_t header = sample->bytes[0]; @@ -765,7 +765,7 @@ srs_error_t SrsVideoFrame::parse_hevc_nalu_type(const SrsSample *sample, SrsHevc return err; } -srs_error_t SrsVideoFrame::parse_hevc_b_frame(const SrsSample *sample, SrsFormat *format, bool &is_b_frame) +srs_error_t SrsVideoFrame::parse_hevc_b_frame(const SrsSample* sample, SrsFormat *format, bool& is_b_frame) { srs_error_t err = srs_success; @@ -774,19 +774,20 @@ srs_error_t SrsVideoFrame::parse_hevc_b_frame(const SrsSample *sample, SrsFormat return srs_error_wrap(err, "parse hevc nalu type error"); } - SrsBuffer stream(sample->bytes, sample->size); - stream.skip(2); - - // @see 7.3.6.1 General slice segment header syntax - // @doc ITU-T-H.265-2021.pdf, page 66. - SrsBitBuffer bs(&stream); - uint8_t first_slice_segment_in_pic_flag = bs.read_bit(); if (nalu_type > SrsHevcNaluType_CODED_SLICE_BLA && nalu_type < SrsHevcNaluType_RESERVED_23) { - bs.skip_bits(1); is_b_frame = false; return err; } + SrsUniquePtr stream(new SrsBuffer(sample->bytes, sample->size)); + stream->skip(2); + + // @see 7.3.6.1 General slice segment header syntax + // @doc ITU-T-H.265-2021.pdf, page 66. + SrsBitBuffer bs(stream.get()); + + uint8_t first_slice_segment_in_pic_flag = bs.read_bit(); + uint32_t slice_pic_parameter_set_id; if ((err = bs.read_bits_ue(slice_pic_parameter_set_id)) != srs_success) { return srs_error_wrap(err, "read slice pic parameter set id"); @@ -798,15 +799,11 @@ srs_error_t SrsVideoFrame::parse_hevc_b_frame(const SrsSample *sample, SrsFormat SrsHevcRbspPps *pps = &(format->vcodec->hevc_dec_conf_record_.pps_table[slice_pic_parameter_set_id]); - uint8_t dependent_slice_segment_flag; + uint8_t dependent_slice_segment_flag = 0; if (!first_slice_segment_in_pic_flag) { if (pps->dependent_slice_segments_enabled_flag) { dependent_slice_segment_flag = bs.read_bit(); - } else { - dependent_slice_segment_flag = 0; } - } else { - dependent_slice_segment_flag = 0; } if (dependent_slice_segment_flag) { @@ -822,7 +819,10 @@ srs_error_t SrsVideoFrame::parse_hevc_b_frame(const SrsSample *sample, SrsFormat return srs_error_wrap(err, "read slice type"); } - is_b_frame = (slice_type == SrsHevcSliceTypeB) ? true : false; + is_b_frame = slice_type == SrsHevcSliceTypeB; + if (is_b_frame) { + srs_verbose("nalu_type=%d, slice type=%d", nalu_type, slice_type); + } // no need to evaluate the rest @@ -2337,7 +2337,7 @@ srs_error_t SrsFormat::avc_demux_sps() // 7.4.1 NAL unit semantics // ISO_IEC_14496-10-AVC-2012.pdf, page 61. // nal_unit_type specifies the type of RBSP data structure contained in the NAL unit as specified in Table 7-1. - SrsAvcNaluType nal_unit_type = (SrsAvcNaluType)(nutv & 0x1f); + SrsAvcNaluType nal_unit_type = SrsAvcNaluTypeParse(nutv); if (nal_unit_type != 7) { return srs_error_new(ERROR_HLS_DECODE_ERROR, "for sps, nal_unit_type shall be equal to 7"); } diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp index e461b07903..e1e19fa1ca 100644 --- a/trunk/src/kernel/srs_kernel_codec.hpp +++ b/trunk/src/kernel/srs_kernel_codec.hpp @@ -502,6 +502,10 @@ enum SrsHevcNaluType { // @see https://datatracker.ietf.org/doc/html/rfc7798#section-1.1.4 #define SrsHevcNaluTypeParse(code) (SrsHevcNaluType)((code & 0x7E) >> 1) +/** + * @see Table 7-7 – Name association to slice_type + * @doc ITU-T-H.265-2021.pdf, page 96. + */ enum SrsHevcSliceType { SrsHevcSliceTypeB = 0, SrsHevcSliceTypeP = 1, diff --git a/trunk/src/kernel/srs_kernel_rtc_rtp.hpp b/trunk/src/kernel/srs_kernel_rtc_rtp.hpp index 7f84dc5eea..791e069d1f 100644 --- a/trunk/src/kernel/srs_kernel_rtc_rtp.hpp +++ b/trunk/src/kernel/srs_kernel_rtc_rtp.hpp @@ -33,9 +33,9 @@ const uint8_t kStapA = 24; const uint8_t kFuA = 28; // @see: https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.2 -const uint8_t kStapHevc = 48; +const uint8_t kStapHevc = 48; // @see: https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.3 -const uint8_t kFuHevc = 49; +const uint8_t kFuHevc = 49; // @see: https://tools.ietf.org/html/rfc6184#section-5.8 const uint8_t kStart = 0x80; // Fu-header start bit @@ -488,7 +488,7 @@ class SrsRtpSTAPPayloadHevc : public ISrsRtpPayloader }; // FU, for one NALU with multiple fragments. -// With more than one payload. For HEVC. +// With more than one payload for HEVC. class SrsRtpFUAPayloadHevc : public ISrsRtpPayloader { public: @@ -511,7 +511,7 @@ class SrsRtpFUAPayloadHevc : public ISrsRtpPayloader }; // FU, for one NALU with multiple fragments. -// With only one payload. For HEVC. +// With only one payload for HEVC. class SrsRtpFUAPayloadHevc2 : public ISrsRtpPayloader { public: From c2e31d1578f3aa6d2085960143952c6dca4e7295 Mon Sep 17 00:00:00 2001 From: chundonglinlin Date: Sat, 22 Feb 2025 23:23:10 +0800 Subject: [PATCH 05/17] Modify the play stream to specify encoding format. --- trunk/src/app/srs_app_rtc_source.cpp | 31 +++++++++---------- trunk/src/app/srs_app_stream_bridge.cpp | 40 +++++++++++++++++++++++++ trunk/src/app/srs_app_stream_bridge.hpp | 3 ++ 3 files changed, 57 insertions(+), 17 deletions(-) diff --git a/trunk/src/app/srs_app_rtc_source.cpp b/trunk/src/app/srs_app_rtc_source.cpp index 676766e0bb..62dd000d7b 100644 --- a/trunk/src/app/srs_app_rtc_source.cpp +++ b/trunk/src/app/srs_app_rtc_source.cpp @@ -483,23 +483,6 @@ void SrsRtcSource::init_for_play_before_publishing() video_track_desc->media_ = video_payload; video_payload->set_h264_param_desc("level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f"); - -#ifdef SRS_H265 - // default h265 video track description - SrsRtcTrackDescription* h265_video_track_desc = new SrsRtcTrackDescription(); - stream_desc->video_track_descs_.push_back(h265_video_track_desc); - - h265_video_track_desc->type_ = "video"; - h265_video_track_desc->id_ = "video-" + srs_random_str(8); - h265_video_track_desc->ssrc_ = video_ssrc; - h265_video_track_desc->direction_ = "recvonly"; - - SrsVideoPayload* h265_video_payload = new SrsVideoPayload(kVideoPayloadType, "H265", kVideoSamplerate); - h265_video_track_desc->media_ = h265_video_payload; - - h265_video_payload->set_h265_param_desc("level-id=180;profile-id=1;tier-flag=0;tx-mode=SRST"); -#endif - } set_stream_desc(stream_desc.get()); @@ -1090,6 +1073,20 @@ srs_error_t SrsRtcRtpBuilder::on_video(SrsSharedPtrMessage* msg) return err; } + // support video codec: h264/h265 + SrsVideoCodecId vcodec = format->vcodec->id; + if (vcodec != SrsVideoCodecIdAVC && vcodec != SrsVideoCodecIdHEVC) { + return err; + } + +#ifdef SRS_H265 + if (vcodec == SrsVideoCodecIdHEVC) { + if ((err = bridge_->update_codec(vcodec)) != srs_success) { + return srs_error_wrap(err, "update codec"); + } + } +#endif + bool has_idr = false; vector samples; if ((err = filter(msg, format, has_idr, samples)) != srs_success) { diff --git a/trunk/src/app/srs_app_stream_bridge.cpp b/trunk/src/app/srs_app_stream_bridge.cpp index b80c49a690..e45d75f130 100644 --- a/trunk/src/app/srs_app_stream_bridge.cpp +++ b/trunk/src/app/srs_app_stream_bridge.cpp @@ -95,6 +95,8 @@ SrsFrameToRtcBridge::SrsFrameToRtcBridge(SrsSharedPtr source) rtp_builder_ = new SrsRtcRtpBuilder(this, audio_ssrc, audio_payload_type, video_ssrc, video_payload_type); #endif + + codec_switched_ = false; } SrsFrameToRtcBridge::~SrsFrameToRtcBridge() @@ -156,6 +158,44 @@ srs_error_t SrsFrameToRtcBridge::on_rtp(SrsRtpPacket* pkt) return source_->on_rtp(pkt); } +srs_error_t SrsFrameToRtcBridge::update_codec(SrsVideoCodecId id) +{ + srs_error_t err = srs_success; + + // Only handle H.265/HEVC codec switch. + if (id != SrsVideoCodecIdHEVC) { + return err; + } + + if (codec_switched_) { + return err; + } + + // Check if H.265 track description exists + if (!source_->get_track_desc("video", "H265").empty()) { + return err; + } + + // Try to convert H.264 track to H.265 + std::vector video_track_descs = source_->get_track_desc("video", "H264"); + if (video_track_descs.empty()) { + return srs_error_new(ERROR_RTC_NO_TRACK, "no H264 track found for conversion"); + } + + SrsRtcTrackDescription* video_track_desc = video_track_descs.at(0); + SrsVideoPayload* video_payload = (SrsVideoPayload*)video_track_desc->media_; + video_payload->name_ = "H265"; + video_payload->set_h265_param_desc("level-id=180;profile-id=1;tier-flag=0;tx-mode=SRST"); + + codec_switched_ = true; + + srs_trace("RTC: Switch video codec %d(%s) to %d(%s)", + SrsVideoCodecIdAVC, srs_video_codec_id2str(SrsVideoCodecIdAVC).c_str(), + id, srs_video_codec_id2str(id).c_str()); + + return err; +} + #endif SrsCompositeBridge::SrsCompositeBridge() diff --git a/trunk/src/app/srs_app_stream_bridge.hpp b/trunk/src/app/srs_app_stream_bridge.hpp index dfa4857c86..5310912525 100644 --- a/trunk/src/app/srs_app_stream_bridge.hpp +++ b/trunk/src/app/srs_app_stream_bridge.hpp @@ -65,6 +65,8 @@ class SrsFrameToRtcBridge : public ISrsStreamBridge #if defined(SRS_FFMPEG_FIT) SrsRtcRtpBuilder* rtp_builder_; #endif +private: + bool codec_switched_; public: SrsFrameToRtcBridge(SrsSharedPtr source); virtual ~SrsFrameToRtcBridge(); @@ -74,6 +76,7 @@ class SrsFrameToRtcBridge : public ISrsStreamBridge virtual void on_unpublish(); virtual srs_error_t on_frame(SrsSharedPtrMessage* frame); srs_error_t on_rtp(SrsRtpPacket* pkt); + srs_error_t update_codec(SrsVideoCodecId id); }; #endif From 6363ed3e2793cb70313734b1412e80d0d77014a9 Mon Sep 17 00:00:00 2001 From: Haibo Chen <495810242@qq.com> Date: Mon, 24 Feb 2025 14:33:48 +0800 Subject: [PATCH 06/17] Fix HLS logging error --- trunk/src/app/srs_app_hls.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trunk/src/app/srs_app_hls.cpp b/trunk/src/app/srs_app_hls.cpp index 47022b9ae5..1a4f9f7f79 100644 --- a/trunk/src/app/srs_app_hls.cpp +++ b/trunk/src/app/srs_app_hls.cpp @@ -1068,7 +1068,7 @@ srs_error_t SrsHlsController::write_video(SrsVideoFrame* frame, int64_t dts) // Refresh the codec ASAP. if (muxer->latest_vcodec() != frame->vcodec()->id) { - srs_trace("HLS: Switch video codec %d(%s) to %d(%s)", muxer->latest_acodec(), srs_video_codec_id2str(muxer->latest_vcodec()).c_str(), + srs_trace("HLS: Switch video codec %d(%s) to %d(%s)", muxer->latest_vcodec(), srs_video_codec_id2str(muxer->latest_vcodec()).c_str(), frame->vcodec()->id, srs_video_codec_id2str(frame->vcodec()->id).c_str()); muxer->set_latest_vcodec(frame->vcodec()->id); } From 4733e7f08746835a59ec064b0244a0d4333ef297 Mon Sep 17 00:00:00 2001 From: Haibo Chen <495810242@qq.com> Date: Mon, 24 Feb 2025 14:39:10 +0800 Subject: [PATCH 07/17] Fix HEVC STAP payload decoding in RTP --- trunk/src/kernel/srs_kernel_rtc_rtp.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/trunk/src/kernel/srs_kernel_rtc_rtp.cpp b/trunk/src/kernel/srs_kernel_rtc_rtp.cpp index 5ba2204aba..7e55bb1578 100644 --- a/trunk/src/kernel/srs_kernel_rtc_rtp.cpp +++ b/trunk/src/kernel/srs_kernel_rtc_rtp.cpp @@ -1657,6 +1657,7 @@ srs_error_t SrsRtpSTAPPayloadHevc::decode(SrsBuffer* buf) // STAP header, RTP payload format for aggregation packets // @see https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.2 uint8_t v = buf->read_1bytes(); + buf->skip(1); // forbidden_zero_bit shoul be zero. // @see https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.2 From 4a1c4099297651a28a261ea6471fdb3fd80bb2f4 Mon Sep 17 00:00:00 2001 From: Haibo Chen <495810242@qq.com> Date: Tue, 4 Mar 2025 15:59:56 +0800 Subject: [PATCH 08/17] prevent memory leaks --- trunk/src/app/srs_app_rtc_source.cpp | 64 ++++++++++++++++++--------- trunk/src/kernel/srs_kernel_codec.hpp | 5 +++ 2 files changed, 48 insertions(+), 21 deletions(-) diff --git a/trunk/src/app/srs_app_rtc_source.cpp b/trunk/src/app/srs_app_rtc_source.cpp index 62dd000d7b..8336fbc863 100644 --- a/trunk/src/app/srs_app_rtc_source.cpp +++ b/trunk/src/app/srs_app_rtc_source.cpp @@ -1053,6 +1053,13 @@ srs_error_t SrsRtcRtpBuilder::package_opus(SrsAudioFrame* audio, SrsRtpPacket* p return err; } +void cleanup_packets(vector& packets) { + for (size_t i = 0; i < packets.size(); i++) { + srs_freep(packets[i]); + } + packets.clear(); +} + srs_error_t SrsRtcRtpBuilder::on_video(SrsSharedPtrMessage* msg) { srs_error_t err = srs_success; @@ -1111,6 +1118,7 @@ srs_error_t SrsRtcRtpBuilder::on_video(SrsSharedPtrMessage* msg) vector pkts; if (merge_nalus && nn_samples > 1) { if ((err = package_nalus(msg, samples, pkts)) != srs_success) { + cleanup_packets(pkts); return srs_error_wrap(err, "package nalus as one"); } } else { @@ -1120,10 +1128,12 @@ srs_error_t SrsRtcRtpBuilder::on_video(SrsSharedPtrMessage* msg) if (sample->size <= kRtpMaxPayloadSize) { if ((err = package_single_nalu(msg, sample, pkts)) != srs_success) { + cleanup_packets(pkts); return srs_error_wrap(err, "package single nalu"); } } else { if ((err = package_fu_a(msg, sample, kRtpMaxPayloadSize, pkts)) != srs_success) { + cleanup_packets(pkts); return srs_error_wrap(err, "package fu-a"); } } @@ -1255,8 +1265,14 @@ srs_error_t SrsRtcRtpBuilder::package_nalus(SrsSharedPtrMessage* msg, const vect { srs_error_t err = srs_success; + SrsFormat* format = meta->vsh_format(); + if (!format || !format->vcodec) { + return err; + } + bool is_hevc = format->vcodec->id == SrsVideoCodecIdHEVC; + SrsRtpRawNALUs* raw_raw = new SrsRtpRawNALUs(); - SrsAvcNaluType first_nalu_type = SrsAvcNaluTypeReserved; + uint8_t first_nalu_type = 0; for (int i = 0; i < (int)samples.size(); i++) { SrsSample* sample = samples[i]; @@ -1265,8 +1281,8 @@ srs_error_t SrsRtcRtpBuilder::package_nalus(SrsSharedPtrMessage* msg, const vect continue; } - if (first_nalu_type == SrsAvcNaluTypeReserved) { - first_nalu_type = SrsAvcNaluTypeParse(sample->bytes[0]); + if (first_nalu_type == 0) { + first_nalu_type = is_hevc ? SrsHevcNaluTypeParse(sample->bytes[0]) : SrsAvcNaluTypeParse(sample->bytes[0]); } raw_raw->push_back(sample->copy()); @@ -1293,20 +1309,12 @@ srs_error_t SrsRtcRtpBuilder::package_nalus(SrsSharedPtrMessage* msg, const vect pkt->set_payload(raw_raw, SrsRtspPacketPayloadTypeNALU); pkt->wrap(msg); } else { - SrsFormat* format = meta->vsh_format(); - if (!format || !format->vcodec) { - return err; - } - - bool is_hevc = format->vcodec->id == SrsVideoCodecIdHEVC; - // H264 FU-A header size is 1 @see: https://datatracker.ietf.org/doc/html/rfc6184#section-5.8 - // H265 FU-A header size is 2 @see: https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.3 - int header_size = is_hevc ? 2 : 1; - // We must free it, should never use RTP packets to free it, // because more than one RTP packet will refer to it. SrsUniquePtr raw(raw_raw); + int header_size = is_hevc ? SrsHevcNaluHeaderSize : SrsAvcNaluHeaderSize; + // Package NALUs in FU-A RTP packets. int fu_payload_size = kRtpMaxPayloadSize; @@ -1397,9 +1405,9 @@ srs_error_t SrsRtcRtpBuilder::package_fu_a(SrsSharedPtrMessage* msg, SrsSample* } bool is_hevc = format->vcodec->id == SrsVideoCodecIdHEVC; - // H264 FU-A header size is 1 @see: https://datatracker.ietf.org/doc/html/rfc6184#section-5.8 - // H265 FU-A header size is 2 @see: https://datatracker.ietf.org/doc/html/rfc7798#section-4.4.3 - int header_size = is_hevc ? 2 : 1; + int header_size = is_hevc ? SrsHevcNaluHeaderSize : SrsAvcNaluHeaderSize; + srs_assert(sample->size >= header_size); + char* p = sample->bytes + header_size; int nb_left = sample->size - header_size; uint8_t header = sample->bytes[0]; @@ -2128,14 +2136,20 @@ SrsMediaPayloadType SrsVideoPayload::generate_media_payload_type() media_payload_type.rtcp_fb_ = rtcp_fbs_; std::ostringstream format_specific_param; + bool has_param = false; + if (!h264_param_.level_asymmerty_allow.empty()) { format_specific_param << "level-asymmetry-allowed=" << h264_param_.level_asymmerty_allow; + has_param = true; } if (!h264_param_.packetization_mode.empty()) { - format_specific_param << ";packetization-mode=" << h264_param_.packetization_mode; + if (has_param) format_specific_param << ";"; + format_specific_param << "packetization-mode=" << h264_param_.packetization_mode; + has_param = true; } if (!h264_param_.profile_level_id.empty()) { - format_specific_param << ";profile-level-id=" << h264_param_.profile_level_id; + if (has_param) format_specific_param << ";"; + format_specific_param << "profile-level-id=" << h264_param_.profile_level_id; } media_payload_type.format_specific_param_ = format_specific_param.str(); @@ -2152,17 +2166,25 @@ SrsMediaPayloadType SrsVideoPayload::generate_media_payload_type_h265() media_payload_type.rtcp_fb_ = rtcp_fbs_; std::ostringstream format_specific_param; + bool has_param = false; + if (!h265_param_.level_id.empty()) { format_specific_param << "level-id=" << h265_param_.level_id; + has_param = true; } if (!h265_param_.profile_id.empty()) { - format_specific_param << ";profile-id=" << h265_param_.profile_id; + if (has_param) format_specific_param << ";"; + format_specific_param << "profile-id=" << h265_param_.profile_id; + has_param = true; } if (!h265_param_.tier_flag.empty()) { - format_specific_param << ";tier-flag=" << h265_param_.tier_flag; + if (has_param) format_specific_param << ";"; + format_specific_param << "tier-flag=" << h265_param_.tier_flag; + has_param = true; } if (!h265_param_.tx_mode.empty()) { - format_specific_param << ";tx-mode=" << h265_param_.tx_mode; + if (has_param) format_specific_param << ";"; + format_specific_param << "tx-mode=" << h265_param_.tx_mode; } media_payload_type.format_specific_param_ = format_specific_param.str(); diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp index e1e19fa1ca..1d498ce710 100644 --- a/trunk/src/kernel/srs_kernel_codec.hpp +++ b/trunk/src/kernel/srs_kernel_codec.hpp @@ -16,6 +16,11 @@ class SrsBuffer; class SrsBitBuffer; class SrsFormat; +// @see: https://datatracker.ietf.org/doc/html/rfc6184#section-1.3 +const int SrsAvcNaluHeaderSize = 1; +// @see: https://datatracker.ietf.org/doc/html/rfc7798#section-1.1.4 +const int SrsHevcNaluHeaderSize = 2; + /** * The video codec id. * @doc video_file_format_spec_v10_1.pdf, page78, E.4.3.1 VIDEODATA From 49cc59939e80ee5bc4b96367555d90b06569621c Mon Sep 17 00:00:00 2001 From: Haibo Chen <495810242@qq.com> Date: Mon, 17 Mar 2025 15:36:02 +0800 Subject: [PATCH 09/17] update Update the page numbers --- trunk/src/kernel/srs_kernel_codec.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp index 1d498ce710..535100bfe3 100644 --- a/trunk/src/kernel/srs_kernel_codec.hpp +++ b/trunk/src/kernel/srs_kernel_codec.hpp @@ -509,7 +509,7 @@ enum SrsHevcNaluType { /** * @see Table 7-7 – Name association to slice_type - * @doc ITU-T-H.265-2021.pdf, page 96. + * @doc ITU-T-H.265-2021.pdf, page 116. */ enum SrsHevcSliceType { SrsHevcSliceTypeB = 0, From 8a9e7aeca2498414210342b8a9e5bae9ce8fbab2 Mon Sep 17 00:00:00 2001 From: Haibo Chen <495810242@qq.com> Date: Tue, 18 Mar 2025 14:16:19 +0800 Subject: [PATCH 10/17] check pointer --- trunk/src/kernel/srs_kernel_codec.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/trunk/src/kernel/srs_kernel_codec.cpp b/trunk/src/kernel/srs_kernel_codec.cpp index c929950247..63ecdc5414 100644 --- a/trunk/src/kernel/srs_kernel_codec.cpp +++ b/trunk/src/kernel/srs_kernel_codec.cpp @@ -798,6 +798,9 @@ srs_error_t SrsVideoFrame::parse_hevc_b_frame(const SrsSample* sample, SrsFormat } SrsHevcRbspPps *pps = &(format->vcodec->hevc_dec_conf_record_.pps_table[slice_pic_parameter_set_id]); + if (!pps) { + return srs_error_new(ERROR_HEVC_DECODE_ERROR, "pps not found"); + } uint8_t dependent_slice_segment_flag = 0; if (!first_slice_segment_in_pic_flag) { From e8a5e3ecf39bf6e844c938c6647d0ffd76fcb0ab Mon Sep 17 00:00:00 2001 From: Haibo Chen <495810242@qq.com> Date: Tue, 18 Mar 2025 14:45:06 +0800 Subject: [PATCH 11/17] fix compile warn --- trunk/src/app/srs_app_rtc_source.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/trunk/src/app/srs_app_rtc_source.cpp b/trunk/src/app/srs_app_rtc_source.cpp index 8336fbc863..db695576e5 100644 --- a/trunk/src/app/srs_app_rtc_source.cpp +++ b/trunk/src/app/srs_app_rtc_source.cpp @@ -1216,7 +1216,7 @@ srs_error_t SrsRtcRtpBuilder::package_stap_a(SrsSharedPtrMessage* msg, SrsRtpPac vector*> params; int size = 0; if (format->vcodec->id == SrsVideoCodecIdHEVC) { - for (int i = 0; i < format->vcodec->hevc_dec_conf_record_.nalu_vec.size(); i++) { + for (size_t i = 0; i < format->vcodec->hevc_dec_conf_record_.nalu_vec.size(); i++) { if (format->vcodec->hevc_dec_conf_record_.nalu_vec[i].nal_unit_type == SrsHevcNaluType_VPS || format->vcodec->hevc_dec_conf_record_.nalu_vec[i].nal_unit_type == SrsHevcNaluType_SPS || format->vcodec->hevc_dec_conf_record_.nalu_vec[i].nal_unit_type == SrsHevcNaluType_PPS) { @@ -1282,7 +1282,7 @@ srs_error_t SrsRtcRtpBuilder::package_nalus(SrsSharedPtrMessage* msg, const vect } if (first_nalu_type == 0) { - first_nalu_type = is_hevc ? SrsHevcNaluTypeParse(sample->bytes[0]) : SrsAvcNaluTypeParse(sample->bytes[0]); + first_nalu_type = is_hevc ? uint8_t(SrsHevcNaluTypeParse(sample->bytes[0])) : uint8_t(SrsAvcNaluTypeParse(sample->bytes[0])); } raw_raw->push_back(sample->copy()); From ea0d2301f6fd31c1965dc0aab4e8d36ec6fd437f Mon Sep 17 00:00:00 2001 From: Haibo Chen <495810242@qq.com> Date: Wed, 19 Mar 2025 10:50:41 +0800 Subject: [PATCH 12/17] add SrsRtspPacketPayloadTypeFUAHevc2 --- trunk/src/app/srs_app_rtc_source.cpp | 2 +- trunk/src/kernel/srs_kernel_rtc_rtp.hpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/trunk/src/app/srs_app_rtc_source.cpp b/trunk/src/app/srs_app_rtc_source.cpp index db695576e5..335d1bc35a 100644 --- a/trunk/src/app/srs_app_rtc_source.cpp +++ b/trunk/src/app/srs_app_rtc_source.cpp @@ -1429,7 +1429,7 @@ srs_error_t SrsRtcRtpBuilder::package_fu_a(SrsSharedPtrMessage* msg, SrsSample* if (is_hevc) { // H265 FU-A header SrsRtpFUAPayloadHevc2* fua = new SrsRtpFUAPayloadHevc2(); - pkt->set_payload(fua, SrsRtspPacketPayloadTypeFUAHevc); + pkt->set_payload(fua, SrsRtspPacketPayloadTypeFUAHevc2); fua->nalu_type = SrsHevcNaluTypeParse(header); fua->start = bool(i == 0); diff --git a/trunk/src/kernel/srs_kernel_rtc_rtp.hpp b/trunk/src/kernel/srs_kernel_rtc_rtp.hpp index 791e069d1f..ff731db0b2 100644 --- a/trunk/src/kernel/srs_kernel_rtc_rtp.hpp +++ b/trunk/src/kernel/srs_kernel_rtc_rtp.hpp @@ -257,6 +257,7 @@ enum SrsRtspPacketPayloadType { SrsRtspPacketPayloadTypeRaw, SrsRtspPacketPayloadTypeFUA2, + SrsRtspPacketPayloadTypeFUAHevc2, SrsRtspPacketPayloadTypeFUA, SrsRtspPacketPayloadTypeFUAHevc, SrsRtspPacketPayloadTypeNALU, From 33b24010700fa24ebc96b6a84c8708f8e3cbc3c4 Mon Sep 17 00:00:00 2001 From: Haibo Chen <495810242@qq.com> Date: Wed, 19 Mar 2025 17:22:04 +0800 Subject: [PATCH 13/17] add utest: VideoFrame --- trunk/src/kernel/srs_kernel_codec.cpp | 4 +- trunk/src/utest/srs_utest_kernel.cpp | 200 +++++++++++++++++++++++++- 2 files changed, 201 insertions(+), 3 deletions(-) diff --git a/trunk/src/kernel/srs_kernel_codec.cpp b/trunk/src/kernel/srs_kernel_codec.cpp index 63ecdc5414..1982b9d03a 100644 --- a/trunk/src/kernel/srs_kernel_codec.cpp +++ b/trunk/src/kernel/srs_kernel_codec.cpp @@ -721,7 +721,7 @@ srs_error_t SrsVideoFrame::parse_avc_b_frame(const SrsSample* sample, bool& is_b return srs_error_wrap(err, "parse avc nalu type error"); } - if (nalu_type != SrsAvcNaluTypeNonIDR && nalu_type != SrsAvcNaluTypeDataPartitionA && nalu_type != SrsAvcNaluTypeIDR) { + if (nalu_type != SrsAvcNaluTypeNonIDR && nalu_type != SrsAvcNaluTypeDataPartitionA) { is_b_frame = false; return err; } @@ -774,7 +774,7 @@ srs_error_t SrsVideoFrame::parse_hevc_b_frame(const SrsSample* sample, SrsFormat return srs_error_wrap(err, "parse hevc nalu type error"); } - if (nalu_type > SrsHevcNaluType_CODED_SLICE_BLA && nalu_type < SrsHevcNaluType_RESERVED_23) { + if (nalu_type > SrsHevcNaluType_CODED_SLICE_TFD) { is_b_frame = false; return err; } diff --git a/trunk/src/utest/srs_utest_kernel.cpp b/trunk/src/utest/srs_utest_kernel.cpp index b0d428aa7a..17992957cf 100644 --- a/trunk/src/utest/srs_utest_kernel.cpp +++ b/trunk/src/utest/srs_utest_kernel.cpp @@ -3567,6 +3567,204 @@ VOID TEST(KernelCodecTest, AVFrameNoConfig) HELPER_EXPECT_SUCCESS(f.add_sample((char*)"\x05", 1)); } } +VOID TEST(KernelCodecTest, VideoFrameH264) +{ + srs_error_t err; + + if (true) { + // I Frame + uint8_t data[] = {0x05, 0x00, 0x00, 0x00}; + SrsSample sample((char*)data, sizeof(data)); + + SrsAvcNaluType nalu_type = SrsAvcNaluTypeForbidden; + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_avc_nalu_type(&sample, nalu_type)); + EXPECT_EQ(nalu_type, SrsAvcNaluTypeIDR); + + // P Frame + uint8_t data2[] = {0x01, 0x00, 0x00, 0x00}; + SrsSample sample2((char*)data2, sizeof(data2)); + + nalu_type = SrsAvcNaluTypeForbidden; + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_avc_nalu_type(&sample2, nalu_type)); + EXPECT_EQ(nalu_type, SrsAvcNaluTypeNonIDR); + + // SPS + uint8_t data3[] = {0x07, 0x00, 0x00, 0x00}; + SrsSample sample3((char*)data3, sizeof(data3)); + + nalu_type = SrsAvcNaluTypeForbidden; + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_avc_nalu_type(&sample3, nalu_type)); + EXPECT_EQ(nalu_type, SrsAvcNaluTypeSPS); + + // PPS + uint8_t data4[] = {0x08, 0x00, 0x00, 0x00}; + SrsSample sample4((char*)data4, sizeof(data4)); + + nalu_type = SrsAvcNaluTypeForbidden; + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_avc_nalu_type(&sample4, nalu_type)); + EXPECT_EQ(nalu_type, SrsAvcNaluTypePPS); + + // Empty Sample + SrsSample empty_sample(NULL, 0); + HELPER_EXPECT_FAILED(SrsVideoFrame::parse_avc_nalu_type(&empty_sample, nalu_type)); + } + + if (true) { + // B Frame, slice_type=1(B Frame) + uint8_t data[] = {0x01, 0xA8, 0x00, 0x00}; + SrsSample sample((char*)data, sizeof(data)); + + bool is_b_frame = false; + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_avc_b_frame(&sample, is_b_frame)); + EXPECT_TRUE(is_b_frame); + + // Non-B Frame, slice_type=0(P Frame) + uint8_t data2[] = {0x01, 0x88, 0x00, 0x00}; + SrsSample sample2((char*)data2, sizeof(data2)); + + is_b_frame = true; + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_avc_b_frame(&sample2, is_b_frame)); + EXPECT_FALSE(is_b_frame); + + // SPS + uint8_t data3[] = {0x07, 0xA8, 0x00, 0x00}; + SrsSample sample3((char*)data3, sizeof(data3)); + + is_b_frame = true; + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_avc_b_frame(&sample3, is_b_frame)); + EXPECT_FALSE(is_b_frame); + + // PPS + uint8_t data4[] = {0x08, 0xA8, 0x00, 0x00}; + SrsSample sample4((char*)data4, sizeof(data4)); + + is_b_frame = true; + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_avc_b_frame(&sample4, is_b_frame)); + EXPECT_FALSE(is_b_frame); + + // IDR + uint8_t data5[] = {0x05, 0xA8, 0x00, 0x00}; + SrsSample sample5((char*)data5, sizeof(data5)); + + is_b_frame = true; + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_avc_b_frame(&sample5, is_b_frame)); + EXPECT_FALSE(is_b_frame); + + // Empty Sample + SrsSample empty_sample(NULL, 0); + HELPER_EXPECT_FAILED(SrsVideoFrame::parse_avc_b_frame(&empty_sample, is_b_frame)); + } +} + +#ifdef SRS_H265 +VOID TEST(KernelCodecTest, VideoFrameH265) +{ + srs_error_t err; + + if (true) { + // I Frame + uint8_t data[] = {0x26, 0x01, 0x00, 0x00}; + SrsSample sample((char*)data, sizeof(data)); + + SrsHevcNaluType nalu_type = SrsHevcNaluType_INVALID; + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_nalu_type(&sample, nalu_type)); + EXPECT_EQ(nalu_type, SrsHevcNaluType_CODED_SLICE_IDR); + + // P Frame + uint8_t data2[] = {0x02, 0x01, 0x00, 0x00}; + SrsSample sample2((char*)data2, sizeof(data2)); + + nalu_type = SrsHevcNaluType_INVALID; + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_nalu_type(&sample2, nalu_type)); + EXPECT_EQ(nalu_type, SrsHevcNaluType_CODED_SLICE_TRAIL_R); + + // VPS + uint8_t data3[] = {0x40, 0x01, 0x00, 0x00}; + SrsSample sample3((char*)data3, sizeof(data3)); + + nalu_type = SrsHevcNaluType_INVALID; + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_nalu_type(&sample3, nalu_type)); + EXPECT_EQ(nalu_type, SrsHevcNaluType_VPS); + + // SPS + uint8_t data4[] = {0x42, 0x01, 0x00, 0x00}; + SrsSample sample4((char*)data4, sizeof(data4)); + + nalu_type = SrsHevcNaluType_INVALID; + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_nalu_type(&sample4, nalu_type)); + EXPECT_EQ(nalu_type, SrsHevcNaluType_SPS); + + // PPS + uint8_t data5[] = {0x44, 0x01, 0x00, 0x00}; + SrsSample sample5((char*)data5, sizeof(data5)); + + nalu_type = SrsHevcNaluType_INVALID; + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_nalu_type(&sample5, nalu_type)); + EXPECT_EQ(nalu_type, SrsHevcNaluType_PPS); + + // Empty Sample + SrsSample empty_sample(NULL, 0); + HELPER_EXPECT_FAILED(SrsVideoFrame::parse_hevc_nalu_type(&empty_sample, nalu_type)); + } + + if (true) { + SrsFormat format; + HELPER_EXPECT_SUCCESS(format.initialize()); + + // B Frame, slice_type=0(B Frame) + uint8_t data[] = {0x02, 0x01, 0xE0, 0x44}; + SrsSample sample((char*)data, sizeof(data)); + + bool is_b_frame = false; + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_b_frame(&sample, &format, is_b_frame)); + EXPECT_TRUE(is_b_frame); + + // Non-B Frame, slice_type=1(P Frame) + uint8_t data2[] = {0x02, 0x01, 0xD0, 0x30}; + SrsSample sample2((char*)data2, sizeof(data2)); + + is_b_frame = true; + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_b_frame(&sample2, &format, is_b_frame)); + EXPECT_FALSE(is_b_frame); + + // VPS + uint8_t data3[] = {0x40, 0x01, 0xE0, 0x44}; + SrsSample sample3((char*)data3, sizeof(data3)); + + is_b_frame = true; + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_b_frame(&sample3, &format, is_b_frame)); + EXPECT_FALSE(is_b_frame); + + // SPS + uint8_t data4[] = {0x42, 0x01, 0xE0, 0x44}; + SrsSample sample4((char*)data4, sizeof(data4)); + + is_b_frame = true; + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_b_frame(&sample4, &format, is_b_frame)); + EXPECT_FALSE(is_b_frame); + + // PPS + uint8_t data5[] = {0x44, 0x01, 0xE0, 0x44}; + SrsSample sample5((char*)data5, sizeof(data5)); + + is_b_frame = true; + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_b_frame(&sample5, &format, is_b_frame)); + EXPECT_FALSE(is_b_frame); + + // IDR + uint8_t data6[] = {0x26, 0x01, 0xE0, 0x44}; + SrsSample sample6((char*)data6, sizeof(data6)); + + is_b_frame = true; + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_b_frame(&sample6, &format, is_b_frame)); + EXPECT_FALSE(is_b_frame); + + // Empty Sample + SrsSample empty_sample(NULL, 0); + HELPER_EXPECT_FAILED(SrsVideoFrame::parse_hevc_b_frame(&empty_sample, &format, is_b_frame)); + } +} +#endif VOID TEST(KernelCodecTest, IsSequenceHeaderSpecial) { @@ -4221,7 +4419,7 @@ VOID TEST(KernelCodecTest, HevcVideoFormat) if (true) { SrsFormat f; HELPER_EXPECT_SUCCESS(f.initialize()); - + // firstly demux sequence header HELPER_EXPECT_SUCCESS(f.on_video(0, (char*)ext_vps_sps_pps, sizeof(ext_vps_sps_pps))); EXPECT_EQ(1, f.video->frame_type); From a7cb838cd7ebdfeb5d086388708acac3805af503 Mon Sep 17 00:00:00 2001 From: Haibo Chen <495810242@qq.com> Date: Thu, 20 Mar 2025 17:01:35 +0800 Subject: [PATCH 14/17] move to another pull request. --- trunk/src/app/srs_app_rtc_source.cpp | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/trunk/src/app/srs_app_rtc_source.cpp b/trunk/src/app/srs_app_rtc_source.cpp index 335d1bc35a..382a01b8fc 100644 --- a/trunk/src/app/srs_app_rtc_source.cpp +++ b/trunk/src/app/srs_app_rtc_source.cpp @@ -1053,13 +1053,6 @@ srs_error_t SrsRtcRtpBuilder::package_opus(SrsAudioFrame* audio, SrsRtpPacket* p return err; } -void cleanup_packets(vector& packets) { - for (size_t i = 0; i < packets.size(); i++) { - srs_freep(packets[i]); - } - packets.clear(); -} - srs_error_t SrsRtcRtpBuilder::on_video(SrsSharedPtrMessage* msg) { srs_error_t err = srs_success; @@ -1118,7 +1111,6 @@ srs_error_t SrsRtcRtpBuilder::on_video(SrsSharedPtrMessage* msg) vector pkts; if (merge_nalus && nn_samples > 1) { if ((err = package_nalus(msg, samples, pkts)) != srs_success) { - cleanup_packets(pkts); return srs_error_wrap(err, "package nalus as one"); } } else { @@ -1128,12 +1120,10 @@ srs_error_t SrsRtcRtpBuilder::on_video(SrsSharedPtrMessage* msg) if (sample->size <= kRtpMaxPayloadSize) { if ((err = package_single_nalu(msg, sample, pkts)) != srs_success) { - cleanup_packets(pkts); return srs_error_wrap(err, "package single nalu"); } } else { if ((err = package_fu_a(msg, sample, kRtpMaxPayloadSize, pkts)) != srs_success) { - cleanup_packets(pkts); return srs_error_wrap(err, "package fu-a"); } } From 9e6fd12cbb62d3e2697d411400ddebda030f1a4e Mon Sep 17 00:00:00 2001 From: Haibo Chen <495810242@qq.com> Date: Fri, 21 Mar 2025 09:31:10 +0800 Subject: [PATCH 15/17] use c++03 style --- trunk/src/app/srs_app_rtc_source.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/trunk/src/app/srs_app_rtc_source.cpp b/trunk/src/app/srs_app_rtc_source.cpp index 382a01b8fc..f587181f1f 100644 --- a/trunk/src/app/srs_app_rtc_source.cpp +++ b/trunk/src/app/srs_app_rtc_source.cpp @@ -1234,7 +1234,8 @@ srs_error_t SrsRtcRtpBuilder::package_stap_a(SrsSharedPtrMessage* msg, SrsRtpPac } char* payload = pkt->wrap(size); - for (vector* param : params) { + for (vector*>::iterator it = params.begin(); it != params.end(); ++it) { + vector* param = *it; SrsSample* sample = new SrsSample(); sample->bytes = payload; sample->size = param->size(); From a33aa5971c32dd7f83a2d20977fc52781d2b2bc4 Mon Sep 17 00:00:00 2001 From: Haibo Chen <495810242@qq.com> Date: Fri, 21 Mar 2025 09:42:47 +0800 Subject: [PATCH 16/17] Update AVC frame parsing logic to correctly identify IDR, SPS, and PPS NALU types. --- trunk/src/kernel/srs_kernel_codec.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trunk/src/kernel/srs_kernel_codec.cpp b/trunk/src/kernel/srs_kernel_codec.cpp index 1982b9d03a..093e302a25 100644 --- a/trunk/src/kernel/srs_kernel_codec.cpp +++ b/trunk/src/kernel/srs_kernel_codec.cpp @@ -721,7 +721,7 @@ srs_error_t SrsVideoFrame::parse_avc_b_frame(const SrsSample* sample, bool& is_b return srs_error_wrap(err, "parse avc nalu type error"); } - if (nalu_type != SrsAvcNaluTypeNonIDR && nalu_type != SrsAvcNaluTypeDataPartitionA) { + if (nalu_type == SrsAvcNaluTypeIDR || nalu_type == SrsAvcNaluTypeSPS || nalu_type == SrsAvcNaluTypePPS) { is_b_frame = false; return err; } From dc4acb582e16fed00455f3aa2040f58bcf7c5f97 Mon Sep 17 00:00:00 2001 From: chenhaibo <495810242@qq.com> Date: Fri, 4 Apr 2025 17:20:07 +0800 Subject: [PATCH 17/17] rename `parse_avc_b_frame` and `parse_hevc_b_frame` to `parse_avc_bframe` and `parse_hevc_bframe` --- trunk/src/app/srs_app_rtc_source.cpp | 4 ++-- trunk/src/kernel/srs_kernel_codec.cpp | 4 ++-- trunk/src/kernel/srs_kernel_codec.hpp | 4 ++-- trunk/src/utest/srs_utest_kernel.cpp | 26 +++++++++++++------------- 4 files changed, 19 insertions(+), 19 deletions(-) diff --git a/trunk/src/app/srs_app_rtc_source.cpp b/trunk/src/app/srs_app_rtc_source.cpp index f587181f1f..36dc860c82 100644 --- a/trunk/src/app/srs_app_rtc_source.cpp +++ b/trunk/src/app/srs_app_rtc_source.cpp @@ -1167,11 +1167,11 @@ srs_error_t SrsRtcRtpBuilder::filter(SrsSharedPtrMessage* msg, SrsFormat* format if (!keep_bframe) { bool is_b_frame = false; if (format->vcodec->id == SrsVideoCodecIdAVC) { - if ((err = SrsVideoFrame::parse_avc_b_frame(sample, is_b_frame)) != srs_success) { + if ((err = SrsVideoFrame::parse_avc_bframe(sample, is_b_frame)) != srs_success) { return srs_error_wrap(err, "parse bframe"); } } else if (format->vcodec->id == SrsVideoCodecIdHEVC) { - if ((err = SrsVideoFrame::parse_hevc_b_frame(sample, format, is_b_frame)) != srs_success) { + if ((err = SrsVideoFrame::parse_hevc_bframe(sample, format, is_b_frame)) != srs_success) { return srs_error_wrap(err, "parse bframe"); } } diff --git a/trunk/src/kernel/srs_kernel_codec.cpp b/trunk/src/kernel/srs_kernel_codec.cpp index 093e302a25..f1a73ae326 100644 --- a/trunk/src/kernel/srs_kernel_codec.cpp +++ b/trunk/src/kernel/srs_kernel_codec.cpp @@ -712,7 +712,7 @@ srs_error_t SrsVideoFrame::parse_avc_nalu_type(const SrsSample* sample, SrsAvcNa return err; } -srs_error_t SrsVideoFrame::parse_avc_b_frame(const SrsSample* sample, bool& is_b_frame) +srs_error_t SrsVideoFrame::parse_avc_bframe(const SrsSample* sample, bool& is_b_frame) { srs_error_t err = srs_success; @@ -765,7 +765,7 @@ srs_error_t SrsVideoFrame::parse_hevc_nalu_type(const SrsSample* sample, SrsHevc return err; } -srs_error_t SrsVideoFrame::parse_hevc_b_frame(const SrsSample* sample, SrsFormat *format, bool& is_b_frame) +srs_error_t SrsVideoFrame::parse_hevc_bframe(const SrsSample* sample, SrsFormat *format, bool& is_b_frame) { srs_error_t err = srs_success; diff --git a/trunk/src/kernel/srs_kernel_codec.hpp b/trunk/src/kernel/srs_kernel_codec.hpp index 535100bfe3..f267fdc017 100644 --- a/trunk/src/kernel/srs_kernel_codec.hpp +++ b/trunk/src/kernel/srs_kernel_codec.hpp @@ -1339,10 +1339,10 @@ class SrsVideoFrame : public SrsFrame virtual SrsVideoCodecConfig* vcodec(); public: static srs_error_t parse_avc_nalu_type(const SrsSample* sample, SrsAvcNaluType& avc_nalu_type); - static srs_error_t parse_avc_b_frame(const SrsSample* sample, bool& is_b_frame); + static srs_error_t parse_avc_bframe(const SrsSample* sample, bool& is_b_frame); static srs_error_t parse_hevc_nalu_type(const SrsSample* sample, SrsHevcNaluType& hevc_nalu_type); - static srs_error_t parse_hevc_b_frame(const SrsSample* sample, SrsFormat* format, bool& is_b_frame); + static srs_error_t parse_hevc_bframe(const SrsSample* sample, SrsFormat* format, bool& is_b_frame); }; /** diff --git a/trunk/src/utest/srs_utest_kernel.cpp b/trunk/src/utest/srs_utest_kernel.cpp index 17992957cf..7b0b51a652 100644 --- a/trunk/src/utest/srs_utest_kernel.cpp +++ b/trunk/src/utest/srs_utest_kernel.cpp @@ -3615,7 +3615,7 @@ VOID TEST(KernelCodecTest, VideoFrameH264) SrsSample sample((char*)data, sizeof(data)); bool is_b_frame = false; - HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_avc_b_frame(&sample, is_b_frame)); + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_avc_bframe(&sample, is_b_frame)); EXPECT_TRUE(is_b_frame); // Non-B Frame, slice_type=0(P Frame) @@ -3623,7 +3623,7 @@ VOID TEST(KernelCodecTest, VideoFrameH264) SrsSample sample2((char*)data2, sizeof(data2)); is_b_frame = true; - HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_avc_b_frame(&sample2, is_b_frame)); + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_avc_bframe(&sample2, is_b_frame)); EXPECT_FALSE(is_b_frame); // SPS @@ -3631,7 +3631,7 @@ VOID TEST(KernelCodecTest, VideoFrameH264) SrsSample sample3((char*)data3, sizeof(data3)); is_b_frame = true; - HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_avc_b_frame(&sample3, is_b_frame)); + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_avc_bframe(&sample3, is_b_frame)); EXPECT_FALSE(is_b_frame); // PPS @@ -3639,7 +3639,7 @@ VOID TEST(KernelCodecTest, VideoFrameH264) SrsSample sample4((char*)data4, sizeof(data4)); is_b_frame = true; - HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_avc_b_frame(&sample4, is_b_frame)); + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_avc_bframe(&sample4, is_b_frame)); EXPECT_FALSE(is_b_frame); // IDR @@ -3647,12 +3647,12 @@ VOID TEST(KernelCodecTest, VideoFrameH264) SrsSample sample5((char*)data5, sizeof(data5)); is_b_frame = true; - HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_avc_b_frame(&sample5, is_b_frame)); + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_avc_bframe(&sample5, is_b_frame)); EXPECT_FALSE(is_b_frame); // Empty Sample SrsSample empty_sample(NULL, 0); - HELPER_EXPECT_FAILED(SrsVideoFrame::parse_avc_b_frame(&empty_sample, is_b_frame)); + HELPER_EXPECT_FAILED(SrsVideoFrame::parse_avc_bframe(&empty_sample, is_b_frame)); } } @@ -3716,7 +3716,7 @@ VOID TEST(KernelCodecTest, VideoFrameH265) SrsSample sample((char*)data, sizeof(data)); bool is_b_frame = false; - HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_b_frame(&sample, &format, is_b_frame)); + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_bframe(&sample, &format, is_b_frame)); EXPECT_TRUE(is_b_frame); // Non-B Frame, slice_type=1(P Frame) @@ -3724,7 +3724,7 @@ VOID TEST(KernelCodecTest, VideoFrameH265) SrsSample sample2((char*)data2, sizeof(data2)); is_b_frame = true; - HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_b_frame(&sample2, &format, is_b_frame)); + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_bframe(&sample2, &format, is_b_frame)); EXPECT_FALSE(is_b_frame); // VPS @@ -3732,7 +3732,7 @@ VOID TEST(KernelCodecTest, VideoFrameH265) SrsSample sample3((char*)data3, sizeof(data3)); is_b_frame = true; - HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_b_frame(&sample3, &format, is_b_frame)); + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_bframe(&sample3, &format, is_b_frame)); EXPECT_FALSE(is_b_frame); // SPS @@ -3740,7 +3740,7 @@ VOID TEST(KernelCodecTest, VideoFrameH265) SrsSample sample4((char*)data4, sizeof(data4)); is_b_frame = true; - HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_b_frame(&sample4, &format, is_b_frame)); + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_bframe(&sample4, &format, is_b_frame)); EXPECT_FALSE(is_b_frame); // PPS @@ -3748,7 +3748,7 @@ VOID TEST(KernelCodecTest, VideoFrameH265) SrsSample sample5((char*)data5, sizeof(data5)); is_b_frame = true; - HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_b_frame(&sample5, &format, is_b_frame)); + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_bframe(&sample5, &format, is_b_frame)); EXPECT_FALSE(is_b_frame); // IDR @@ -3756,12 +3756,12 @@ VOID TEST(KernelCodecTest, VideoFrameH265) SrsSample sample6((char*)data6, sizeof(data6)); is_b_frame = true; - HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_b_frame(&sample6, &format, is_b_frame)); + HELPER_EXPECT_SUCCESS(SrsVideoFrame::parse_hevc_bframe(&sample6, &format, is_b_frame)); EXPECT_FALSE(is_b_frame); // Empty Sample SrsSample empty_sample(NULL, 0); - HELPER_EXPECT_FAILED(SrsVideoFrame::parse_hevc_b_frame(&empty_sample, &format, is_b_frame)); + HELPER_EXPECT_FAILED(SrsVideoFrame::parse_hevc_bframe(&empty_sample, &format, is_b_frame)); } } #endif