diff --git a/erizo/generateProject.sh b/erizo/generateProject.sh index f06073bba..bd8810c17 100755 --- a/erizo/generateProject.sh +++ b/erizo/generateProject.sh @@ -19,6 +19,20 @@ EOF } +usage() +{ +cat << EOF +usage: $0 options + +Generate Erizo projects. It will generate all builds if no option is passed. + +OPTIONS: + -h Show this message + -d Generate debug + -r Generate release +EOF +} + generateVersion() { echo "generating $1" BIN_DIR="$BASE_BIN_DIR/$1" @@ -58,4 +72,28 @@ while getopts “hdr” OPTION done fi -generateVersion release +if [ "$#" -eq 0 ] +then + generateVersion debug + generateVersion release +else +while getopts “hdr” OPTION + do + case $OPTION in + h) + usage + exit 1 + ;; + d) + generateVersion debug + ;; + r) + generateVersion release + ;; + ?) + usage + exit + ;; + esac + done +fi diff --git a/erizo/src/erizo/MediaStream.cpp b/erizo/src/erizo/MediaStream.cpp index 4a1db9bb4..c3dc5f9b4 100644 --- a/erizo/src/erizo/MediaStream.cpp +++ b/erizo/src/erizo/MediaStream.cpp @@ -37,6 +37,7 @@ #include "rtp/PliPriorityHandler.h" #include "rtp/PliPacerHandler.h" #include "rtp/RtpPaddingGeneratorHandler.h" +#include "rtp/StreamSwitchHandler.h" #include "rtp/RtpUtils.h" #include "rtp/PacketCodecParser.h" @@ -462,6 +463,7 @@ void MediaStream::initializePipeline() { addHandlerInPosition(AFTER_READER, handler_pointer_dic, handler_order); pipeline_->addFront(std::make_shared()); pipeline_->addFront(std::make_shared()); + pipeline_->addFront(std::make_shared()); pipeline_->addFront(std::make_shared()); pipeline_->addFront(std::make_shared()); pipeline_->addFront(std::make_shared()); @@ -574,6 +576,12 @@ void MediaStream::onTransportData(std::shared_ptr incoming_packet, T char* buf = packet->data; RtpHeader *head = reinterpret_cast (buf); RtcpHeader *chead = reinterpret_cast (buf); + + if (chead->isFeedback()) { + if (RtpUtils::isPLI(packet)) { + ELOG_WARN("Received PLI from subscriber%s", stream_ptr->getLabel()); + } + } if (!chead->isFeedback()) { uint32_t recv_ssrc; if (chead->isRtcp()) { diff --git a/erizo/src/erizo/MediaStream.h b/erizo/src/erizo/MediaStream.h index a2fe83190..67000c23f 100644 --- a/erizo/src/erizo/MediaStream.h +++ b/erizo/src/erizo/MediaStream.h @@ -31,6 +31,20 @@ namespace erizo { +class MediaStreamSwitchEvent : public MediaEvent { + public: + MediaStreamSwitchEvent(bool is_connected, bool has_audio, bool has_video) + : is_connected{is_connected}, has_audio{has_audio}, has_video{has_video} {} + + std::string getType() const override { + return "MediaStreamSwitchEvent"; + } + + bool is_connected; + bool has_audio; + bool has_video; +}; + class MediaStreamStatsListener { public: virtual ~MediaStreamStatsListener() { diff --git a/erizo/src/erizo/OneToManyProcessor.cpp b/erizo/src/erizo/OneToManyProcessor.cpp index 0ccce4c1c..724429766 100644 --- a/erizo/src/erizo/OneToManyProcessor.cpp +++ b/erizo/src/erizo/OneToManyProcessor.cpp @@ -150,8 +150,11 @@ namespace erizo { const std::string& peer_id) { ELOG_DEBUG("Adding subscriber"); boost::mutex::scoped_lock lock(monitor_mutex_); + if (!subscriber_stream) { + return; + } ELOG_DEBUG("From %u, %u ", publisher_->getAudioSourceSSRC(), publisher_->getVideoSourceSSRC()); - ELOG_DEBUG("Subscribers ssrcs: Audio %u, video, %u from %u, %u ", + ELOG_WARN("Subscribers ssrcs: Audio %u, video, %u from %u, %u ", subscriber_stream->getAudioSinkSSRC(), subscriber_stream->getVideoSinkSSRC(), publisher_->getAudioSourceSSRC() , publisher_->getVideoSourceSSRC()); std::shared_ptr fbsource = subscriber_stream->getFeedbackSource().lock(); @@ -166,6 +169,9 @@ namespace erizo { subscribers_.erase(peer_id); } subscribers_[peer_id] = subscriber_stream; + bool has_audio = publisher_->getAudioSourceSSRC() != kDefaultAudioSinkSSRC; + bool has_video = publisher_->getVideoSourceSSRC() != kDefaultVideoSinkSSRC; + subscriber_stream->deliverEvent(std::make_shared(true, has_audio, has_video)); } std::shared_ptr OneToManyProcessor::getSubscriber(const std::string& peer_id) { @@ -179,7 +185,9 @@ namespace erizo { void OneToManyProcessor::removeSubscriber(const std::string& peer_id) { ELOG_DEBUG("Remove subscriber %s", peer_id.c_str()); boost::mutex::scoped_lock lock(monitor_mutex_); - if (subscribers_.find(peer_id) != subscribers_.end()) { + auto subscriber_it = subscribers_.find(peer_id); + if (subscriber_it != subscribers_.end()) { + subscriber_it->second->deliverEvent(std::make_shared(false, false, false)); subscribers_.erase(peer_id); } } diff --git a/erizo/src/erizo/SdpInfo.cpp b/erizo/src/erizo/SdpInfo.cpp index 2fc1103cb..d21e8ed48 100644 --- a/erizo/src/erizo/SdpInfo.cpp +++ b/erizo/src/erizo/SdpInfo.cpp @@ -106,7 +106,17 @@ namespace erizo { return nullptr; } - RtpMap *SdpInfo::getCodecByName(const std::string codecName, const unsigned int clockRate) { + RtpMap *SdpInfo::getCodecByName(const std::string codecName) { + for (unsigned int it = 0; it < internalPayloadVector_.size(); it++) { + RtpMap& rtp = internalPayloadVector_[it]; + if (rtp.encoding_name == codecName) { + return &rtp; + } + } + return NULL; + } + + RtpMap *SdpInfo::getCodecByNameAndClockRate(const std::string codecName, const unsigned int clockRate) { for (unsigned int it = 0; it < internalPayloadVector_.size(); it++) { RtpMap& rtp = internalPayloadVector_[it]; if (rtp.encoding_name == codecName && rtp.clock_rate == clockRate) { @@ -117,7 +127,7 @@ namespace erizo { } bool SdpInfo::supportCodecByName(const std::string codecName, const unsigned int clockRate) { - RtpMap *rtp = getCodecByName(codecName, clockRate); + RtpMap *rtp = getCodecByNameAndClockRate(codecName, clockRate); if (rtp != NULL) { return supportPayloadType(rtp->payload_type); } @@ -128,7 +138,7 @@ namespace erizo { if (inOutPTMap.count(payloadType) > 0) { for (unsigned int it = 0; it < payloadVector.size(); it++) { const RtpMap& rtp = payloadVector[it]; - if (inOutPTMap[rtp.payload_type] == payloadType) { + if (inOutPTMap[payloadType] == rtp.payload_type) { return true; } } diff --git a/erizo/src/erizo/SdpInfo.h b/erizo/src/erizo/SdpInfo.h index a28b77f25..49d1e1cc5 100644 --- a/erizo/src/erizo/SdpInfo.h +++ b/erizo/src/erizo/SdpInfo.h @@ -271,7 +271,9 @@ class SdpInfo { std::string getPassword(MediaType media) const; - RtpMap* getCodecByName(const std::string codecName, const unsigned int clockRate); + RtpMap* getCodecByName(const std::string codecName); + + RtpMap* getCodecByNameAndClockRate(const std::string codecName, const unsigned int clockRate); bool supportCodecByName(const std::string codecName, const unsigned int clockRate); diff --git a/erizo/src/erizo/WebRtcConnection.cpp b/erizo/src/erizo/WebRtcConnection.cpp index aa7693d00..15e172e51 100644 --- a/erizo/src/erizo/WebRtcConnection.cpp +++ b/erizo/src/erizo/WebRtcConnection.cpp @@ -1279,6 +1279,15 @@ void WebRtcConnection::write(std::shared_ptr packet) { if (transport == nullptr) { return; } + if (packet->type == VIDEO_PACKET && packet->is_keyframe) { + ELOG_WARN("Sending keyframe!"); + } + + RtcpHeader *chead = reinterpret_cast (packet->data); + if (chead->isRtcp() && chead->getPacketType() == RTCP_PS_Feedback_PT && chead->getBlockCount() == RTCP_PLI_FMT) { + ELOG_WARN("Sending PLI!"); + } + extension_processor_.processRtpExtensions(packet); transport->write(packet->data, packet->length); } diff --git a/erizo/src/erizo/rtp/BandwidthEstimationHandler.cpp b/erizo/src/erizo/rtp/BandwidthEstimationHandler.cpp index 03b08f074..bf9a59a85 100644 --- a/erizo/src/erizo/rtp/BandwidthEstimationHandler.cpp +++ b/erizo/src/erizo/rtp/BandwidthEstimationHandler.cpp @@ -104,6 +104,9 @@ void BandwidthEstimationHandler::updateExtensionMap(bool is_video, std::arrayfireRead(RtpUtils::createPLI(video_sink_ssrc_, video_source_ssrc_)); } diff --git a/erizo/src/erizo/rtp/PeriodicPliHandler.cpp b/erizo/src/erizo/rtp/PeriodicPliHandler.cpp index 4ad8bedd2..cd35035df 100644 --- a/erizo/src/erizo/rtp/PeriodicPliHandler.cpp +++ b/erizo/src/erizo/rtp/PeriodicPliHandler.cpp @@ -65,6 +65,7 @@ void PeriodicPliHandler::write(Context *ctx, std::shared_ptr packet) } void PeriodicPliHandler::sendPLI() { + ELOG_WARN("Sending PLI from PeriodicPliHandler"); getContext()->fireWrite(RtpUtils::createPLI(video_source_ssrc_, video_sink_ssrc_, HIGH_PRIORITY)); } diff --git a/erizo/src/erizo/rtp/PliPacerHandler.cpp b/erizo/src/erizo/rtp/PliPacerHandler.cpp index 8377161f3..03605ed0d 100644 --- a/erizo/src/erizo/rtp/PliPacerHandler.cpp +++ b/erizo/src/erizo/rtp/PliPacerHandler.cpp @@ -46,6 +46,7 @@ void PliPacerHandler::read(Context *ctx, std::shared_ptr packet) { } void PliPacerHandler::sendPLI() { + ELOG_WARN("Sending PLI in PliPacerHandler"); getContext()->fireWrite(RtpUtils::createPLI(video_source_ssrc_, video_sink_ssrc_)); scheduleNextPLI(); } diff --git a/erizo/src/erizo/rtp/PliPriorityHandler.cpp b/erizo/src/erizo/rtp/PliPriorityHandler.cpp index c42d74812..e9de0cefd 100644 --- a/erizo/src/erizo/rtp/PliPriorityHandler.cpp +++ b/erizo/src/erizo/rtp/PliPriorityHandler.cpp @@ -59,6 +59,7 @@ void PliPriorityHandler::write(Context *ctx, std::shared_ptr packet) } void PliPriorityHandler::sendPLI() { + ELOG_WARN("Sending PLI from PliPriorityHandler"); getContext()->fireWrite(RtpUtils::createPLI(video_source_ssrc_, video_sink_ssrc_, LOW_PRIORITY)); } diff --git a/erizo/src/erizo/rtp/QualityFilterHandler.cpp b/erizo/src/erizo/rtp/QualityFilterHandler.cpp index f2bc67da8..1051eb049 100644 --- a/erizo/src/erizo/rtp/QualityFilterHandler.cpp +++ b/erizo/src/erizo/rtp/QualityFilterHandler.cpp @@ -34,6 +34,7 @@ void QualityFilterHandler::handleFeedbackPackets(const std::shared_ptrgetBlockCount() == RTCP_PLI_FMT || chead->getBlockCount() == RTCP_SLI_FMT || chead->getBlockCount() == RTCP_FIR_FMT)) { + ELOG_WARN("Handling PLI"); sendPLI(); } }); @@ -57,6 +58,7 @@ void QualityFilterHandler::read(Context *ctx, std::shared_ptr packet void QualityFilterHandler::checkLayers() { int new_spatial_layer = quality_manager_->getSpatialLayer(); if (new_spatial_layer != target_spatial_layer_ && !changing_spatial_layer_) { + ELOG_WARN("Change spatial layer %d -> %d", target_spatial_layer_, new_spatial_layer); if (new_spatial_layer > target_spatial_layer_) { sendPLI(LOW_PRIORITY); } else { @@ -80,6 +82,7 @@ bool QualityFilterHandler::checkSSRCChange(uint32_t ssrc) { } void QualityFilterHandler::sendPLI(packetPriority priority) { + ELOG_WARN("Sending PLI from QualityFilterHandler"); getContext()->fireRead(RtpUtils::createPLI(video_sink_ssrc_, video_source_ssrc_, priority)); } @@ -97,6 +100,7 @@ void QualityFilterHandler::changeSpatialLayerOnKeyframeReceived(const std::share future_spatial_layer_ = -1; changing_spatial_layer_ = false; } else if (now - time_change_started_ > kSwitchTimeout) { + ELOG_WARN("Timeout when changing spatial layer"); sendPLI(); target_spatial_layer_ = future_spatial_layer_; future_spatial_layer_ = -1; @@ -122,6 +126,7 @@ void QualityFilterHandler::updatePictureID(const std::shared_ptr &pa RtpHeader *rtp_header = reinterpret_cast(packet->data); unsigned char* start_buffer = reinterpret_cast (packet->data); start_buffer = start_buffer + rtp_header->getHeaderLength(); + packet->picture_id = new_picture_id; RtpVP8Parser::setVP8PictureID(start_buffer, packet->length - rtp_header->getHeaderLength(), new_picture_id); } } @@ -131,6 +136,7 @@ void QualityFilterHandler::updateTL0PicIdx(const std::shared_ptr &pa RtpHeader *rtp_header = reinterpret_cast(packet->data); unsigned char* start_buffer = reinterpret_cast (packet->data); start_buffer = start_buffer + rtp_header->getHeaderLength(); + packet->tl0_pic_idx = new_tl0_pic_idx; RtpVP8Parser::setVP8TL0PicIdx(start_buffer, packet->length - rtp_header->getHeaderLength(), new_tl0_pic_idx); } } @@ -233,8 +239,10 @@ void QualityFilterHandler::write(Context *ctx, std::shared_ptr packe chead->setTimestamp(sr_timestamp + timestamp_offset_); } */ + ELOG_DEBUG(" packet, ssrc: %u, sn: %u, ts: %u, pid: %d, tl0pic: %d, keyframe: %d", + ssrc, sequence_number_info.output, last_timestamp_sent_, picture_id_info.output, tl0_pic_idx_sent, + packet->is_keyframe); } - ctx->fireWrite(packet); } diff --git a/erizo/src/erizo/rtp/QualityManager.cpp b/erizo/src/erizo/rtp/QualityManager.cpp index 5746fa7ff..e70bcb5de 100644 --- a/erizo/src/erizo/rtp/QualityManager.cpp +++ b/erizo/src/erizo/rtp/QualityManager.cpp @@ -209,8 +209,9 @@ void QualityManager::selectLayer(bool try_higher_layers) { } if (next_temporal_layer != temporal_layer_ || next_spatial_layer != spatial_layer_) { - ELOG_DEBUG("message: Layer Switch, current_layer: %d/%d, new_layer: %d/%d", - spatial_layer_, temporal_layer_, next_spatial_layer, next_temporal_layer); + ELOG_WARN("message: Layer Switch, current_layer: %d/%d, new_layer: %d/%d, max_layer: %d/%d", + spatial_layer_, temporal_layer_, next_spatial_layer, next_temporal_layer, + max_active_spatial_layer_, max_active_temporal_layer_); setTemporalLayer(next_temporal_layer); setSpatialLayer(next_spatial_layer); diff --git a/erizo/src/erizo/rtp/RtpExtensionProcessor.h b/erizo/src/erizo/rtp/RtpExtensionProcessor.h index 371d29297..85112706d 100644 --- a/erizo/src/erizo/rtp/RtpExtensionProcessor.h +++ b/erizo/src/erizo/rtp/RtpExtensionProcessor.h @@ -21,7 +21,10 @@ enum RTPExtensions { TRANSPORT_CC, // http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01 PLAYBACK_TIME, // http:// www.webrtc.org/experiments/rtp-hdrext/playout-delay RTP_ID, // urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id - MID // urn:ietf:params:rtp-hdrext:sdes:mid + MID, // urn:ietf:params:rtp-hdrext:sdes:mid + VIDEO_CONTENT_TYPE, // http://www.webrtc.org/experiments/rtp-hdrext/video-content-type + VIDEO_TIMING, // http://www.webrtc.org/experiments/rtp-hdrext/video-timing + COLOR_SPACE // http://www.webrtc.org/experiments/rtp-hdrext/color-space }; class RtpExtensionProcessor{ diff --git a/erizo/src/erizo/rtp/RtpUtils.cpp b/erizo/src/erizo/rtp/RtpUtils.cpp index 18b7bea1f..26239ed24 100644 --- a/erizo/src/erizo/rtp/RtpUtils.cpp +++ b/erizo/src/erizo/rtp/RtpUtils.cpp @@ -3,6 +3,9 @@ #include #include +#include "RtpExtensionProcessor.h" +#include "rtp/RtpHeaders.h" + namespace erizo { @@ -179,10 +182,23 @@ std::shared_ptr RtpUtils::makePaddingPacket(std::shared_ptr RtpUtils::makeVP8BlackKeyframePacket(std::shared_ptr packet) { - uint8_t vp8_keyframe[] = { - (uint8_t) 0x90, (uint8_t) 0xe0, (uint8_t) 0x80, (uint8_t) 0x01, // payload header 1 - (uint8_t) 0x00, (uint8_t) 0x20, (uint8_t) 0x10, (uint8_t) 0x0f, // payload header 2 +uint8_t vp8_keyframe[] = { + // PAYLOAD DESCRIPTOR + // X=1, N=0, S=1, PID=0 + (uint8_t) 0x90, + // X: I=1, L=1, T=1, K=0 + (uint8_t) 0xe0, + // I: M=1, PictureID (1st part)=0 + (uint8_t) 0x80, + // I: PictureID (2nd part)=1 + (uint8_t) 0x01, + // L: TL0PICIDX=0 + (uint8_t) 0x00, + // T: TID=1, Y=0, KEYIDX=0 (ignore) + (uint8_t) 0x20, + // PAYLOAD HEADER + // P=0 (keyframe) + (uint8_t) 0x10, (uint8_t) 0x0f, (uint8_t) 0x00, (uint8_t) 0x9d, (uint8_t) 0x01, (uint8_t) 0x2a, (uint8_t) 0x40, (uint8_t) 0x01, (uint8_t) 0xb4, (uint8_t) 0x00, (uint8_t) 0x07, (uint8_t) 0x07, (uint8_t) 0x09, (uint8_t) 0x03, @@ -218,6 +234,7 @@ std::shared_ptr RtpUtils::makeVP8BlackKeyframePacket(std::shared_ptr (uint8_t) 0xfe, (uint8_t) 0xef, (uint8_t) 0xb9, (uint8_t) 0x00 }; +std::shared_ptr RtpUtils::makeVP8BlackKeyframePacket(std::shared_ptr packet) { uint16_t keyframe_length = sizeof(vp8_keyframe)/sizeof(vp8_keyframe[0]); erizo::RtpHeader *header = reinterpret_cast(packet->data); const uint16_t packet_length = header->getHeaderLength() + keyframe_length; @@ -231,8 +248,19 @@ std::shared_ptr RtpUtils::makeVP8BlackKeyframePacket(std::shared_ptr std::shared_ptr keyframe_packet = std::make_shared(packet->comp, packet_buffer, packet_length, packet->type); keyframe_packet->is_keyframe = true; + + keyframe_packet->picture_id = packet->picture_id; + keyframe_packet->tl0_pic_idx = packet->tl0_pic_idx; keyframe_packet->rid = packet->rid; keyframe_packet->mid = packet->mid; + keyframe_packet->priority = packet->priority; + keyframe_packet->received_time_ms = packet->received_time_ms; + keyframe_packet->compatible_spatial_layers = packet->compatible_spatial_layers; + keyframe_packet->compatible_temporal_layers = packet->compatible_spatial_layers; + keyframe_packet->ending_of_layer_frame = true; + keyframe_packet->codec = packet->codec; + keyframe_packet->clock_rate = packet->clock_rate; + keyframe_packet->is_padding = packet->is_padding; return keyframe_packet; } diff --git a/erizo/src/erizo/rtp/RtpVP8Parser.cpp b/erizo/src/erizo/rtp/RtpVP8Parser.cpp index 692bca7b7..ced1ec852 100644 --- a/erizo/src/erizo/rtp/RtpVP8Parser.cpp +++ b/erizo/src/erizo/rtp/RtpVP8Parser.cpp @@ -390,11 +390,6 @@ RTPPayloadVP8* RtpVP8Parser::parseVP8(unsigned char* data, int dataLength) { } else { vp8->frameType = kVP8PFrame; } - if (0 == ParseVP8FrameSize(vp8, dataPtr, dataLength)) { - if (vp8->frameWidth != 640) { - ELOG_WARN("VP8 Frame width changed! = %d need postprocessing", vp8->frameWidth); - } - } vp8->data = dataPtr; vp8->dataLength = (unsigned int) dataLength; diff --git a/erizo/src/erizo/rtp/StreamSwitchHandler.cpp b/erizo/src/erizo/rtp/StreamSwitchHandler.cpp new file mode 100644 index 000000000..0e00ef57e --- /dev/null +++ b/erizo/src/erizo/rtp/StreamSwitchHandler.cpp @@ -0,0 +1,357 @@ +#include "rtp/StreamSwitchHandler.h" + +#include + +#include "./MediaDefinitions.h" +#include "./MediaStream.h" +#include "rtp/RtpUtils.h" +#include "rtp/RtpVP8Parser.h" + +namespace erizo { + +constexpr uint64_t kPliPeriodMs = 1000; + +DEFINE_LOGGER(StreamSwitchHandler, "rtp.StreamSwitchHandler"); + +StreamSwitchHandler::StreamSwitchHandler(std::shared_ptr the_clock) + : stream_{nullptr}, video_ssrc_{0}, generate_video_{false}, generated_seq_number_{0}, + clock_{the_clock}, fir_seq_number_{0}, enable_plis_{true}, plis_scheduled_{false} {} + +void StreamSwitchHandler::enable() {} + +void StreamSwitchHandler::disable() {} + +void StreamSwitchHandler::notifyUpdate() { + if (stream_) { + return; + } + auto pipeline = getContext()->getPipelineShared(); + stream_ = pipeline->getService().get(); + video_ssrc_ = stream_->getVideoSourceSSRC(); + std::shared_ptr remote_sdp = stream_->getRemoteSdpInfo(); + RtpMap *pt = remote_sdp->getCodecByName("VP8"); + if (!pt) { + pt = remote_sdp->getCodecByName("VP9"); + } + if (!pt) { + pt = remote_sdp->getCodecByName("H264"); + } + if (pt) { + video_pt_ = pt->payload_type; + video_codec_name_ = pt->encoding_name; + video_clock_rate_ = pt->clock_rate; + } +} + +void StreamSwitchHandler::notifyEvent(MediaEventPtr event) { + if (event->getType() == "MediaStreamSwitchEvent") { + auto media_stream_switch_event = std::static_pointer_cast(event); + uint32_t now = getNow(); + bool is_connected = media_stream_switch_event->is_connected; + bool has_video = media_stream_switch_event->has_video; + ELOG_DEBUG("Sending PLI? %u %u %s", is_connected, has_video, stream_->getLabel()); + if (is_connected && has_video) { + enable_plis_ = true; + sendPLI(); + schedulePLI(); + } else { + enable_plis_ = false; + } + std::for_each(state_map_.begin(), state_map_.end(), + [this, now, is_connected] (std::pair> state_pair) { + auto state = state_pair.second; + if (is_connected) { + ELOG_DEBUG("Mark as switched SSRC %u %s", state_pair.first, stream_->getLabel()); + state->switched = true; + state->keyframe_received = false; + state->frame_received = false; + state->switch_called_at = now; + if (state->last_timestamp_sent_at > 0) { + uint32_t time_with_silence = now - state->last_timestamp_sent_at; + + if (state->clock_rate > 0) { + state->time_with_silence = (1 + time_with_silence) * (state->clock_rate / 1000); + ELOG_DEBUG("Adding silence of %d, %d, %d", time_with_silence, state->clock_rate, state->time_with_silence); + } + } + } else { + if (state->last_packet) { + if (state->clock_rate > 0) { + sendBlackKeyframe(state->last_packet, 1, state->clock_rate, state); + } + state->last_packet.reset(); + } + state->last_timestamp_sent_at = now; + state->time_with_silence = 0; + } + }); + } +} + +void StreamSwitchHandler::sendBlackKeyframe(std::shared_ptr incoming_packet, int additional, + uint32_t clock_rate, const std::shared_ptr &state) { + if (incoming_packet->codec == "VP8") { + auto packet = RtpUtils::makeVP8BlackKeyframePacket(incoming_packet); + packet->is_keyframe = true; + packet->compatible_temporal_layers = {0, 1, 2}; + + RtpHeader *rtp_header = reinterpret_cast(packet->data); + packet->picture_id = state->last_picture_id_received + additional; + packet->tl0_pic_idx = state->last_tl0_pic_idx_received + additional; + rtp_header->setSeqNumber(state->last_sequence_number_received + additional); + // We use a big margin for the timestamp to make sure that Chrome renders it. We were using just + // `additional` and Chrome usually dropped it because it was too close to the previous frame. + rtp_header->setTimestamp(state->last_timestamp_received + additional * (clock_rate / 1000)); + ELOG_DEBUG("Sending keyframe before switch"); + write(getContext(), packet); + } +} + +void StreamSwitchHandler::sendPLI() { + if (enable_plis_) { + ELOG_DEBUG("message: Sending PLI"); + getContext()->fireRead(RtpUtils::createPLI(stream_->getVideoSinkSSRC(), stream_->getVideoSourceSSRC())); + } +} + +void StreamSwitchHandler::schedulePLI() { + if (plis_scheduled_) { + return; + } + plis_scheduled_ = true; + std::weak_ptr weak_this = shared_from_this(); + stream_->getWorker()->scheduleEvery([weak_this] { + if (auto this_ptr = weak_this.lock()) { + bool pli_needed = false; + std::for_each(this_ptr->state_map_.begin(), this_ptr->state_map_.end(), + [&pli_needed, this_ptr] (std::pair> state_pair) { + if (!state_pair.second->keyframe_received && + state_pair.second->frame_received && + this_ptr->enable_plis_) { + pli_needed = true; + } + }); + if (pli_needed) { + this_ptr->sendPLI(); + return true; + } else { + this_ptr->plis_scheduled_ = false; + return false; + } + } + return false; + }, std::chrono::milliseconds(kPliPeriodMs)); +} + +void StreamSwitchHandler::handleFeedbackPackets(const std::shared_ptr &packet) { + bool block_packet = false; + RtpUtils::forEachRtcpBlock(packet, [this, &block_packet](RtcpHeader *chead) { + if (chead->packettype == RTCP_PS_Feedback_PT && + (chead->getBlockCount() == RTCP_PLI_FMT || + chead->getBlockCount() == RTCP_SLI_FMT || + chead->getBlockCount() == RTCP_FIR_FMT)) { + uint32_t ssrc = chead->getSourceSSRC(); + ELOG_WARN("PLI through StreamSwitchHandler!%s", stream_->getLabel()); + std::shared_ptr state = getStateForSsrc(ssrc, true); + if ((state && !state->frame_received) || !enable_plis_) { + block_packet = true; + } + } + }); + if (!block_packet) { + getContext()->fireRead(std::move(packet)); + } else { + ELOG_DEBUG("message: Blocking PLI %s", stream_->getLabel()); + } +} + +void StreamSwitchHandler::read(Context *ctx, std::shared_ptr packet) { + RtcpHeader *chead = reinterpret_cast(packet->data); + if (chead->isFeedback()) { + handleFeedbackPackets(packet); + return; + } + ctx->fireRead(std::move(packet)); +} + +uint32_t StreamSwitchHandler::getNow() { + return std::chrono::duration_cast( + clock_->now().time_since_epoch()) + .count(); +} + +void StreamSwitchHandler::storeLastPacket(const std::shared_ptr &state, + const std::shared_ptr &packet) { + state->last_packet = std::make_shared(*packet); + state->last_packet->picture_id = packet->picture_id; + state->last_packet->tl0_pic_idx = packet->tl0_pic_idx; + state->last_packet->rid = packet->rid; + state->last_packet->mid = packet->mid; + state->last_packet->priority = packet->priority; + state->last_packet->received_time_ms = packet->received_time_ms; + state->last_packet->compatible_spatial_layers = packet->compatible_spatial_layers; + state->last_packet->compatible_temporal_layers = packet->compatible_temporal_layers; + state->last_packet->ending_of_layer_frame = packet->ending_of_layer_frame; + state->last_packet->codec = packet->codec; + state->last_packet->clock_rate = packet->clock_rate; + state->last_packet->is_padding = packet->is_padding; +} + +void StreamSwitchHandler::write(Context *ctx, std::shared_ptr packet) { + RtcpHeader *chead = reinterpret_cast(packet->data); + if (!chead->isRtcp()) { + RtpHeader *rtp_header = reinterpret_cast(packet->data); + uint32_t ssrc = rtp_header->getSSRC(); + uint16_t sequence_number = rtp_header->getSeqNumber(); + uint32_t new_timestamp = rtp_header->getTimestamp(); + uint16_t picture_id = 0; + uint8_t tl0_pic_idx = 0; + uint32_t now = getNow(); + if (packet->type == VIDEO_PACKET) { + picture_id = packet->picture_id; + tl0_pic_idx = packet->tl0_pic_idx; + } + std::shared_ptr state = getStateForSsrc(ssrc, true); + + // Flag first frame received + state->frame_received = true; + + // Convert frames to keyframes until we receive the first real keyframe + if (!state->keyframe_received && packet->type == VIDEO_PACKET) { + if (packet->is_keyframe) { + uint32_t time_to_receive_first_keyframe = 0; + if (state->switch_called_at > 0) { + time_to_receive_first_keyframe = now - state->switch_called_at; + } + state->keyframe_received = true; + ELOG_INFO("Switching stream, keyframe received, ssrc: %u, time: %u, audio: 0", ssrc, time_to_receive_first_keyframe); + } else { + // Convert to keyframes until we receive a new keyframe + packet = RtpUtils::makeVP8BlackKeyframePacket(packet); + packet->compatible_temporal_layers = {0, 1, 2}; + rtp_header = reinterpret_cast(packet->data); + if (!plis_scheduled_) { + schedulePLI(); + } + } + } + if (!state->keyframe_received && packet->type == AUDIO_PACKET) { + state->keyframe_received = true; + } + + // Reset translators and calculate offsets + if (state->switched) { + uint32_t time_to_finish_stream_switch = 0; + if (state->switch_called_at > 0) { + time_to_finish_stream_switch = now - state->switch_called_at; + } + ELOG_INFO("Switching stream, ssrc: %u, time: %u, audio: %u", ssrc, time_to_finish_stream_switch, packet->type); + state->switched = false; + state->sequence_number_translator.reset(); + // state->picture_id_translator.reset(); + state->picture_id_offset = state->last_picture_id_sent - picture_id + 1; + state->timestamp_offset = state->last_timestamp_sent - new_timestamp + state->time_with_silence; + state->tl0_pic_idx_offset = state->last_tl0_pic_idx_sent - tl0_pic_idx + 1; + state->time_with_silence = 0; + state->last_timestamp_sent_at = 0; + } + + // Translations (sequence number, timestamp, and picture_id and tl0_pic_idx in video) + SequenceNumber sequence_number_info = state->sequence_number_translator.get(sequence_number, false); + if (sequence_number_info.type != SequenceNumberType::Valid) { + ELOG_DEBUG("message: Dropping packet due to wrong sequence number translation, sequenceNumber: %u", sequence_number); + return; + } + rtp_header->setSeqNumber(sequence_number_info.output); + + uint32_t timestamp = new_timestamp + state->timestamp_offset; + rtp_header->setTimestamp(timestamp); + + state->clock_rate = packet->clock_rate; + + if (packet->type == VIDEO_PACKET) { + // SequenceNumber picture_id_info = state->picture_id_translator.get(picture_id, false); + // if (picture_id_info.type != SequenceNumberType::Valid) { + // ELOG_DEBUG("message: Dropping packet due to wrong picture id translation, pictureId: %u, lastReceived: %u, lastSent: %u", picture_id, state->last_picture_id_received, state->last_picture_id_sent); + // return; + // } + + packet->picture_id = (picture_id + state->picture_id_offset) & 0x7FFF; + updatePictureID(packet, packet->picture_id); + packet->tl0_pic_idx = tl0_pic_idx + state->tl0_pic_idx_offset; + updateTL0PicIdx(packet, packet->tl0_pic_idx); + } + if (!state->initialized) { + state->last_sequence_number_sent = sequence_number_info.output; + state->last_sequence_number_received = sequence_number; + state->initialized = true; + } + // Save references if packet is the highest sequence number we received + bool is_latest_sequence_number = !RtpUtils::sequenceNumberLessThan(sequence_number_info.output, state->last_sequence_number_sent); + if (is_latest_sequence_number) { + state->last_sequence_number_sent = sequence_number_info.output; + state->last_sequence_number_received = sequence_number; + state->last_timestamp_sent = timestamp; + state->last_timestamp_received = new_timestamp; + + if (packet->type == VIDEO_PACKET) { + state->last_picture_id_sent = packet->picture_id; + state->last_picture_id_received = picture_id; + if (packet->belongsToTemporalLayer(0) && packet->is_keyframe) { + storeLastPacket(state, packet); + } + } + } + if (!state->initialized) { + state->last_tl0_pic_idx_sent = packet->tl0_pic_idx; + state->last_tl0_pic_idx_received = tl0_pic_idx; + } + if (packet->type == VIDEO_PACKET && packet->belongsToTemporalLayer(0)) { + if (RtpUtils::numberLessThan(state->last_tl0_pic_idx_sent, packet->tl0_pic_idx, 8)) { + state->last_tl0_pic_idx_sent = packet->tl0_pic_idx; + state->last_tl0_pic_idx_received = tl0_pic_idx; + } + } + if (!state->initialized) { + state->initialized = true; + } + } + + ctx->fireWrite(std::move(packet)); +} + +std::shared_ptr StreamSwitchHandler::getStateForSsrc(uint32_t ssrc, + bool should_create) { + auto state_it = state_map_.find(ssrc); + std::shared_ptr state; + if (state_it != state_map_.end()) { + // ELOG_DEBUG("Found Translator for %u, %s", ssrc, stream_->toLog()); + state = state_it->second; + } else if (should_create) { + ELOG_DEBUG("message: no Translator found creating a new one, ssrc: %u, %s", ssrc, stream_->toLog()); + state = std::make_shared(); + state_map_[ssrc] = state; + } + return state; +} + +void StreamSwitchHandler::updatePictureID(const std::shared_ptr &packet, int new_picture_id) { + if (packet->codec == "VP8") { + RtpHeader *rtp_header = reinterpret_cast(packet->data); + unsigned char* start_buffer = reinterpret_cast (packet->data); + start_buffer = start_buffer + rtp_header->getHeaderLength(); + packet->picture_id = new_picture_id; + RtpVP8Parser::setVP8PictureID(start_buffer, packet->length - rtp_header->getHeaderLength(), new_picture_id); + } +} + +void StreamSwitchHandler::updateTL0PicIdx(const std::shared_ptr &packet, uint8_t new_tl0_pic_idx) { + if (packet->codec == "VP8") { + RtpHeader *rtp_header = reinterpret_cast(packet->data); + unsigned char* start_buffer = reinterpret_cast (packet->data); + start_buffer = start_buffer + rtp_header->getHeaderLength(); + packet->tl0_pic_idx = new_tl0_pic_idx; + RtpVP8Parser::setVP8TL0PicIdx(start_buffer, packet->length - rtp_header->getHeaderLength(), new_tl0_pic_idx); + } +} +} // namespace erizo diff --git a/erizo/src/erizo/rtp/StreamSwitchHandler.h b/erizo/src/erizo/rtp/StreamSwitchHandler.h new file mode 100644 index 000000000..452411e8c --- /dev/null +++ b/erizo/src/erizo/rtp/StreamSwitchHandler.h @@ -0,0 +1,108 @@ +#ifndef ERIZO_SRC_ERIZO_RTP_STREAMSWITCHHANDLER_H_ +#define ERIZO_SRC_ERIZO_RTP_STREAMSWITCHHANDLER_H_ + +#include + +#include "./logger.h" +#include "pipeline/Handler.h" +#include "rtp/SequenceNumberTranslator.h" +#include "lib/ClockUtils.h" + +namespace erizo { + +class MediaStream; + +class TrackState { + public: + TrackState() : + switched{false}, + initialized{false}, + timestamp_offset{0}, + last_timestamp_sent{0}, + last_timestamp_received{0}, + last_timestamp_sent_at{0}, + switch_called_at{0}, + tl0_pic_idx_offset{0}, + last_tl0_pic_idx_sent{0}, + last_tl0_pic_idx_received{0}, + last_picture_id_sent{0}, + last_picture_id_received{0}, + clock_rate{1}, + keyframe_received{false}, + frame_received{false}, + last_sequence_number_sent{0}, + last_sequence_number_received{0}, + time_with_silence{0} {} + + public: + SequenceNumberTranslator sequence_number_translator; + SequenceNumberTranslator picture_id_translator; + bool switched; + bool initialized; + uint32_t timestamp_offset; + uint32_t last_timestamp_sent; + uint32_t last_timestamp_received; + uint32_t last_timestamp_sent_at; + uint32_t switch_called_at; + uint8_t tl0_pic_idx_offset; + uint8_t last_tl0_pic_idx_sent; + uint8_t last_tl0_pic_idx_received; + uint16_t last_picture_id_sent; + uint16_t last_picture_id_received; + uint16_t picture_id_offset; + uint32_t clock_rate; + std::shared_ptr last_packet; + bool keyframe_received; + bool frame_received; + uint16_t last_sequence_number_sent; + uint16_t last_sequence_number_received; + uint32_t time_with_silence; +}; + +class StreamSwitchHandler: public Handler, public std::enable_shared_from_this { + DECLARE_LOGGER(); + + public: + explicit StreamSwitchHandler(std::shared_ptr the_clock = std::make_shared()); + + void enable() override; + void disable() override; + + std::string getName() override { + return "stream-switch-handler"; + } + + void read(Context *ctx, std::shared_ptr packet) override; + void write(Context *ctx, std::shared_ptr packet) override; + void notifyUpdate() override; + void notifyEvent(MediaEventPtr event) override; + + private: + std::shared_ptr getStateForSsrc(uint32_t ssrc, bool should_create); + void storeLastPacket(const std::shared_ptr &state, const std::shared_ptr &packet); + void sendBlackKeyframe(std::shared_ptr packet, int additional, uint32_t clock_rate, const std::shared_ptr &state); + void handleFeedbackPackets(const std::shared_ptr &packet); + void sendPLI(); + void schedulePLI(); + uint32_t getNow(); + void updatePictureID(const std::shared_ptr &packet, int new_picture_id); + void updateTL0PicIdx(const std::shared_ptr &packet, uint8_t new_tl0_pic_idx); + + private: + MediaStream* stream_; + std::map> state_map_; + uint32_t video_ssrc_; + bool generate_video_; + unsigned int video_pt_; + std::string video_codec_name_; + unsigned int video_clock_rate_; + uint16_t generated_seq_number_; + std::shared_ptr clock_; + uint32_t fir_seq_number_; + bool enable_plis_; + bool plis_scheduled_; +}; + +} // namespace erizo + +#endif // ERIZO_SRC_ERIZO_RTP_STREAMSWITCHHANDLER_H_ diff --git a/erizo/src/test/rtp/SequenceNumberTranslatorTest.cpp b/erizo/src/test/rtp/SequenceNumberTranslatorTest.cpp index 6c7e10e67..45e19be1d 100644 --- a/erizo/src/test/rtp/SequenceNumberTranslatorTest.cpp +++ b/erizo/src/test/rtp/SequenceNumberTranslatorTest.cpp @@ -398,6 +398,14 @@ INSTANTIATE_TEST_CASE_P( { 0, PacketState::Reset, 0, SequenceNumberType::Skip}, { 23537, PacketState::Forward, 23537, SequenceNumberType::Valid}}), + // Keep sending after reset + std::vector({{ 5059, PacketState::Forward, 5059, SequenceNumberType::Valid}, + { 0, PacketState::Reset, 0, SequenceNumberType::Skip}, + { 5090, PacketState::Forward, 5060, SequenceNumberType::Valid}, + { 0, PacketState::Reset, 0, SequenceNumberType::Skip}, + { 0, PacketState::Reset, 0, SequenceNumberType::Skip}, + { 5390, PacketState::Forward, 5061, SequenceNumberType::Valid}}), + // Reset after having received skipped packets std::vector({{ 5059, PacketState::Skip, 5059, SequenceNumberType::Skip}, diff --git a/erizoAPI/package.json b/erizoAPI/package.json index 4386d9a97..ae85f0fb6 100644 --- a/erizoAPI/package.json +++ b/erizoAPI/package.json @@ -24,7 +24,7 @@ } ], "scripts": { - "install": "node-gyp rebuild", + "install": "node-gyp rebuild $@", "preinstall": "./lint.sh" } } diff --git a/erizo_controller/common/amqper.js b/erizo_controller/common/amqper.js index ebcd0f3a0..7382b6c29 100644 --- a/erizo_controller/common/amqper.js +++ b/erizo_controller/common/amqper.js @@ -80,7 +80,7 @@ exports.connect = (callback) => { } } catch (err) { log.error('message: error processing message, ' + - `queueName: ${clientQueue.name}, error: ${err.message}`); + `queueName: ${clientQueue.name}, error: ${err.message}`, err.stack); } }); }); diff --git a/erizo_controller/erizoAgent/log4cxx.properties b/erizo_controller/erizoAgent/log4cxx.properties index 7b55b0216..e0b6d8471 100644 --- a/erizo_controller/erizoAgent/log4cxx.properties +++ b/erizo_controller/erizoAgent/log4cxx.properties @@ -76,3 +76,4 @@ log4j.logger.rtp.PliPriorityHandler=WARN log4j.logger.rtp.RtpPaddingGeneratorHandler=WARN log4j.logger.rtp.RtpPaddingManagerHandler=WARN log4j.logger.rtp.PacketCodecParser=WARN +log4j.logger.rtp.StreamSwitchHandler=DEBUG diff --git a/erizo_controller/erizoClient/src/ErizoConnectionManager.js b/erizo_controller/erizoClient/src/ErizoConnectionManager.js index 033da8a41..37c550870 100644 --- a/erizo_controller/erizoClient/src/ErizoConnectionManager.js +++ b/erizo_controller/erizoClient/src/ErizoConnectionManager.js @@ -86,7 +86,12 @@ class ErizoConnection extends EventEmitterConst { if (this.stack.peerConnection) { this.peerConnection = this.stack.peerConnection; // For backwards compatibility this.stack.peerConnection.onaddstream = (evt) => { - this.emit(ConnectionEvent({ type: 'add-stream', stream: evt.stream })); + if (evt.stream.id.startsWith('video_tile')) { + log.info('New Tile, stream:', evt.stream); + this.emit(ConnectionEvent({ type: 'add-tile', stream: evt.stream })); + } else { + this.emit(ConnectionEvent({ type: 'add-stream', stream: evt.stream })); + } }; this.stack.peerConnection.onremovestream = (evt) => { diff --git a/erizo_controller/erizoClient/src/Room.js b/erizo_controller/erizoClient/src/Room.js index c73c739e2..cf9555cbd 100644 --- a/erizo_controller/erizoClient/src/Room.js +++ b/erizo_controller/erizoClient/src/Room.js @@ -45,6 +45,13 @@ const Room = (altIo, altConnectionHelpers, altConnectionManager, specInput) => { that.socket = socket; let remoteStreams = that.remoteStreams; let localStreams = that.localStreams; + + that.videoTiles = { + numberOfVideoTiles: 0, + erizoId: undefined, + connectionId: undefined, + }; + // Private functions const toLog = () => `roomId: ${that.roomID.length > 0 ? that.roomID : 'undefined'}`; @@ -217,6 +224,47 @@ const Room = (altIo, altConnectionHelpers, altConnectionManager, specInput) => { }); }; + const getErizoConnectionOptionsWithoutStream = (connectionId, erizoId, options, isRemote) => { + const connectionOpts = { + callback(message, streamId) { + log.debug(`message: Sending message, data: ${JSON.stringify(message)}, ${toLog()}`); + if (message && message.type && message.type === 'updatestream') { + socket.sendSDP('streamMessage', { + streamId, + erizoId, + msg: message, + browser: '' }, undefined, () => {}); + } else { + socket.sendSDP('connectionMessage', { + connectionId, + erizoId, + msg: message, + browser: '' }, undefined, () => {}); + } + }, + connectionId, + nop2p: true, + audio: options.audio, + video: options.video, + maxAudioBW: options.maxAudioBW, + maxVideoBW: options.maxVideoBW, + limitMaxAudioBW: spec.maxAudioBW, + limitMaxVideoBW: spec.maxVideoBW, + iceServers: that.iceServers, + disableIceRestart: that.disableIceRestart, + forceTurn: that.forceTurn, + p2p: false, + streamRemovedListener: onRemoteStreamRemovedListener, + isRemote, + }; + if (!isRemote) { + connectionOpts.simulcast = options.simulcast; + connectionOpts.startVideoBW = options.startVideoBW; + connectionOpts.hardMinVideoBW = options.hardMinVideoBW; + } + return connectionOpts; + }; + const getErizoConnectionOptions = (stream, connectionId, erizoId, options, isRemote) => { const connectionOpts = { callback(message, streamId = stream.getID()) { @@ -368,7 +416,7 @@ const Room = (altIo, altConnectionHelpers, altConnectionManager, specInput) => { if (connection) { connection.processSignalingMessage(arg.evt); } else { - log.warning(`message: Received signaling message to unknown connectionId, connectionId: ${arg.connectionId}, ${toLog()}`); + log.warning(`message: Received signaling message to unknown connectionId, connectionId: ${arg.connectionId}, ${arg}, ${toLog()}`); } }; @@ -1027,6 +1075,63 @@ const Room = (altIo, altConnectionHelpers, altConnectionManager, specInput) => { } }; + that.createVideoTiles = (numberOfVideoTiles, inputOptions, callback = () => {}) => { + if (!socket) { + return 'Error creating video tiles - no socket'; + } + + that.videoTiles.numberOfVideoTiles = numberOfVideoTiles; + const options = Object.assign({}, inputOptions); + options.maxVideoBW = options.maxVideoBW || spec.defaultVideoBW; + if (options.maxVideoBW > spec.maxVideoBW) { + options.maxVideoBW = spec.maxVideoBW; + } + options.encryptTransport = + (options.encryptTransport === undefined) ? true : options.encryptTransport; + + socket.sendMessage('createVideoTiles', { numberOfVideoTiles, options }, (result, { erizoId, connectionId }) => { + log.error('CreateVideoTile', result, erizoId, connectionId); + if (result) { + that.videoTiles.erizoId = erizoId; + that.videoTiles.connectionId = connectionId; + const connectionOpts = getErizoConnectionOptionsWithoutStream(connectionId, + erizoId, options, true); + const connection = that.erizoConnectionManager + .getOrBuildErizoConnection(connectionOpts, erizoId, spec.singlePC); + connection.on('connection-failed', that.dispatchEvent.bind(this)); + connection.on('add-tile', (evt) => { + that.emit(RoomEvent({ type: 'tile-added', streams: [evt.stream] })); + }); + connection.on('remove-tile', (evt) => { + that.emit(RoomEvent({ type: 'tile-removed', streams: [evt.stream] })); + }); + callback(result); + } + }); + return undefined; + }; + + that.assignStreamsToVideoTiles = (streams) => { + if (streams.length !== that.videoTiles.numberOfVideoTiles) { + return `Error assigning ${streams.length} streams to ${that.videoTiles.numberOfVideoTiles} video tiles`; + } + + if (!that.videoTiles.erizoId || !that.videoTiles.connectionId) { + return 'Error video tiles have not been generated yet'; + } + const streamIds = streams.map((streamInput) => { + const streamId = streamInput && streamInput.getID(); + if (!that.remoteStreams.has(streamId)) { + return undefined; + } + return streamId; + }); + + log.info('Assign video tiles', that.videoTiles.erizoId, streamIds); + socket.sendMessage('assignVideoTiles', { erizoId: that.videoTiles.erizoId, streamIds }); + return undefined; + }; + that.getStreamStats = (stream, callback = () => {}) => { if (!socket) { return 'Error getting stats - no socket'; diff --git a/erizo_controller/erizoClient/src/Stream.js b/erizo_controller/erizoClient/src/Stream.js index 5b1204897..8484716ca 100644 --- a/erizo_controller/erizoClient/src/Stream.js +++ b/erizo_controller/erizoClient/src/Stream.js @@ -5,7 +5,6 @@ import ConnectionHelpers from './utils/ConnectionHelpers'; import ErizoMap from './utils/ErizoMap'; import Random from './utils/Random'; import VideoPlayer from './views/VideoPlayer'; -import AudioPlayer from './views/AudioPlayer'; import Logger from './utils/Logger'; const log = Logger.module('Stream'); @@ -390,18 +389,9 @@ const Stream = (altConnectionHelpers, specInput) => { let player; const nativeStreamContainsVideo = that.stream && that.stream.getVideoTracks().length > 0; const nativeStreamContainsAudio = that.stream && that.stream.getAudioTracks().length > 0; - if (nativeStreamContainsVideo && (that.hasVideo() || that.hasScreen())) { - // Draw on HTML - if (elementID !== undefined) { - player = VideoPlayer({ id: that.getID(), - stream: that, - elementID, - options }); - that.player = player; - that.showing = true; - } - } else if (nativeStreamContainsAudio && that.hasAudio()) { - player = AudioPlayer({ id: that.getID(), + const nativeStreamContainsMedia = nativeStreamContainsVideo || nativeStreamContainsAudio; + if (nativeStreamContainsMedia && (that.hasMedia())) { + player = VideoPlayer({ id: that.getID(), stream: that, elementID, options }); diff --git a/erizo_controller/erizoClient/src/views/AudioPlayer.js b/erizo_controller/erizoClient/src/views/AudioPlayer.js index 4ade20264..d366079a3 100644 --- a/erizo_controller/erizoClient/src/views/AudioPlayer.js +++ b/erizo_controller/erizoClient/src/views/AudioPlayer.js @@ -1,7 +1,7 @@ -/* global document */ +/* global window, MediaStream */ import View from './View'; -import Bar from './Bar'; +// import Bar from './Bar'; /* * AudioPlayer represents a Licode Audio component that shows either a local or a remote Audio. @@ -9,10 +9,13 @@ import Bar from './Bar'; * A AudioPlayer is also a View component. */ +const AudioContext = window.AudioContext || window.webkitAudioContext; +const context = new AudioContext(); + const AudioPlayer = (spec) => { const that = View({}); - let onmouseover; - let onmouseout; + // let onmouseover; + // let onmouseout; // Variables @@ -26,76 +29,94 @@ const AudioPlayer = (spec) => { that.elementID = spec.elementID; - // Audio tag - that.audio = document.createElement('audio'); - that.audio.setAttribute('id', `stream${that.id}`); - that.audio.setAttribute('class', 'licode_stream'); - that.audio.setAttribute('style', 'width: 100%; height: 100%; position: absolute'); - that.audio.setAttribute('autoplay', 'autoplay'); - - if (spec.stream.local) { that.audio.volume = 0; } - - if (that.elementID !== undefined) { - // It will stop the AudioPlayer and remove it from the HTML - that.destroy = () => { - that.audio.pause(); - that.parentNode.removeChild(that.div); - }; - - onmouseover = () => { - that.bar.display(); - }; - - onmouseout = () => { - that.bar.hide(); - }; - - // Container - that.div = document.createElement('div'); - that.div.setAttribute('id', `player_${that.id}`); - that.div.setAttribute('class', 'licode_player'); - that.div.setAttribute('style', 'width: 100%; height: 100%; position: relative; ' + - 'overflow: hidden;'); - - // Check for a passed DOM node. - if (typeof that.elementID === 'object' && - typeof that.elementID.appendChild === 'function') { - that.container = that.elementID; - } else { - that.container = document.getElementById(that.elementID); - } - that.container.appendChild(that.div); - - that.parentNode = that.div.parentNode; - - that.div.appendChild(that.audio); - - // Bottom Bar - if (spec.options.bar !== false) { - that.bar = Bar({ elementID: `player_${that.id}`, - id: that.id, - stream: spec.stream, - media: that.audio, - options: spec.options }); - - that.div.onmouseover = onmouseover; - that.div.onmouseout = onmouseout; - } else { - // Expose a consistent object to manipulate the media. - that.media = that.audio; - } - } else { - // It will stop the AudioPlayer and remove it from the HTML - that.destroy = () => { - that.audio.pause(); - that.parentNode.removeChild(that.audio); - }; - - document.body.appendChild(that.audio); - that.parentNode = document.body; + // // Audio tag + // that.audio = document.createElement('audio'); + // that.audio.setAttribute('id', `stream${that.id}`); + // that.audio.setAttribute('class', 'licode_stream'); + // that.audio.setAttribute('style', 'width: 100%; height: 100%; position: absolute'); + // that.audio.setAttribute('autoplay', 'autoplay'); + + // if (spec.stream.local) { that.audio.volume = 0; } + + // if (that.elementID !== undefined) { + // // It will stop the AudioPlayer and remove it from the HTML + // that.destroy = () => { + // that.audio.pause(); + // that.parentNode.removeChild(that.div); + // }; + + // onmouseover = () => { + // that.bar.display(); + // }; + + // onmouseout = () => { + // that.bar.hide(); + // }; + + // // Container + // that.div = document.createElement('div'); + // that.div.setAttribute('id', `player_${that.id}`); + // that.div.setAttribute('class', 'licode_player'); + // that.div.setAttribute('style', 'width: 100%; height: 100%; position: relative; ' + + // 'overflow: hidden;'); + + // // Check for a passed DOM node. + // if (typeof that.elementID === 'object' && + // typeof that.elementID.appendChild === 'function') { + // that.container = that.elementID; + // } else { + // that.container = document.getElementById(that.elementID); + // } + // that.container.appendChild(that.div); + + // that.parentNode = that.div.parentNode; + + // that.div.appendChild(that.audio); + + // // Bottom Bar + // if (spec.options.bar !== false) { + // that.bar = Bar({ elementID: `player_${that.id}`, + // id: that.id, + // stream: spec.stream, + // media: that.audio, + // options: spec.options }); + + // that.div.onmouseover = onmouseover; + // that.div.onmouseout = onmouseout; + // } else { + // // Expose a consistent object to manipulate the media. + // that.media = that.audio; + // } + // } else { + // // It will stop the AudioPlayer and remove it from the HTML + // that.destroy = () => { + // that.audio.pause(); + // that.parentNode.removeChild(that.audio); + // }; + + // document.body.appendChild(that.audio); + // that.parentNode = document.body; + // } + + // that.audio.srcObject = that.stream; + let audioMediaStream; + let peerInput; + + if (!spec.stream.local) { + audioMediaStream = new MediaStream(that.stream.getAudioTracks()); + peerInput = context.createMediaStreamSource(); + // const panner = context.createPanner(); + // panner.setPosition(0, 0, 0); + // peerInput.connect(panner); + peerInput.connect(context.destination); } - that.audio.srcObject = that.stream; + that.destroy = () => { + if (!spec.stream.local) { + audioMediaStream.destroy(); + peerInput.disconnect(); + } + }; return that; }; diff --git a/erizo_controller/erizoClient/src/views/VideoPlayer.js b/erizo_controller/erizoClient/src/views/VideoPlayer.js index 5375e4b84..f979e2330 100644 --- a/erizo_controller/erizoClient/src/views/VideoPlayer.js +++ b/erizo_controller/erizoClient/src/views/VideoPlayer.js @@ -1,4 +1,4 @@ -/* global document */ +/* global document, MediaStream */ import View from './View'; import Bar from './Bar'; @@ -36,6 +36,7 @@ const VideoPlayer = (spec) => { // It will stop the VideoPlayer and remove it from the HTML that.destroy = () => { that.video.pause(); + that.audio.pause(); that.parentNode.removeChild(that.div); }; @@ -68,7 +69,28 @@ const VideoPlayer = (spec) => { that.video.setAttribute('autoplay', 'autoplay'); that.video.setAttribute('playsinline', 'playsinline'); - if (spec.stream.local) { that.video.volume = 0; } + // Audio tag + that.audio = document.createElement('audio'); + that.audio.setAttribute('id', `audio${that.id}`); + that.audio.setAttribute('class', 'licode_stream'); + that.audio.setAttribute('style', 'width: 0%; height: 0%; position: absolute;'); + that.audio.setAttribute('autoplay', 'autoplay'); + that.audio.setAttribute('playsinline', 'playsinline'); + + that.dataAlreadyLoaded = false; + const onloadeddata = () => { + if (!that.dataAlreadyLoaded) { + that.dataAlreadyLoaded = true; + if (that.onloadeddata instanceof Function) { + that.onloadeddata(); + } + } + }; + + that.video.onloadeddata = onloadeddata; + that.audio.onloadeddata = onloadeddata; + + if (spec.stream.local) { that.audio.volume = 0; } if (that.elementID !== undefined) { // Check for a passed DOM node. @@ -89,6 +111,7 @@ const VideoPlayer = (spec) => { that.div.appendChild(that.loader); } that.div.appendChild(that.video); + that.div.appendChild(that.audio); that.containerWidth = 0; that.containerHeight = 0; @@ -108,7 +131,8 @@ const VideoPlayer = (spec) => { that.media = that.video; } - that.video.srcObject = that.stream; + that.video.srcObject = new MediaStream(that.stream.getVideoTracks()); + that.audio.srcObject = new MediaStream(that.stream.getAudioTracks()); return that; }; diff --git a/erizo_controller/erizoController/models/Client.js b/erizo_controller/erizoController/models/Client.js index c252783dd..7dca4eff0 100644 --- a/erizo_controller/erizoController/models/Client.js +++ b/erizo_controller/erizoController/models/Client.js @@ -47,6 +47,8 @@ class Client extends events.EventEmitter { this.socketEventListeners.set('stopRecorder', this.onStopRecorder.bind(this)); this.socketEventListeners.set('unpublish', this.onUnpublish.bind(this)); this.socketEventListeners.set('unsubscribe', this.onUnsubscribe.bind(this)); + this.socketEventListeners.set('createVideoTiles', this.onCreateVideoTiles.bind(this)); + this.socketEventListeners.set('assignVideoTiles', this.onAssignVideoTiles.bind(this)); this.socketEventListeners.set('getStreamStats', this.onGetStreamStats.bind(this)); this.socketEventListeners.set('clientDisconnection', this.onClientDisconnection.bind(this)); this.socketEventListeners.set('setStreamPriorityStrategy', this.onSetStreamPriorityStrategy.bind(this)); @@ -722,6 +724,32 @@ class Client extends events.EventEmitter { } } + onCreateVideoTiles({ options, numberOfVideoTiles }, callback) { + log.debug(`message: create video tiles, clientId: ${this.id}`); + if (!this.hasPermission(Permission.SUBSCRIBE)) { + log.info('message: unauthorized createVideoTiles request'); + if (callback) callback(null, 'Unauthorized'); + return; + } + options.mediaConfiguration = this.token.mediaConfiguration; + options.singlePC = this.options.singlePC || false; + options.unifiedPlan = this.options.unifiedPlan || false; + options.streamPriorityStrategy = this.options.streamPriorityStrategy; + if (this.room !== undefined && !this.room.p2p) { + this.room.controller.createVideoTiles(this.id, numberOfVideoTiles, options, (signMess) => { + log.debug(`message: callback create video tiles, clientId: ${this.id}, connectionId: ${signMess.connectionId}`); + if (callback) callback(true, signMess); + }); + } + } + + onAssignVideoTiles({ streamIds, erizoId }) { + log.info(`message: assign video tiles, clientId: ${this.id}, erizoId: ${erizoId}, length: ${streamIds.length}`); + if (this.room !== undefined && !this.room.p2p) { + this.room.controller.assignVideoTiles(erizoId, this.id, streamIds); + } + } + onClientDisconnection() { log.info(`message: Client requests disconnection, clientId: ${this.id},`, logger.objectToLog(this.token)); diff --git a/erizo_controller/erizoController/roomController.js b/erizo_controller/erizoController/roomController.js index 07e656eb7..7712f9314 100644 --- a/erizo_controller/erizoController/roomController.js +++ b/erizo_controller/erizoController/roomController.js @@ -363,6 +363,35 @@ exports.RoomController = (spec) => { } }); }; + that.createVideoTiles = (clientId, numberOfVideoTiles, options, callback = () => {}) => { + // We create a new ErizoJS with the streamId. + getErizoJS((erizoId, agentId) => { + if (erizoId === 'timeout') { + log.error(`message: createVideoTiles ErizoAgent timeout, clientId: ${clientId},`); + callback('timeout-agent'); + return; + } + log.info('message: createVideoTiles erizoJs assigned, ', + `clientId: ${clientId}, erizoId: ${erizoId}, agentId: ${agentId}`); + + const args = [clientId, numberOfVideoTiles, options]; + amqper.callRpc(getErizoQueueFromErizoId(erizoId), 'createVideoTiles', args, { + callback: (data) => { + data.erizoId = erizoId; + log.info('message: createVideoTiles finished, ' + + `response: ${JSON.stringify(data)}, ` + + `clientId: ${clientId}`); + callback(data); + }, + }); + }); + }; + + that.assignVideoTiles = (erizoId, clientId, streamIds) => { + const args = [clientId, streamIds]; + amqper.callRpc(getErizoQueueFromErizoId(erizoId), 'assignVideoTiles', args); + }; + that.getStreamStats = (streamId, callback) => { if (!streamManager.hasPublishedStream(streamId)) { log.warn('message: getStreamStats publisher not found, ' + diff --git a/erizo_controller/erizoJS/erizoJSController.js b/erizo_controller/erizoJS/erizoJSController.js index 8a95a9d8e..b7158e05c 100644 --- a/erizo_controller/erizoJS/erizoJSController.js +++ b/erizo_controller/erizoJS/erizoJSController.js @@ -4,8 +4,11 @@ const perfHooks = require('perf_hooks'); const logger = require('./../common/logger').logger; const amqper = require('./../common/amqper'); const RovReplManager = require('./../common/ROV/rovReplManager').RovReplManager; +const SdpInfo = require('./../common/semanticSdp/SDPInfo'); +const MediaInfo = require('./../common/semanticSdp/MediaInfo'); const Client = require('./models/Client').Client; const Publisher = require('./models/Publisher').Publisher; +const Subscriber = require('./models/Subscriber').Subscriber; const ExternalInput = require('./models/Publisher').ExternalInput; const PublisherManager = require('./models/PublisherManager').PublisherManager; const PerformanceStats = require('../common/PerformanceStats'); @@ -349,7 +352,8 @@ exports.ErizoJSController = (erizoJSId, threadPool, ioThreadPool) => { const connection = client.getOrCreateConnection(options); // eslint-disable-next-line no-param-reassign options.label = publisher.label; - subscriber = publisher.addSubscriber(clientId, connection, options); + subscriber = new Subscriber(clientId, publisher.streamId, connection, publisher, options); + publisher.addSubscriber(subscriber, options); subscriber.initMediaStream(); subscriber.copySdpInfoFromPublisher(); @@ -466,6 +470,103 @@ exports.ErizoJSController = (erizoJSId, threadPool, ioThreadPool) => { return Promise.all(closePromises); }; + that.createVideoTiles = (clientId, numberOfVideoTiles, inputOptions, callbackRpc) => { + log.info('message: creating video tiles, clientId:', clientId, ', videoTiles:', numberOfVideoTiles, + ', options:', inputOptions); + if (clients.has(clientId)) { + try { + const client = clients.get(clientId); + const commonOptions = Object.assign({}, inputOptions); + commonOptions.audio = true; + commonOptions.video = true; + commonOptions.publicIP = that.publicIP; + commonOptions.privateRegexp = that.privateRegexp; + commonOptions.isRemote = false; + + const connection = client.getOrCreateConnection(commonOptions); + + const tiles = client.getTiles(); + + if (tiles.length > numberOfVideoTiles) { + // TODO(javier): Remove latest video tiles + } + + const sdpInfo = new SdpInfo(); + const audio = new MediaInfo(0, 0, 'audio'); + audio.addExtension(1, 'urn:ietf:params:rtp-hdrext:ssrc-audio-level'); + audio.addExtension(2, 'http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time'); + + const video = new MediaInfo(0, 0, 'video'); + video.addExtension(14, 'urn:ietf:params:rtp-hdrext:toffset'); + video.addExtension(2, 'http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time'); + video.addExtension(13, 'urn:3gpp:video-orientation'); + video.addExtension(12, 'http://www.webrtc.org/experiments/rtp-hdrext/playout-delay'); + video.addExtension(11, 'http://www.webrtc.org/experiments/rtp-hdrext/video-content-type'); + video.addExtension(7, 'http://www.webrtc.org/experiments/rtp-hdrext/video-timing'); + video.addExtension(8, 'http://www.webrtc.org/experiments/rtp-hdrext/color-space'); + + sdpInfo.addMedia(audio); + sdpInfo.addMedia(video); + + for (let videoTile = tiles.length; videoTile < numberOfVideoTiles; videoTile += 1) { + const options = Object.assign({}, commonOptions); + options.label = `video_tile_${videoTile}`; + const subscriber = new Subscriber(clientId, `${clientId}_${videoTile}`, connection, undefined, options); + tiles.push({ subscriber, options }); + subscriber.initMediaStream(); + subscriber.configureWithSdpInfo(sdpInfo); + subscriber.on('callback', onAdaptSchemeNotify.bind(this, callbackRpc, 'callback')); + subscriber.on('periodic_stats', onPeriodicStats.bind(this, clientId, options.label)); + } + log.error('Response in createVideoTiles', connection.id); + callbackRpc('callback', { connectionId: connection.id }); + } catch (e) { + log.error('Error in createVideoTiles', e.stack); + } + } + }; + + that.assignVideoTiles = (clientId, streamIds) => { + log.info(`message: Trying to assign ${streamIds} video tiles for client ${clientId}`); + if (clients.has(clientId)) { + const client = clients.get(clientId); + const tiles = client.getTiles(); + if (streamIds.length !== tiles.length) { + const message = 'Trying to assign a wrong number of streams to tiles'; + log.warn(`message: ${message}, streams: ${streamIds.length}, tiles: ${tiles.length}, client: ${clientId}`); + return; + } + for (let index = 0; index < streamIds.length; index += 1) { + const { subscriber } = tiles[index]; + const streamId = streamIds[index]; + if (subscriber) { + const oldPublisher = subscriber.publisher; + if (oldPublisher && oldPublisher.streamId !== streamId) { + log.info(`message: Assigning tile - Removing old streamId, index: ${index}, tile: ${subscriber && subscriber.label}, streamId: ${streamId}, oldStreamId: ${oldPublisher.streamId}`); + oldPublisher.removeSubscriber(subscriber.clientId); + subscriber.publisher = undefined; + } + } else { + log.error(`message: Tile does not exist when trying to assign to it, index: ${index}`); + } + } + for (let index = 0; index < streamIds.length; index += 1) { + const { subscriber, options } = tiles[index]; + const streamId = streamIds[index]; + if (subscriber && !subscriber.publisher) { + const publisher = publisherManager.getPublisherById(streamId); + log.info(`message: Assigning tile - Adding new streamId, index: ${index}, tile: ${subscriber && subscriber.label}, streamId: ${streamId}`); + if (publisher) { + publisher.addSubscriber(subscriber, options); + } + subscriber.switchPublisher(publisher); + } else if (!subscriber) { + log.error(`message: Tile does not exist when trying to assign to it, index: ${index}`); + } + } + } + }; + that.setClientStreamPriorityStrategy = (clientId, strategyId) => { log.debug(`message: Trying to set streamPriorityStrategy to for client ${clientId}`); if (clients.has(clientId)) { diff --git a/erizo_controller/erizoJS/models/Client.js b/erizo_controller/erizoJS/models/Client.js index c18bcc70b..594a3d079 100644 --- a/erizo_controller/erizoJS/models/Client.js +++ b/erizo_controller/erizoJS/models/Client.js @@ -27,6 +27,7 @@ class Client extends EventEmitter { this.connectionTargetBw = options.connectionTargetBw || 0; this.connectionClientId = 0; this.options = options; + this.tiles = []; } static _getStreamPriorityStrategy(streamPriorityStrategy) { @@ -199,6 +200,10 @@ class Client extends EventEmitter { }); } + getTiles() { + return this.tiles; + } + setStreamPriorityStrategy(streamPriorityStrategy) { this.streamPriorityStrategy = Client._getStreamPriorityStrategy(streamPriorityStrategy); this.connectionTargetBw = this.streamPriorityStrategy.connectionTargetBw; diff --git a/erizo_controller/erizoJS/models/Publisher.js b/erizo_controller/erizoJS/models/Publisher.js index 6fa5f6c8a..f4be78bc4 100644 --- a/erizo_controller/erizoJS/models/Publisher.js +++ b/erizo_controller/erizoJS/models/Publisher.js @@ -3,7 +3,6 @@ /* eslint-disable no-param-reassign */ const NodeClass = require('./Node').Node; -const Subscriber = require('./Subscriber').Subscriber; // eslint-disable-next-line const erizo = require(`./../../../erizoAPI/build/Release/${global.config.erizo.addon}`); const logger = require('./../../common/logger').logger; @@ -42,11 +41,10 @@ class Source extends NodeClass { } } - - addSubscriber(clientId, connection, options) { + addSubscriber(subscriber, options) { + const clientId = subscriber.clientId; log.info(`message: Adding subscriber, clientId: ${clientId}, streamId ${this.streamId},`, logger.objectToLog(this.options), logger.objectToLog(this.options.metadata)); - const subscriber = new Subscriber(clientId, this.streamId, connection, this, options); this.subscribers[clientId] = subscriber; this.muxer.addSubscriber(subscriber.mediaStream, subscriber.mediaStream.id); diff --git a/erizo_controller/erizoJS/models/RTCPeerConnection.js b/erizo_controller/erizoJS/models/RTCPeerConnection.js index 8d3d03fe8..a34f0e50e 100644 --- a/erizo_controller/erizoJS/models/RTCPeerConnection.js +++ b/erizo_controller/erizoJS/models/RTCPeerConnection.js @@ -618,6 +618,10 @@ class RTCPeerConnection extends EventEmitter { this.internalConnection.copySdpInfoFromConnection(connection.internalConnection); } + configureWithSdpInfo(sdpInfo) { + this.internalConnection.configureWithSdpInfo(sdpInfo); + } + getStats(callback) { return this.internalConnection.getStats(callback); } diff --git a/erizo_controller/erizoJS/models/Subscriber.js b/erizo_controller/erizoJS/models/Subscriber.js index c8d537c42..165db384e 100644 --- a/erizo_controller/erizoJS/models/Subscriber.js +++ b/erizo_controller/erizoJS/models/Subscriber.js @@ -11,7 +11,7 @@ class Subscriber extends NodeClass { constructor(clientId, streamId, connection, publisher, options) { super(clientId, streamId, options); this.connection = connection; - this.connection.mediaConfiguration = options.mediaConfiguration; + this.connection.mediaConfiguration = options.mediaConfiguration || this.connection.mediaConfiguration || 'default'; this.promise = this.connection.addStream(this.erizoStreamId, options, false); this.onReady = new Promise((resolve, reject) => { this._readyResolveFunction = resolve; @@ -19,22 +19,42 @@ class Subscriber extends NodeClass { }); this._mediaStreamListener = this._onMediaStreamEvent.bind(this); connection.on('media_stream_event', this._mediaStreamListener); + this.alreadyCalledToInitialize = false; connection.onReady.then(() => { - if (this.clientId && this.options.browser === 'bowser') { - this.publisher.requestVideoKeyFrame(); - } - if (this.options.slideShowMode === true || - Number.isSafeInteger(this.options.slideShowMode)) { - this.publisher.setSlideShow(this.options.slideShowMode, this.clientId); - } + this._initializePublisher(); }); this.mediaStream = connection.getStream(this.erizoStreamId); this.publisher = publisher; this.setMaxVideoBW(); } + _initializePublisher() { + this.alreadyCalledToInitialize = true; + if (this.clientId && this.options.browser === 'bowser' && this.publisher) { + this.publisher.requestVideoKeyFrame(); + } + if ((this.options.slideShowMode === true || + Number.isSafeInteger(this.options.slideShowMode)) && this.publisher) { + this.publisher.setSlideShow(this.options.slideShowMode, this.clientId); + } + } + + switchPublisher(newPublisher) { + this.publisher = newPublisher; + if (this.alreadyCalledToInitialize && this.publisher) { + this._initializePublisher(); + } + this.setMaxVideoBW(); + } + copySdpInfoFromPublisher() { - this.connection.copySdpInfoFromConnection(this.publisher.connection); + if (this.publisher) { + this.connection.copySdpInfoFromConnection(this.publisher.connection); + } + } + + configureWithSdpInfo(sdpInfo) { + this.connection.configureWithSdpInfo(sdpInfo); } updatePublisherMaxVideoBW() { @@ -42,6 +62,9 @@ class Subscriber extends NodeClass { } setMaxVideoBW(maxVideoBW) { + if (!this.publisher) { + return; + } let updatedMaxVideoBW; if (maxVideoBW) { this.maxVideoBW = maxVideoBW; @@ -83,20 +106,20 @@ class Subscriber extends NodeClass { onStreamMessage(msg) { if (msg.type === 'updatestream') { if (msg.config) { - if (msg.config.slideShowMode !== undefined) { + if (msg.config.slideShowMode !== undefined && this.publisher) { this.publisher.setSlideShow(msg.config.slideShowMode, this.clientId); } - if (msg.config.muteStream !== undefined) { + if (msg.config.muteStream !== undefined && this.publisher) { this.publisher.muteStream(msg.config.muteStream, this.clientId); } - if (msg.config.qualityLayer !== undefined) { + if (msg.config.qualityLayer !== undefined && this.publisher) { this.publisher.setQualityLayer(msg.config.qualityLayer, this.clientId); } - if (msg.config.slideShowBelowLayer !== undefined) { + if (msg.config.slideShowBelowLayer !== undefined && this.publisher) { this.publisher.enableSlideShowBelowSpatialLayer( msg.config.slideShowBelowLayer, this.clientId); } - if (msg.config.video !== undefined) { + if (msg.config.video !== undefined && this.publisher) { this.publisher.setVideoConstraints(msg.config.video, this.clientId); } if (msg.config.maxVideoBW) { @@ -106,7 +129,7 @@ class Subscriber extends NodeClass { this.mediaStream.setPriority(msg.config.priorityLevel); } } - } else if (msg.type === 'control') { + } else if (msg.type === 'control' && this.publisher) { this.publisher.processControlMessage(this.clientId, msg.action); } } diff --git a/erizo_controller/erizoJS/models/WebRtcConnection.js b/erizo_controller/erizoJS/models/WebRtcConnection.js index c9ee8e3e5..0209af37b 100644 --- a/erizo_controller/erizoJS/models/WebRtcConnection.js +++ b/erizo_controller/erizoJS/models/WebRtcConnection.js @@ -375,6 +375,11 @@ class WebRtcConnection extends EventEmitter { } } + configureWithSdpInfo(sdpInfo) { + const sessionDescription = new SessionDescription(sdpInfo, this.mediaConfiguration); + this.wrtc.copySdpToLocalDescription(sessionDescription.connectionDescription); + } + _logSdp(...message) { log.debug('negotiation:', ...message, ', id:', this.id, ', lockReason: ', this.lockReason, ',', logger.objectToLog(this.options), logger.objectToLog(this.options.metadata)); diff --git a/extras/basic_example/public/index.html b/extras/basic_example/public/index.html index f0aab9580..94179f27b 100644 --- a/extras/basic_example/public/index.html +++ b/extras/basic_example/public/index.html @@ -11,6 +11,7 @@ +
@@ -23,5 +24,6 @@

Press the start button to start receiving streams

+
diff --git a/extras/basic_example/public/script.js b/extras/basic_example/public/script.js index 1bf94c406..039d17b94 100644 --- a/extras/basic_example/public/script.js +++ b/extras/basic_example/public/script.js @@ -6,6 +6,7 @@ const serverUrl = '/'; let localStream; +let dataStream; let room; let recording = false; let recordingId = ''; @@ -25,6 +26,7 @@ const configFlags = { autoSubscribe: false, simulcast: false, unencrypted: false, + numVideoTiles: 3, }; const createSubscriberContainer = (stream) => { @@ -168,6 +170,68 @@ const publish = (video, audio, screen) => { stream.init(); }; +const createVideoTile = () => { + room.createVideoTiles(configFlags.numVideoTiles, {}); +}; + +const assignVideoTiles = (streamIds) => { + console.log('Assigning video tiles', streamIds); + const streams = streamIds.map(streamId => room.remoteStreams.get(streamId)); + room.assignStreamsToVideoTiles(streams); +}; + +function shuffle(array) { + let currentIndex = array.length; + let randomIndex; + + // While there remain elements to shuffle... + while (currentIndex !== 0) { + // Pick a remaining element... + randomIndex = Math.floor(Math.random() * currentIndex); + currentIndex -= 1; + + // And swap it with the current element. + [array[currentIndex], array[randomIndex]] = [ + array[randomIndex], array[currentIndex]]; + } + + return array; +} + +const onDataFromStream = (stream, evt) => { + console.log(`Event from ${stream.getID()}, evt: ${JSON.stringify(evt.msg)}`); + if (evt.msg && evt.msg.startsWith('shuffle-video-tile-')) { + let streamIds = evt.msg.replace('shuffle-video-tile-', ''); + streamIds = streamIds.split(', '); + assignVideoTiles(streamIds); + } +}; + +// eslint-disable-next-line no-unused-vars +const applyAssignVideoTilesToEveryone = () => { + let streamIds = []; + const numVideoTiles = configFlags.numVideoTiles; + room.remoteStreams.forEach((stream) => { + if (stream.hasMedia()) { + streamIds.push(stream.getID()); + } + }); + + shuffle(streamIds); + streamIds = streamIds.splice(0, numVideoTiles); + while (streamIds.length < numVideoTiles) { + streamIds.push(undefined); + } + console.log(streamIds); + shuffle(streamIds); + + const msg = `shuffle-video-tile-${streamIds.join(', ')}`; + console.log('Applying video tiles to all', msg, dataStream.hasData()); + + dataStream.sendData(msg); + assignVideoTiles(streamIds); +}; + const startBasicExample = () => { document.getElementById('startButton').disabled = true; document.getElementById('slideShowMode').disabled = false; @@ -193,7 +257,9 @@ const startBasicExample = () => { } Erizo.Logger.setLogLevel(Erizo.Logger.TRACE); localStream = Erizo.Stream(config); + dataStream = Erizo.Stream({ audio: false, video: false, screen: false, data: true }); window.localStream = localStream; + window.dataStream = dataStream; const createToken = (roomData, callback) => { const req = new XMLHttpRequest(); const url = `${serverUrl}createToken/`; @@ -223,7 +289,7 @@ const startBasicExample = () => { const subscribeToStreams = (streams) => { streams.forEach((stream) => { - if (!stream.local) { + if (!stream.local && stream.hasMedia()) { const streamContainer = document.createElement('div'); streamContainer.setAttribute('id', `stream_element_${stream.getID()}`); const subscribeButton = document.createElement('button'); @@ -235,6 +301,10 @@ const startBasicExample = () => { }; streamContainer.appendChild(subscribeButton); document.getElementById('remoteStreamList').appendChild(streamContainer); + } else if (!stream.local && !stream.hasMedia() && stream.hasData()) { + room.subscribe(stream); + console.log('Subscribing to stream-data'); + stream.addEventListener('stream-data', onDataFromStream.bind(null, stream)); } }); @@ -249,9 +319,13 @@ const startBasicExample = () => { }; streams.forEach((stream) => { - if (localStream.getID() !== stream.getID()) { + if (!stream.local) { room.subscribe(stream, { slideShowMode, metadata: { type: 'subscriber' }, video: !configFlags.onlyAudio, encryptTransport: !configFlags.unencrypted }); stream.addEventListener('bandwidth-alert', cb); + } else if (!stream.local && !stream.hasMedia() && stream.hasData()) { + room.subscribe(stream); + console.log('Subscribing to stream-data'); + stream.addEventListener('stream-data', onDataFromStream.bind(null, stream)); } }); }; @@ -266,14 +340,53 @@ const startBasicExample = () => { if (!configFlags.onlySubscribe) { room.publish(localStream, options); } + room.publish(dataStream); room.addEventListener('quality-level', (qualityEvt) => { console.log(`New Quality Event, connection quality: ${qualityEvt.message}`); }); + createVideoTile(); + }); + + room.addEventListener('tile-added', (roomEvent) => { + console.log('Tile added'); + const streams = roomEvent.streams; + const stream = streams[0]; + const div = document.createElement('div'); + div.setAttribute('style', 'width: 320px; height: 240px;float:left;'); + div.setAttribute('id', `test${stream.id}`); + + const player = document.createElement('div'); + player.setAttribute('style', 'width: 100%; height: 100%; position: relative; background-color: black; overflow: hidden;'); + player.setAttribute('id', `player${stream.id}`); + + const video = document.createElement('video'); + video.setAttribute('id', `stream${stream.id}`); + video.setAttribute('class', 'licode_stream'); + video.setAttribute('style', 'width: 100%; height: 100%; position: absolute; object-fit: cover'); + video.setAttribute('autoplay', 'autoplay'); + video.setAttribute('playsinline', 'playsinline'); + video.srcObject = new MediaStream(stream.getVideoTracks()); + + const audio = document.createElement('audio'); + audio.setAttribute('id', `stream_audio_${stream.id}`); + audio.setAttribute('class', 'licode_stream'); + audio.setAttribute('style', 'width: 0%; height: 0%; position: absolute; object-fit: cover'); + audio.setAttribute('autoplay', 'autoplay'); + audio.setAttribute('playsinline', 'playsinline'); + audio.srcObject = new MediaStream(stream.getAudioTracks()); + + div.appendChild(player); + player.appendChild(video); + player.appendChild(audio); + + document.getElementById('videoTileContainer').appendChild(div); }); room.addEventListener('stream-subscribed', (streamEvent) => { const stream = streamEvent.stream; - createSubscriberContainer(stream); + if (stream.hasMedia()) { + createSubscriberContainer(stream); + } }); room.addEventListener('stream-unsubscribed', (streamEvent) => { diff --git a/package-lock.json b/package-lock.json index ac5768cec..007cc68e7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -170,8 +170,16 @@ "@types/node": { "version": "15.3.0", "resolved": "https://registry.npmjs.org/@types/node/-/node-15.3.0.tgz", - "integrity": "sha512-8/bnjSZD86ZfpBsDlCIkNXIvm+h6wi9g7IqL+kmFkQ+Wvu3JrasgLElfiPgoo8V8vVfnEi0QVS12gbl94h9YsQ==", - "dev": true + "integrity": "sha512-8/bnjSZD86ZfpBsDlCIkNXIvm+h6wi9g7IqL+kmFkQ+Wvu3JrasgLElfiPgoo8V8vVfnEi0QVS12gbl94h9YsQ==" + }, + "@types/yauzl": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/@types/yauzl/-/yauzl-2.9.1.tgz", + "integrity": "sha512-A1b8SU4D10uoPjwb0lnHmmu8wZhR9d+9o2PKBQT2jU5YPTKsxac6M2qGAdY7VcL+dHHhARVUDmeg0rOrcd9EjA==", + "optional": true, + "requires": { + "@types/node": "*" + } }, "@ungap/promise-all-settled": { "version": "1.1.2", @@ -404,6 +412,29 @@ } } }, + "agent-base": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz", + "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==", + "requires": { + "debug": "4" + }, + "dependencies": { + "debug": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "ajv": { "version": "5.5.2", "resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz", @@ -1165,6 +1196,11 @@ "isarray": "^1.0.0" } }, + "buffer-crc32": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", + "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" + }, "buffer-equal": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/buffer-equal/-/buffer-equal-1.0.0.tgz", @@ -2066,6 +2102,11 @@ "integrity": "sha1-9B8cEL5LAOh7XxPaaAdZ8sW/0+I=", "dev": true }, + "devtools-protocol": { + "version": "0.0.883894", + "resolved": "https://registry.npmjs.org/devtools-protocol/-/devtools-protocol-0.0.883894.tgz", + "integrity": "sha512-33idhm54QJzf3Q7QofMgCvIVSd2o9H3kQPWaKT/fhoZh+digc+WSiMhbkeG3iN79WY4Hwr9G05NpbhEVrsOYAg==" + }, "di": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/di/-/di-0.0.1.tgz", @@ -2226,7 +2267,6 @@ "version": "1.4.4", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", - "dev": true, "requires": { "once": "^1.4.0" } @@ -2913,6 +2953,49 @@ } } }, + "extract-zip": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extract-zip/-/extract-zip-2.0.1.tgz", + "integrity": "sha512-GDhU9ntwuKyGXdZBUgTIe+vXnWj0fppUEtMDL0+idd5Sta8TGpHssn/eusA9mrPr9qNDym6SxAYZjNvCn/9RBg==", + "requires": { + "@types/yauzl": "^2.9.1", + "debug": "^4.1.1", + "get-stream": "^5.1.0", + "yauzl": "^2.10.0" + }, + "dependencies": { + "debug": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "requires": { + "ms": "2.1.2" + } + }, + "get-stream": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", + "integrity": "sha512-nBF+F1rAZVCu/p7rjzgA+Yb4lfYXrpl7a6VmJrU8wF9I1CKvP/QwPNZHnOlwbTkY6dvtFIzFMSyQXbLoTQPRpA==", + "requires": { + "pump": "^3.0.0" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "pump": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.0.tgz", + "integrity": "sha512-LwZy+p3SFs1Pytd/jYct4wpv49HiYCqd9Rlc5ZVdk0V+8Yzv6jR5Blk3TRmPL1ft69TxP0IMZGJ+WPFU2BFhww==", + "requires": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + } + } + }, "extsprintf": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz", @@ -2959,6 +3042,14 @@ "integrity": "sha512-Utm6CdzT+6xsDk2m8S6uL8VHxNwI6Jub+e9NYTcAms28T84pTa25GJQV9j0CY0N1rM8hK4x6grpF2BQf+2qwVA==", "dev": true }, + "fd-slicer": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fd-slicer/-/fd-slicer-1.1.0.tgz", + "integrity": "sha1-JcfInLH5B3+IkbvmHY85Dq4lbx4=", + "requires": { + "pend": "~1.2.0" + } + }, "fecha": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/fecha/-/fecha-2.3.3.tgz", @@ -4918,6 +5009,30 @@ "integrity": "sha1-7AbBDgo0wPL68Zn3/X/Hj//QPHM=", "dev": true }, + "https-proxy-agent": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.0.tgz", + "integrity": "sha512-EkYm5BcKUGiduxzSt3Eppko+PiNWNEpa4ySk9vTC6wDsQJW9rHSa+UhGNJoRYp7bz6Ht1eaRIa6QaJqO5rCFbA==", + "requires": { + "agent-base": "6", + "debug": "4" + }, + "dependencies": { + "debug": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + } + } + }, "iconv-lite": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", @@ -7014,6 +7129,21 @@ "path-to-regexp": "^1.7.0" } }, + "node-environment-flags": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/node-environment-flags/-/node-environment-flags-1.0.5.tgz", + "integrity": "sha512-VNYPRfGfmZLx0Ye20jWzHUjyTW/c+6Wq+iLhDzUI4XmhrDd9l/FozXV3F2xOaXjvp0co0+v1YSR3CMP6g+VvLQ==", + "dev": true, + "requires": { + "object.getownpropertydescriptors": "^2.0.3", + "semver": "^5.7.0" + } + }, + "node-fetch": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.1.tgz", + "integrity": "sha512-V4aYg89jEoVRxRb2fJdAg8FHvI7cEyYdVAh94HH0UIK8oJxUfkjlDQN9RbMx+bEjP7+ggMiFRprSti032Oipxw==" + }, "node-gyp": { "version": "7.1.2", "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-7.1.2.tgz", @@ -7614,6 +7744,11 @@ "sha.js": "^2.4.8" } }, + "pend": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/pend/-/pend-1.2.0.tgz", + "integrity": "sha1-elfrVQpng/kRUzH89GY9XI4AelA=" + }, "performance-now": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz", @@ -7765,6 +7900,11 @@ "react-is": "^16.8.1" } }, + "proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" + }, "prr": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/prr/-/prr-1.0.1.tgz", @@ -7830,6 +7970,109 @@ "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz", "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A==" }, + "puppeteer": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-10.1.0.tgz", + "integrity": "sha512-bsyDHbFBvbofZ63xqF7hMhuKBX1h4WsqFIAoh1GuHr/Y9cewh+EFNAOdqWSkQRHLiBU/MY6M+8PUnXXjAPtuSg==", + "requires": { + "debug": "4.3.1", + "devtools-protocol": "0.0.883894", + "extract-zip": "2.0.1", + "https-proxy-agent": "5.0.0", + "node-fetch": "2.6.1", + "pkg-dir": "4.2.0", + "progress": "2.0.1", + "proxy-from-env": "1.1.0", + "rimraf": "3.0.2", + "tar-fs": "2.0.0", + "unbzip2-stream": "1.3.3", + "ws": "7.4.6" + }, + "dependencies": { + "debug": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "requires": { + "ms": "2.1.2" + } + }, + "find-up": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz", + "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==", + "requires": { + "locate-path": "^5.0.0", + "path-exists": "^4.0.0" + } + }, + "locate-path": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz", + "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", + "requires": { + "p-locate": "^4.1.0" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "p-limit": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", + "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", + "requires": { + "p-try": "^2.0.0" + } + }, + "p-locate": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz", + "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", + "requires": { + "p-limit": "^2.2.0" + } + }, + "p-try": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", + "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" + }, + "path-exists": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz", + "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==" + }, + "pkg-dir": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz", + "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==", + "requires": { + "find-up": "^4.0.0" + } + }, + "progress": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.1.tgz", + "integrity": "sha512-OE+a6vzqazc+K6LxJrX5UPyKFvGnL5CYmq2jFGNIBWHpc4QyE49/YOumcrpQFJpfejmvRtbJzgO1zPmMCqlbBg==" + }, + "rimraf": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", + "integrity": "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==", + "requires": { + "glob": "^7.1.3" + } + }, + "ws": { + "version": "7.4.6", + "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.6.tgz", + "integrity": "sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==" + } + } + }, "qjobs": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/qjobs/-/qjobs-1.2.0.tgz", @@ -9247,8 +9490,7 @@ "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", - "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=", - "dev": true + "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, "through2": { "version": "2.0.5", diff --git a/package.json b/package.json index 015b60aa8..315b8947b 100644 --- a/package.json +++ b/package.json @@ -65,6 +65,7 @@ "buildErizoAPI": "export ERIZO_HOME=$(pwd)/erizo/ && echo $ERIZO_HOME && cd ./erizoAPI/ && env JOBS=4 ./build.sh" }, "dependencies": { - "node-gyp": "^7.1.2" + "node-gyp": "^7.1.2", + "puppeteer": "^10.1.0" } } diff --git a/test/negotiation/index.js b/test/negotiation/index.js index bc1b81548..778c81bd3 100644 --- a/test/negotiation/index.js +++ b/test/negotiation/index.js @@ -1,4 +1,5 @@ const describeNegotiationTest = require('./utils/NegotiationTest'); +const describeStreamSwitchTest = require('./utils/StreamSwitchTest'); describeNegotiationTest('SDP negotiations started by client', function(ctx) { ctx.publishToErizoStreamStep(); @@ -14,39 +15,31 @@ describeNegotiationTest('SDP negotiations started by Erizo', function(ctx) { ctx.subscribeToErizoStreamStep(); }); -describeNegotiationTest('Conflicting SDP negotiation started by Erizo and Client', function(ctx) { - ctx.publishAndSubscribeStreamsStep([ - 'client-add-stream', - 'erizo-publish-stream', - 'erizo-subscribe-stream', - 'erizo-get-offer', - 'client-get-offer', - // Erizo is not polite 'erizo-process-offer', - 'client-process-offer', - 'client-get-answer', - 'erizo-process-answer', - 'get-and-process-candidates', - 'client-get-offer', - 'erizo-process-offer', - 'erizo-get-answer', - 'client-process-answer', - 'wait-for-being-connected', - ]); - ctx.publishAndSubscribeStreamsStep([ - 'client-add-stream', - 'erizo-publish-stream', - 'client-get-offer', - // Erizo is not polite 'erizo-process-offer', - 'erizo-subscribe-stream', - 'erizo-get-offer', - 'client-process-offer', - 'client-get-answer', - 'erizo-process-answer', - 'client-get-offer', - 'erizo-process-offer', - 'erizo-get-answer', - 'client-process-answer', - 'get-and-process-candidates', - 'wait-for-being-connected', - ]); -}); +describeStreamSwitchTest('Basic Stream Switch Test', async function(ctx) { + let publisherA, publisherB, subscriberA, subscriberB, subscriberC; + before(async function() { + publisherA = await ctx.createClientStream('streamA', undefined, 400); + publisherB = await ctx.createClientStream('streamB', "#ff0000", 1000); + publisherB = await ctx.createClientStream('streamC', "#0000ff", undefined); + + subscriberA = await ctx.createErizoStream('erizoStreamA', publisherA.label, true, false); + subscriberB = await ctx.createErizoStream('erizoStreamB', publisherB.label, true, true); + subscriberC = await ctx.createErizoStream('erizoStreamB', publisherB.label, false, true); + }); + + ctx.publishStream('pub', 'pub', 'streamA'); + ctx.publishStream('pub', 'pub', 'streamB'); + ctx.publishStream('pub', 'pub', 'streamC'); + ctx.subscribeToStream('sub', 'sub', 'erizoStreamA', 'pub'); + + for (let i = 0; i < 20; i++) { + // ctx.linkSubToPub('sub', 'pub', 'sub', 'streamA', 'erizoStreamA'); + // ctx.unlinkSubToPub('sub', 'pub', 'sub', 'streamA', 'erizoStreamA'); + ctx.linkSubToPub('sub', 'pub', 'sub', 'streamB', 'erizoStreamA'); + ctx.unlinkSubToPub('sub', 'pub', 'sub', 'streamB', 'erizoStreamA'); + ctx.linkSubToPub('sub', 'pub', 'sub', 'streamC', 'erizoStreamA'); + ctx.unlinkSubToPub('sub', 'pub', 'sub', 'streamC', 'erizoStreamA'); + ctx.linkSubToPub('sub', 'pub', 'sub', 'streamB', 'erizoStreamA'); + ctx.unlinkSubToPub('sub', 'pub', 'sub', 'streamB', 'erizoStreamA'); + } +}, true); \ No newline at end of file diff --git a/test/negotiation/log4cxx.properties b/test/negotiation/log4cxx.properties index 86bd906f6..61d718ed8 100644 --- a/test/negotiation/log4cxx.properties +++ b/test/negotiation/log4cxx.properties @@ -74,5 +74,6 @@ log4j.logger.rtp.PliPriorityHandler=WARN log4j.logger.rtp.RtpPaddingGeneratorHandler=WARN log4j.logger.rtp.RtpPaddingManagerHandler=WARN log4j.logger.rtp.PacketCodecParser=WARN +log4j.logger.rtp.StreamSwitchHandler=DEBUG diff --git a/test/negotiation/log4js_configuration.json b/test/negotiation/log4js_configuration.json index 4b3f0622a..7fbb88984 100644 --- a/test/negotiation/log4js_configuration.json +++ b/test/negotiation/log4js_configuration.json @@ -1,7 +1,7 @@ { "appenders": { "out": { - "type": "stdout", + "type": "console", "layout": { "type": "pattern", "pattern": "%d - %p: %c - %m", @@ -10,23 +10,25 @@ } }, "categories": { - "AMQPER": { "appenders": ["out"], "level": "ERROR" }, - "Client": { "appenders": ["out"], "level": "ERROR" }, - "Connection": { "appenders": ["out"], "level": "ERROR" }, - "ErizoController": { "appenders": ["out"], "level": "ERROR" }, - "ErizoJS": { "appenders": ["out"], "level": "ERROR" }, - "ErizoJSController": { "appenders": ["out"], "level": "ERROR" }, - "ErizoAgent": { "appenders": ["out"], "level": "ERROR" }, - "ErizoAgentReporter": { "appenders": ["out"], "level": "ERROR" }, - "EcCloudHandler": { "appenders": ["out"], "level": "ERROR" }, - "Publisher": { "appenders": ["out"], "level": "ERROR" }, - "Subscriber": { "appenders": ["out"], "level": "ERROR" }, - "Room": { "appenders": ["out"], "level": "ERROR" }, - "RoomController": { "appenders": ["out"], "level": "ERROR" }, - "RovClient":{ "appenders": ["out"], "level": "ERROR" }, - "RovMetricsServer":{ "appenders": ["out"], "level": "ERROR" }, - "RovReplManager":{ "appenders": ["out"], "level": "ERROR" }, - "RPCPublic":{ "appenders": ["out"], "level": "ERROR" }, - "WebRtcConnection": { "appenders": ["out"], "level": "ERROR" } - } + "default": { "appenders": ["out"], "level": "ERROR" }, + "AMQPER": { "appenders": ["out"], "level": "ERROR" }, + "Client": { "appenders": ["out"], "level": "INFO" }, + "Connection": { "appenders": ["out"], "level": "INFO" }, + "ErizoController": { "appenders": ["out"], "level": "INFO" }, + "ErizoJS": { "appenders": ["out"], "level": "INFO" }, + "ErizoJSController": { "appenders": ["out"], "level": "INFO" }, + "ErizoAgent": { "appenders": ["out"], "level": "INFO" }, + "ErizoAgentReporter": { "appenders": ["out"], "level": "ERROR" }, + "EcCloudHandler": { "appenders": ["out"], "level": "INFO" }, + "PerformanceStats": { "appenders": ["out"], "level": "INFO" }, + "Publisher": { "appenders": ["out"], "level": "INFO" }, + "Subscriber": { "appenders": ["out"], "level": "INFO" }, + "Room": { "appenders": ["out"], "level": "INFO" }, + "RoomController": { "appenders": ["out"], "level": "INFO" }, + "RovClient": { "appenders": ["out"], "level": "INFO" }, + "RovMetricsServer": { "appenders": ["out"], "level": "INFO" }, + "RovReplManager": { "appenders": ["out"], "level": "INFO" }, + "RPCPublic": { "appenders": ["out"], "level": "ERROR" } + } } + diff --git a/test/negotiation/utils/BrowserInstaller.js b/test/negotiation/utils/BrowserInstaller.js index 3459db23a..4e79924db 100644 --- a/test/negotiation/utils/BrowserInstaller.js +++ b/test/negotiation/utils/BrowserInstaller.js @@ -33,7 +33,7 @@ const BrowserInstaller = { format: `Downloading Chromium r${chromeRevision} [{bar}] {percentage}% | {value}MB/{total}MB` }, cliProgress.Presets.legacy); let barStarted = false; - chromeRevision = '801937'; + chromeRevision = '869685'; BrowserInstaller.revisionInfo = await browserFetcher.download(chromeRevision, (downloadedBytes, totalBytes) => { const totalMB = parseInt(totalBytes / 1000000); const downloadedMB = parseInt(downloadedBytes / 1000000); diff --git a/test/negotiation/utils/ClientConnection.js b/test/negotiation/utils/ClientConnection.js index 683145843..e36b99d1b 100644 --- a/test/negotiation/utils/ClientConnection.js +++ b/test/negotiation/utils/ClientConnection.js @@ -38,6 +38,14 @@ class ClientConnection { }, this.connectionId, this.sessionId, this.erizoId, this.options); } + registerFrameProcessingFunctions() { + return this.page.evaluate(() => { + navigator.getImageData = function (canvas, img) { + return canvas.getContext('2d').getImageData(0, 0, img.width, img.height); + } + }); + } + addStream(stream) { stream.addedToConnection = true; return this.page.evaluate((streamId, connectionId) => navigator.connections[connectionId].addStream(navigator.streams[streamId]), stream.id, this.connectionId); @@ -48,6 +56,86 @@ class ClientConnection { return this.page.evaluate((streamId, connectionId) => navigator.connections[connectionId].removeStream(navigator.streams[streamId]), stream.id, this.connectionId); } + showStream(erizoStream) { + return this.page.evaluate((streamId, connectionId) => { + const stream = navigator.connections[connectionId] + .stack.peerConnection.getRemoteStreams().find(s => s.label = streamId); + if (!stream) { + return; + } + const div = document.createElement('div'); + div.setAttribute('id', `player_${streamId}`); + div.setAttribute('class', 'licode_player'); + div.setAttribute('style', 'width: 100%; height: 100%; position: relative; ' + + 'background-color: black; overflow: hidden;'); + video = document.createElement('video'); + video.setAttribute('id', `stream${streamId}`); + video.setAttribute('class', 'licode_stream'); + video.setAttribute('style', 'width: 100%; height: 100%; position: absolute; object-fit: cover'); + video.setAttribute('autoplay', 'autoplay'); + video.setAttribute('playsinline', 'playsinline'); + const container = document.createElement('div'); + container.setAttribute('style', 'width: 320px; height: 240px;float:left;'); + container.setAttribute('id', `test${streamId}`); + + document.getElementById('videoContainer').appendChild(container); + container.appendChild(div); + div.appendChild(video); + video.srcObject = stream; + video.muted = true; + + // const context = new AudioContext(); + // const peerInput = context.createMediaStreamSource(stream); + // const panner = document.context.createPanner(); + // panner.setPosition(0, 0, 0); + // peerInput.connect(panner); + // peerInput.connect(context.destination); + const audioStream = new MediaStream(stream.getAudioTracks()); + // const recvAudio = new Audio(); + // recvAudio.srcObject = audioStream; + // recvAudio.autoplay = true; + // recvAudio.muted = true; + const audioCtx = new AudioContext(); + const source = audioCtx.createMediaStreamSource(audioStream); + source.connect(audioCtx.destination); + console.log("OK"); + + }, erizoStream.label, this.connectionId); + } + + async getImageData(erizoStream) { + const tryImageData = () => { + return this.page.evaluate(async (streamId, connectionId) => { + try { + const stream = navigator.connections[connectionId] + .stack.peerConnection.getRemoteStreams().find(s => s.label = streamId); + if (!stream) { + return; + } + + const imageCapture = new ImageCapture(stream.getVideoTracks()[0]); + const imageBitmap = await imageCapture.grabFrame(); + const width = imageBitmap.width; + const height = imageBitmap.height; + const canvas = Object.assign(document.createElement('canvas'), {width, height}); + canvas.getContext('2d').drawImage(imageBitmap, 0, 0, width, height); + const image = canvas.getContext('2d').getImageData(0, 0, width, height); + + return { result: image.data }; + } catch(e) { + return { error: e.message }; + } + }, erizoStream.label, this.connectionId); + } + for (let attempt = 0; attempt < 3; attempt++) { + const result = await tryImageData(); + if (result.error !== 'The associated Track is in an invalid state') { + return result.result; + } + } + return 'The associated Track is in an invalid state'; + } + setLocalDescription() { return this.page.evaluate(async (connectionId) => navigator.connections[connectionId].stack.setLocalDescription(), this.connectionId); } diff --git a/test/negotiation/utils/ClientStream.js b/test/negotiation/utils/ClientStream.js index c3d79bfa6..35dbbc75a 100644 --- a/test/negotiation/utils/ClientStream.js +++ b/test/negotiation/utils/ClientStream.js @@ -1,6 +1,6 @@ let currentClientStreamId = 0; class ClientStream { - constructor(page) { + constructor(page, color, frequency) { this.page = page; this.id = currentClientStreamId++; this.audio = true; @@ -8,22 +8,86 @@ class ClientStream { this.data = true; this.label = this.id; this.addedToConnection = false; + this.color = color; + this.frequency = frequency; + this.backgroundColor = color ? "#ffffff" : undefined; + } + + get expectedColor() { + return this.color || "#ffffff"; } init() { - return this.page.evaluate((streamId, audio, video, data) => { + return this.page.evaluate((streamId, audio, video, data, background, color, frequency) => { if (!navigator.streamsAccepted) { navigator.streamsAccepted = {}; navigator.streams = {}; } const stream = Erizo.Stream({ audio, video, data, attributes: {} }); + stream.stream = navigator.startLocalVideo(background, color, frequency); navigator.streamsAccepted[streamId] = false; stream.on('access-accepted', () => { navigator.streamsAccepted[streamId] = true; }); - stream.init(); + // stream.init(); + navigator.streamsAccepted[streamId] = true; navigator.streams[streamId] = stream; - }, this.id, this.audio, this.video, this.data); + }, this.id, this.audio, this.video, this.data, this.backgroundColor, this.color, this.frequency); + } + + registerLocalVideoCreator() { + return this.page.evaluate(() => { + navigator.startLocalVideo = (fill = undefined, background = undefined, frequency = undefined) => { + let tone = (frequency) => { + let ctx = new AudioContext(), oscillator = ctx.createOscillator(); + const gainNode = ctx.createGain(); + const dst = ctx.createMediaStreamDestination(); + oscillator.type = 'sine'; + oscillator.frequency.setValueAtTime(frequency, ctx.currentTime); // value in hertz + + oscillator.connect(gainNode); + gainNode.connect(dst); + + gainNode.gain.setValueAtTime(1, ctx.currentTime); + + oscillator.start(); + setInterval(function() { + gainNode.gain.setValueAtTime(1, ctx.currentTime); + setTimeout(function () { + gainNode.gain.setValueAtTime(0, ctx.currentTime); + }, 100); + }, 500); + + return dst.stream.getAudioTracks()[0]; + } + function whiteNoise(width, height, fill, background) { + const canvas = Object.assign(document.createElement('canvas'), {width, height}); + const ctx = canvas.getContext('2d'); + var start = new Date().getTime(); + requestAnimationFrame(function draw () { + ctx.clearRect(0,0,width,height); + ctx.fillStyle = background; + ctx.fillRect(0, 0, width, height); + ctx.textBaseline = "top"; + ctx.font = '48px serif'; + ctx.fillStyle = fill; + var now = new Date().getTime(); + ctx.fillText(now - start, 50, 50); + requestAnimationFrame(draw); + }); + let stream = canvas.captureStream(); + return stream.getVideoTracks()[0]; + } + const tracks = []; + if (fill && background) { + tracks.push(whiteNoise(640, 480, fill, background)); + } + if (frequency) { + tracks.push(tone(frequency)); + } + return new MediaStream(tracks); + }; + }); } async getLabel() { @@ -37,6 +101,18 @@ class ClientStream { return this.page.waitForFunction((streamId) => navigator.streamsAccepted[streamId], {}, this.id); } + async show() { + await this.page.evaluate((streamId) => { + const stream = navigator.streams[streamId]; + const div = document.createElement('div'); + div.setAttribute('style', 'width: 320px; height: 240px;float:left;'); + div.setAttribute('id', `test${streamId}`); + + document.getElementById('videoContainer').appendChild(div); + stream.show(`test${streamId}`); + }, this.id); + } + async remove() { await this.page.evaluate((streamId) => { navigator.streams[streamId].close(); diff --git a/test/negotiation/utils/ErizoConnection.js b/test/negotiation/utils/ErizoConnection.js index ae6ae3625..af8b264ba 100644 --- a/test/negotiation/utils/ErizoConnection.js +++ b/test/negotiation/utils/ErizoConnection.js @@ -8,7 +8,9 @@ global.config = { erizo: { useConnectionQualityCheck: true, networkinterface: 'en0', - addon: 'addonDebug', + addon: 'addon', + useNicer: true, + handlerProfiles: [[], [{"name":"Logger"}]], }, }; global.mediaConfig = mediaConfig; @@ -25,7 +27,7 @@ const ioThreadPool = new erizo.IOThreadPool(1); ioThreadPool.start(); class ErizoConnection { - constructor(connectionId) { + constructor(connectionId, isRemote) { this.webRtcConnectionConfiguration = { threadPool, ioThreadPool, @@ -36,6 +38,7 @@ class ErizoConnection { erizoControllerId: 'erizoControllerTest1', clientId: 'clientTest1', encryptTransport: true, + isRemote, options: {}, }; this.connectionId = connectionId; @@ -62,6 +65,10 @@ class ErizoConnection { } } + createOneToManyProcessor() { + return new erizo.OneToManyProcessor(); + } + onceNegotiationIsNeeded() { return new Promise(resolve => { this.connection.once('negotiationneeded', resolve); @@ -105,9 +112,14 @@ class ErizoConnection { label: stream.label, audio: true, video: true, + handlerProfile: 1, }, isPublisher); } + async getStream(stream) { + return this.connection.getStream(stream.id); + } + async removeStream(stream) { if (this.connection.getStream(stream.id)) { await this.connection.removeStream(stream.id); diff --git a/test/negotiation/utils/NegotiationTest.js b/test/negotiation/utils/NegotiationTest.js index 7dd02658f..2a8ad7b73 100644 --- a/test/negotiation/utils/NegotiationTest.js +++ b/test/negotiation/utils/NegotiationTest.js @@ -45,6 +45,9 @@ const describeNegotiationTest = function(title, test, only = false) { erizoStreams: {}, expectedClientSdpVersion: 2, expectedErizoSdpVersion: 0, + erizo: {}, + client: {}, + candidates: {}, }; let page; @@ -91,25 +94,25 @@ const describeNegotiationTest = function(title, test, only = false) { delete ctx.erizoStreams[erizoStream.id]; }; - ctx.createClientConnection = async function() { + ctx.createClientConnection = async function(idx) { const connectionId = parseInt(Math.random() * 1000, 0); const conn = new ClientConnection(page, connectionId); - ctx.client = conn; + ctx.client[idx] = conn; await conn.init(); return conn; }; - ctx.createErizoConnection = async function() { + ctx.createErizoConnection = async function(idx) { const connectionId = parseInt(Math.random() * 1000, 0); const conn = new ErizoConnection(connectionId); - ctx.erizo = conn; + ctx.erizo[idx] = conn; return conn; }; after(async function() { await page.close(); - await ctx.erizo.removeAllStreams(); - ctx.erizo.close(); + await ctx.erizo['pub'].removeAllStreams(); + ctx.erizo['pub'].close(); }); before(async function() { @@ -122,69 +125,73 @@ const describeNegotiationTest = function(title, test, only = false) { }); before(async function() { - await ctx.createClientConnection(); - await ctx.createErizoConnection(); + await ctx.createClientConnection('pub'); + await ctx.createErizoConnection('pub'); + await ctx.createClientConnection('sub'); + await ctx.createErizoConnection('sub'); }); - ctx.checkClientSentCorrectOffer = (getOffer) => { + ctx.checkClientSentCorrectOffer = (getOffer, isPubConnection) => { it('Client should send the correct offer', function() { const sdp = new SdpChecker(getOffer()); sdp.expectType('offer'); - sdp.expectToHaveStreams(ctx.clientStreams); + if (isPubConnection) { + sdp.expectToHaveStreams(ctx.clientStreams); + } }); }; - ctx.checkErizoSentCorrectOffer = (getOffer) => { + ctx.checkErizoSentCorrectOffer = (getOffer, isPubConnection) => { it('Erizo should send the correct offer', function() { const sdp = new SdpChecker(getOffer()); sdp.expectType('offer'); sdp.expectToIncludeCandidates(); - sdp.expectToHaveStreams(ctx.erizoStreams); + if (!isPubConnection) { + sdp.expectToHaveStreams(ctx.erizoStreams); + } }); }; - ctx.checkClientSentCorrectAnswer = (getAnswer) => { + ctx.checkClientSentCorrectAnswer = (getAnswer, isPubConnection) => { it('Client should send the correct answer', function() { const sdp = new SdpChecker(getAnswer()); sdp.expectType('answer'); - sdp.expectToHaveStreams(ctx.clientStreams); + if (isPubConnection) { + sdp.expectToHaveStreams(ctx.clientStreams); + } }); }; - ctx.checkErizoSentCorrectAnswer = (getAnswer) => { + ctx.checkErizoSentCorrectAnswer = (getAnswer, isPubConnection) => { it('Erizo should send an answer', function() { const sdp = new SdpChecker(getAnswer()); sdp.expectType('answer'); sdp.expectToIncludeCandidates(); - sdp.expectToHaveStreams(ctx.erizoStreams); + if (!isPubConnection) { + sdp.expectToHaveStreams(ctx.erizoStreams); + } }); }; - ctx.checkClientSentCandidates = () => { + ctx.checkClientSentCandidates = (idx) => { it('Client should send candidates', function() { - for (const candidate of ctx.candidates) { + for (const candidate of ctx.candidates[idx]) { const sdp = new SdpChecker(candidate); sdp.expectType('candidate'); } }); }; - ctx.checkErizoFinishedSuccessfully = () => { + ctx.checkErizoFinishedSuccessfully = (idx) => { it('Erizo should finish successfully', function() { - expect(ctx.erizo.isReady).to.be.true; - }); - }; - - ctx.checkClientFSMStateIsStable = () => { - it('Client FSM state should be stable', async function() { - const state = await ctx.client.getFSMState(); - expect(state).to.be.equals('stable'); + expect(ctx.erizo[idx].isReady).to.be.true; + // expect(ctx.erizo['sub'].isReady).to.be.true; }); }; - ctx.checkClientHasNotFailed = () => { + ctx.checkClientHasNotFailed = (idx) => { it('Client should not be failed', async function() { - const connectionFailed = await ctx.client.isConnectionFailed(); + const connectionFailed = await ctx.client[idx].isConnectionFailed(); expect(connectionFailed).to.be.false; }); }; @@ -201,31 +208,31 @@ const describeNegotiationTest = function(title, test, only = false) { before(async function() { clientStream = await ctx.createClientStream(); - await ctx.client.addStream(clientStream); - await ctx.erizo.publishStream(clientStream); - - await ctx.client.setLocalDescription(); - offer = { type: 'offer', sdp: await ctx.client.getLocalDescription() }; - await ctx.erizo.setRemoteDescription(offer); - await ctx.erizo.setLocalDescription(); - answer = { type: 'answer', sdp: await ctx.erizo.getLocalDescription()}; - await ctx.client.setRemoteDescription(answer); - if (!ctx.candidates) { - await ctx.client.waitForCandidates(); - ctx.candidates = await ctx.client.getAllCandidates(); - for (const candidate of ctx.candidates) { - await ctx.erizo.addIceCandidate(candidate); + await ctx.client['pub'].addStream(clientStream); + await ctx.erizo['pub'].publishStream(clientStream); + + await ctx.client['pub'].setLocalDescription(); + offer = { type: 'offer', sdp: await ctx.client['pub'].getLocalDescription() }; + await ctx.erizo['pub'].setRemoteDescription(offer); + await ctx.erizo['pub'].createAnswer(); + answer = { type: 'answer', sdp: await ctx.erizo['pub'].getLocalDescription()}; + await ctx.client['pub'].setRemoteDescription(answer); + if (!ctx.candidates['pub']) { + await ctx.client['pub'].waitForCandidates(); + ctx.candidates['pub'] = await ctx.client['pub'].getAllCandidates(); + for (const candidate of ctx.candidates['pub']) { + await ctx.erizo['pub'].addIceCandidate(candidate); } } - await ctx.client.waitForConnected(); - await ctx.erizo.waitForReadyMessage(); + await ctx.client['pub'].waitForConnected(); + await ctx.erizo['pub'].waitForReadyMessage(); }); - ctx.checkClientSentCorrectOffer(() => offer); - ctx.checkErizoSentCorrectAnswer(() => answer); - ctx.checkClientSentCandidates(); - ctx.checkErizoFinishedSuccessfully(); - ctx.checkClientHasNotFailed(); + ctx.checkClientSentCorrectOffer(() => offer, true); + ctx.checkErizoSentCorrectAnswer(() => answer, true); + ctx.checkClientSentCandidates('pub'); + ctx.checkErizoFinishedSuccessfully('pub'); + ctx.checkClientHasNotFailed('pub'); }); }; @@ -235,36 +242,35 @@ const describeNegotiationTest = function(title, test, only = false) { before(async function() { clientStream = ctx.getFirstClientStream(); - await ctx.client.removeStream(clientStream); - await ctx.erizo.unpublishStream(clientStream); - - await ctx.client.setLocalDescription(); - offer = { type: 'offer', sdp: await ctx.client.getLocalDescription() }; - - await ctx.erizo.setRemoteDescription(offer); - answer = { type: 'answer', sdp: await ctx.erizo.getLocalDescription() }; - - await ctx.client.setRemoteDescription(answer); - - if (!ctx.candidates) { - await ctx.client.waitForCandidates(); - ctx.candidates = await ctx.client.getAllCandidates(); - for (const candidate of ctx.candidates) { - await ctx.erizo.addIceCandidate(candidate); + await ctx.client['pub'].removeStream(clientStream); + await ctx.erizo['pub'].unpublishStream(clientStream); + + await ctx.client['pub'].setLocalDescription(); + offer = { type: 'offer', sdp: await ctx.client['pub'].getLocalDescription() }; + await ctx.erizo['pub'].setRemoteDescription(offer); + await ctx.erizo['pub'].createAnswer(); + answer = { type: 'answer', sdp: await ctx.erizo['pub'].getLocalDescription() }; + await ctx.client['pub'].setRemoteDescription(answer); + + if (!ctx.candidates['pub']) { + await ctx.client['pub'].waitForCandidates(); + ctx.candidates['pub'] = await ctx.client['pub'].getAllCandidates(); + for (const candidate of ctx.candidates['pub']) { + await ctx.erizo['pub'].addIceCandidate(candidate); } } - await ctx.client.waitForConnected(); - await ctx.erizo.waitForReadyMessage(); + await ctx.client['pub'].waitForConnected(); + await ctx.erizo['pub'].waitForReadyMessage(); await ctx.deleteClientStream(clientStream); }); - ctx.checkClientSentCorrectOffer(() => offer); - ctx.checkErizoSentCorrectAnswer(() => answer); - ctx.checkClientSentCandidates(); - ctx.checkErizoFinishedSuccessfully(); - ctx.checkClientHasNotFailed(); + ctx.checkClientSentCorrectOffer(() => offer, true); + ctx.checkErizoSentCorrectAnswer(() => answer, true); + ctx.checkClientSentCandidates('pub'); + ctx.checkErizoFinishedSuccessfully('pub'); + ctx.checkClientHasNotFailed('pub'); }); }; @@ -275,34 +281,34 @@ const describeNegotiationTest = function(title, test, only = false) { before(async function() { erizoStream = await ctx.createErizoStream(); - const negotiationNeeded = ctx.erizo.onceNegotiationIsNeeded(); - await ctx.erizo.subscribeStream(erizoStream); + const negotiationNeeded = ctx.erizo['sub'].onceNegotiationIsNeeded(); + await ctx.erizo['sub'].subscribeStream(erizoStream); await negotiationNeeded; - await ctx.erizo.setLocalDescription(); - offer = { type: 'offer', sdp: await ctx.erizo.getLocalDescription() }; - await ctx.client.setRemoteDescription(offer); - await ctx.client.setLocalDescription(); - answer = { type: 'answer', sdp: await ctx.client.getLocalDescription() }; - await ctx.erizo.setRemoteDescription(answer); - if (!ctx.candidates) { - await ctx.client.waitForCandidates(); - ctx.candidates = await ctx.client.getAllCandidates(); - for (const candidate of ctx.candidates) { - await ctx.erizo.addIceCandidate(candidate); + await ctx.erizo['sub'].setLocalDescription(); + offer = { type: 'offer', sdp: await ctx.erizo['sub'].getLocalDescription() }; + await ctx.client['sub'].setRemoteDescription(offer); + await ctx.client['sub'].setLocalDescription(); + answer = { type: 'answer', sdp: await ctx.client['sub'].getLocalDescription() }; + await ctx.erizo['sub'].setRemoteDescription(answer); + if (!ctx.candidates['sub']) { + await ctx.client['sub'].waitForCandidates(); + ctx.candidates['sub'] = await ctx.client['sub'].getAllCandidates(); + for (const candidate of ctx.candidates['sub']) { + await ctx.erizo['sub'].addIceCandidate(candidate); } } - await ctx.client.waitForConnected(); - await ctx.erizo.waitForReadyMessage(); - // await ctx.erizo.sleep(1000 * 60 * 5); + await ctx.client['sub'].waitForConnected(); + await ctx.erizo['sub'].waitForReadyMessage(); + // await ctx.erizo['sub'].sleep(1000 * 60 * 5); }); - ctx.checkErizoSentCorrectOffer(() => offer); - ctx.checkClientSentCorrectAnswer(() => answer); - ctx.checkClientSentCandidates(); - ctx.checkErizoFinishedSuccessfully(); - ctx.checkClientHasNotFailed(); + ctx.checkErizoSentCorrectOffer(() => offer, false); + ctx.checkClientSentCorrectAnswer(() => answer, false); + ctx.checkClientSentCandidates('sub'); + ctx.checkErizoFinishedSuccessfully('sub'); + ctx.checkClientHasNotFailed('sub'); }); }; @@ -313,34 +319,34 @@ const describeNegotiationTest = function(title, test, only = false) { before(async function() { erizoStream = ctx.getFirstErizoStream(); - const negotiationNeeded = ctx.erizo.onceNegotiationIsNeeded(); - await ctx.erizo.unsubscribeStream(erizoStream); + const negotiationNeeded = ctx.erizo['sub'].onceNegotiationIsNeeded(); + await ctx.erizo['sub'].unsubscribeStream(erizoStream); await negotiationNeeded; - await ctx.erizo.setLocalDescription(); - offer = { type: 'offer', sdp: await ctx.erizo.getLocalDescription() }; - - await ctx.client.setRemoteDescription(offer); - answer = { type: 'answer', sdp: await ctx.client.getLocalDescription() }; - - await ctx.erizo.setRemoteDescription(answer); - if (!ctx.candidates) { - await ctx.client.waitForCandidates(); - ctx.candidates = await ctx.client.getAllCandidates(); - for (const candidate of ctx.candidates) { - await ctx.erizo.addIceCandidate(candidate); + await ctx.erizo['sub'].setLocalDescription(); + offer = { type: 'offer', sdp: await ctx.erizo['sub'].getLocalDescription() }; + + await ctx.client['sub'].setRemoteDescription(offer); + answer = { type: 'answer', sdp: await ctx.client['sub'].getLocalDescription() }; + + await ctx.erizo['sub'].setRemoteDescription(answer); + if (!ctx.candidates['sub']) { + await ctx.client['sub'].waitForCandidates(); + ctx.candidates['sub'] = await ctx.client['sub'].getAllCandidates(); + for (const candidate of ctx.candidates['sub']) { + await ctx.erizo['sub'].addIceCandidate(candidate); } } - await ctx.client.waitForConnected(); - await ctx.erizo.waitForReadyMessage(); + await ctx.client['sub'].waitForConnected(); + await ctx.erizo['sub'].waitForReadyMessage(); await ctx.deleteErizoStream(erizoStream); }); - ctx.checkErizoSentCorrectOffer(() => offer); - ctx.checkClientSentCorrectAnswer(() => answer); - ctx.checkClientSentCandidates(); - ctx.checkErizoFinishedSuccessfully(); - ctx.checkClientHasNotFailed(); + ctx.checkErizoSentCorrectOffer(() => offer, false); + ctx.checkClientSentCorrectAnswer(() => answer, false); + ctx.checkClientSentCandidates('sub'); + ctx.checkErizoFinishedSuccessfully('sub'); + ctx.checkClientHasNotFailed('sub'); }); }; @@ -356,70 +362,80 @@ const describeNegotiationTest = function(title, test, only = false) { for (const step of steps) { switch (step) { case 'client-add-stream': - await ctx.client.addStream(clientStream); + await ctx.client['pub'].addStream(clientStream); break; case 'client-add-stream-and-process-erizo-offer': - addStreamPromise = ctx.client.addStream(clientStream); - processOfferPromise = ctx.client.setRemoteDescription(erizoOffer); + addStreamPromise = ctx.client['pub'].addStream(clientStream); + processOfferPromise = ctx.client['sub'].setRemoteDescription(erizoOffer); await addStreamPromise; await processOfferPromise; break; case 'client-get-offer-and-process-erizo-offer': - processOfferPromise = ctx.client.setRemoteDescription(erizoOffer); - clientOfferPromise = ctx.client.setLocalDescription(); + processOfferPromise = ctx.client['sub'].setRemoteDescription(erizoOffer); + clientOfferPromise = ctx.client['pub'].setLocalDescription(); await clientOfferPromise; - clientOffer = { type: 'offer', sdp: await ctx.client.getLocalDescription() }; + clientOffer = { type: 'offer', sdp: await ctx.client['pub'].getLocalDescription() }; break; case 'erizo-publish-stream': - await ctx.erizo.publishStream(clientStream); + await ctx.erizo['pub'].publishStream(clientStream); break; case 'erizo-subscribe-stream': - await ctx.erizo.subscribeStream(erizoStream); + await ctx.erizo['sub'].subscribeStream(erizoStream); break; case 'erizo-get-offer': - await ctx.erizo.setLocalDescription(); - erizoOffer = { type: 'offer', sdp: await ctx.erizo.getLocalDescription() }; + await ctx.erizo['sub'].setLocalDescription(); + erizoOffer = { type: 'offer', sdp: await ctx.erizo['sub'].getLocalDescription() }; break; case 'erizo-get-answer': - await ctx.erizo.setLocalDescription(); - erizoAnswer = { type: 'answer', sdp: await ctx.erizo.getLocalDescription() }; + await ctx.erizo['pub'].setLocalDescription(); + erizoAnswer = { type: 'answer', sdp: await ctx.erizo['pub'].getLocalDescription() }; break; case 'client-get-offer': - await ctx.client.setLocalDescription(); - clientOffer = { type: 'offer', sdp: await ctx.client.getLocalDescription() }; + await ctx.client['pub'].setLocalDescription(); + clientOffer = { type: 'offer', sdp: await ctx.client['pub'].getLocalDescription() }; break; case 'client-get-answer': - await ctx.client.setLocalDescription(); - clientAnswer = { type: 'answer', sdp: await ctx.client.getLocalDescription() }; + await ctx.client['sub'].setLocalDescription(); + clientAnswer = { type: 'answer', sdp: await ctx.client['sub'].getLocalDescription() }; break; case 'erizo-process-offer': - await ctx.erizo.setRemoteDescription(clientOffer); + await ctx.erizo['pub'].setRemoteDescription(clientOffer); break; case 'erizo-process-answer': - await ctx.erizo.setRemoteDescription(clientAnswer); + await ctx.erizo['sub'].setRemoteDescription(clientAnswer); break; case 'client-process-offer': - await ctx.client.setRemoteDescription(erizoOffer); + await ctx.client['sub'].setRemoteDescription(erizoOffer); break; case 'client-process-answer': - await ctx.client.setRemoteDescription(erizoAnswer); + await ctx.client['pub'].setRemoteDescription(erizoAnswer); break; case 'get-and-process-candidates': - if (!ctx.candidates) { - await ctx.client.waitForCandidates(); - ctx.candidates = await ctx.client.getAllCandidates(); - for (const candidate of ctx.candidates) { - await ctx.erizo.addIceCandidate(candidate); + if (!ctx.candidates['pub'] && ctx.client['pub']) { + await ctx.client['pub'].waitForCandidates(); + ctx.candidates['pub'] = await ctx.client['pub'].getAllCandidates(); + for (const candidate of ctx.candidates['pub']) { + await ctx.erizo['pub'].addIceCandidate(candidate); + } + } + + if (!ctx.candidates['sub'] && ctx.client['sub']) { + await ctx.client['sub'].waitForCandidates(); + ctx.candidates['sub'] = await ctx.client['sub'].getAllCandidates(); + for (const candidate of ctx.candidates['sub']) { + await ctx.erizo['sub'].addIceCandidate(candidate); } } break; case 'wait-for-being-connected': - await ctx.erizo.waitForStartedMessage(); - await ctx.client.waitForConnected(); + await ctx.erizo['pub'].waitForStartedMessage(); + await ctx.client['pub'].waitForConnected(); + await ctx.erizo['sub'].waitForStartedMessage(); + await ctx.client['sub'].waitForConnected(); break; case '': - await ctx.erizo.subscribeStream(erizoStream); + await ctx.erizo['sub'].subscribeStream(erizoStream); break; default: break; diff --git a/test/negotiation/utils/SdpUtils.js b/test/negotiation/utils/SdpUtils.js index 49b14eb90..46cf6bf7b 100644 --- a/test/negotiation/utils/SdpUtils.js +++ b/test/negotiation/utils/SdpUtils.js @@ -41,6 +41,9 @@ class SdpChecker { } expectToHaveStreams(streams) { + if (!streams) { + return; + } const streamIds = Object.keys(streams); for (const streamId of streamIds) { this.expectToHaveStream(streams[streamId]); diff --git a/test/negotiation/utils/StreamSwitchTest.js b/test/negotiation/utils/StreamSwitchTest.js new file mode 100644 index 000000000..75713fe4f --- /dev/null +++ b/test/negotiation/utils/StreamSwitchTest.js @@ -0,0 +1,488 @@ +const path = require('path'); +const puppeteer = require('puppeteer-core'); +const expect = require('chai').expect; + +const BrowserInstaller = require('./BrowserInstaller'); +const ClientStream = require('./ClientStream'); +const ClientConnection = require('./ClientConnection'); +const ErizoConnection = require('./ErizoConnection'); +const SdpChecker = require('./SdpUtils'); + +let browser, browser2; +let currentErizoStreamId = 10; + +before(async function() { + this.timeout(30000); + if (!BrowserInstaller.installed) { + this.timeout(300000); + await BrowserInstaller.install('canary'); + } + browser = await puppeteer.launch({ + headless: false, + dumpio: false, + executablePath: BrowserInstaller.revisionInfo.executablePath, + // executablePath: '/Users/jcague/development/chromium/src/out/Default2/Chromium.app/Contents/MacOS/Chromium', + args: [ + '--use-fake-ui-for-media-stream', + '--use-fake-device-for-media-stream', + '-enable-logging', + '--v=0', + `--vmodule=*/webrtc/*=20,*peerconnection*=20,*video*=20,*video_renderer_algorithm*=0`, + '--enable-logging=stderr', + ] + }); + + browser2 = await puppeteer.launch({ + headless: false, + dumpio: true, + executablePath: BrowserInstaller.revisionInfo.executablePath, + // executablePath: '/Users/jcague/development/chromium/src/out/Default2/Chromium.app/Contents/MacOS/Chromium', + args: [ + '--use-fake-ui-for-media-stream', + '--use-fake-device-for-media-stream', + '-enable-logging', + '--v=0', + `--vmodule=*/webrtc/*=20,*peerconnection*=20,*video*=20,*video_renderer_algorithm*=0`, + '--enable-logging=stderr', + ] + }); + + const internalPage = await browser.newPage(); + // await internalPage.goto(`chrome://webrtc-internals`); + + const internalPage2 = await browser2.newPage(); + // await internalPage2.goto(`chrome://webrtc-internals`); +}); + +after(async function() { + await browser.close(); + await browser2.close(); +}); + +const describeStreamSwitchTest = function(title, test, only = false) { + const describeImpl = only ? describe.only : describe; + describeImpl(title, function() { + this.timeout(500000); + + const ctx = { + clientStreams: {}, + erizoStreams: {}, + expectedClientSdpVersion: 2, + expectedErizoSdpVersion: 0, + erizo: {}, + client: {}, + }; + + let page, page2; + const erizoConns = {}; + + ctx.createClientStream = async function(idx, color, frequency) { + const stream = new ClientStream(page2, color, frequency); + ctx.clientStreams[idx] = stream; + await stream.registerLocalVideoCreator(); + await stream.init(); + await stream.waitForAccepted(); + await stream.getLabel(); + return stream; + }; + + ctx.deleteClientStream = async function(idx) { + if (ctx.clientStreams[idx]) { + ctx.clientStreams[idx].remove(); + delete ctx.clientStreams[idx]; + } + }; + + ctx.createErizoStream = async function(idx, label, audio, video) { + const id = parseInt(currentErizoStreamId++, 0); + const stream = { id, label: label || id, audio, video, addedToConnection: false }; + ctx.erizoStreams[idx] = stream; + return stream; + }; + + ctx.deleteErizoStream = async function(idx) { + delete ctx.erizoStreams[idx]; + }; + + ctx.createClientConnection = async function(idx) { + const connectionId = parseInt(Math.random() * 1000, 0); + const conn = new ClientConnection(idx === 'pub' ? page2 : page, connectionId); + ctx.client[idx] = conn; + await conn.init(); + await conn.registerFrameProcessingFunctions(); + return conn; + }; + + ctx.createErizoConnection = async function(idx) { + const connectionId = parseInt(Math.random() * 1000, 0); + const conn = new ErizoConnection(connectionId, idx === 'pub'); + ctx.erizo[idx] = conn; + return conn; + }; + + after(async function() { + await page.close(); + await page2.close(); + await ctx.erizo['pub'].removeAllStreams(); + ctx.erizo['pub'].close(); + }); + + before(async function() { + const currentProcessPath = process.cwd(); + const htmlPath = path.join(currentProcessPath, '../../extras/basic_example/public/index.html'); + page = await browser.newPage(); + + // page.on('console', msg => console.log('PAGE LOG:', msg.text())); + await page.goto(`file://${htmlPath}?forceStart=1`); + + page2 = await browser2.newPage(); + + // page.on('console', msg => console.log('PAGE LOG:', msg.text())); + await page2.goto(`file://${htmlPath}?forceStart=1`); + }); + + before(async function() { + await ctx.createClientConnection('pub'); + await ctx.createErizoConnection('pub'); + await ctx.createClientConnection('sub'); + await ctx.createErizoConnection('sub'); + }); + + ctx.checkClientSentCorrectOffer = (getOffer, clientId) => { + it('Client should send the correct offer', function() { + const sdp = new SdpChecker(getOffer()); + sdp.expectType('offer'); + sdp.expectToHaveStreams(ctx.client[clientId].streams); + }); + }; + + ctx.checkErizoSentCorrectOffer = (getOffer, erizoId) => { + it('Erizo should send the correct offer', function() { + const sdp = new SdpChecker(getOffer()); + sdp.expectType('offer'); + sdp.expectToIncludeCandidates(); + sdp.expectToHaveStreams(ctx.erizo[erizoId].streams); + }); + }; + + ctx.checkClientSentCorrectAnswer = (getAnswer, clientId) => { + it('Client should send the correct answer', function() { + const sdp = new SdpChecker(getAnswer()); + sdp.expectType('answer'); + sdp.expectToHaveStreams(ctx.client[clientId].streams); + }); + }; + + ctx.checkErizoSentCorrectAnswer = (getAnswer, erizoId) => { + it('Erizo should send an answer', function() { + const sdp = new SdpChecker(getAnswer()); + sdp.expectType('answer'); + sdp.expectToIncludeCandidates(); + sdp.expectToHaveStreams(ctx.erizo[erizoId].streams); + }); + }; + + ctx.checkClientSentCandidates = (clientId) => { + it('Client should send candidates', function() { + for (const candidate of ctx.client[clientId].candidates) { + const sdp = new SdpChecker(candidate); + sdp.expectType('candidate'); + } + }); + }; + + ctx.checkErizoFinishedSuccessfully = (idx) => { + it('Erizo should finish successfully', function() { + expect(ctx.erizo[idx].isReady).to.be.true; + // expect(erizo.isReady).to.be.true; + }); + }; + + ctx.checkClientHasNotFailed = (idx) => { + it('Client should not be failed', async function() { + const connectionFailed = await ctx.client[idx].isConnectionFailed(); + expect(connectionFailed).to.be.false; + }); + }; + + ctx.checkClientDroppedOffer = (getOffer) => { + it('Client should drop erizo offer', function() { + expect(getOffer()).to.have.property('type', 'offer-dropped'); + }); + }; + + ctx.publishStream = async function(clientId, erizoId, clientStreamId) { + describe('Publish One Client Stream', function() { + let offer, answer, client, erizo, clientStream; + + before(async function() { + client = ctx.client[clientId]; + erizo = ctx.erizo[erizoId]; + clientStream = ctx.clientStreams[clientStreamId]; + await client.addStream(clientStream); + await erizo.publishStream(clientStream); + + await client.setLocalDescription(); + offer = { type: 'offer', sdp: await client.getLocalDescription() }; + await erizo.setRemoteDescription(offer); + await erizo.createAnswer(); + answer = { type: 'answer', sdp: await erizo.getLocalDescription()}; + console.log(offer,answer); + await client.setRemoteDescription(answer); + if (!client.candidates) { + await client.waitForCandidates(); + client.candidates = await client.getAllCandidates(); + for (const candidate of client.candidates) { + await erizo.addIceCandidate(candidate); + } + } + await client.waitForConnected(); + await erizo.waitForReadyMessage(); + await clientStream.show(); + }); + + ctx.checkClientSentCorrectOffer(() => offer, clientId); + ctx.checkErizoSentCorrectAnswer(() => answer, erizoId); + ctx.checkClientSentCandidates(clientId); + ctx.checkErizoFinishedSuccessfully(erizoId); + ctx.checkClientHasNotFailed(clientId); + }); + }; + + ctx.unpublishStream = async function(client, erizo, clientStream) { + describe('Unpublish One Client Stream', function() { + let offer, answer; + before(async function() { + await client.removeStream(clientStream); + await erizo.unpublishStream(clientStream); + + await client.setLocalDescription(); + offer = { type: 'offer', sdp: await client.getLocalDescription() }; + await erizo.setRemoteDescription(offer); + await erizo.createAnswer(); + answer = { type: 'answer', sdp: await erizo.getLocalDescription() }; + await client.setRemoteDescription(answer); + + if (!client.candidates) { + await client.waitForCandidates(); + client.candidates = await client.getAllCandidates(); + for (const candidate of client.candidates) { + await erizo.addIceCandidate(candidate); + } + } + + await client.waitForConnected(); + await erizo.waitForReadyMessage(); + + await ctx.deleteClientStream(clientStream); + }); + + ctx.checkClientSentCorrectOffer(() => offer, client); + ctx.checkErizoSentCorrectAnswer(() => answer, erizo); + ctx.checkClientSentCandidates(client); + ctx.checkErizoFinishedSuccessfully(erizo); + ctx.checkClientHasNotFailed(client); + }); + }; + + function RGBToHex(r,g,b) { + r = r.toString(16); + g = g.toString(16); + b = b.toString(16); + + if (r.length == 1) + r = "0" + r; + if (g.length == 1) + g = "0" + g; + if (b.length == 1) + b = "0" + b; + + return "#" + r + g + b; + } + + function hexToRGB(hex) { + var result = /^#?([a-f\d]{2})([a-f\d]{2})([a-f\d]{2})$/i.exec(hex); + return result ? [parseInt(result[1], 16), parseInt(result[2], 16), parseInt(result[3], 16)] : null; + } + + function colorDistance(v1, v2){ + var i, + d = 0; + + for (i = 0; i < v1.length; i++) { + d += (v1[i] - v2[i])*(v1[i] - v2[i]); + } + return Math.sqrt(d); + }; + + ctx.linkSubToPub = function(clientId, erizoPubId, erizoSubId, streamPubId, streamSubId, wait = 5) { + describe('Link Subscriber to Publisher', async function() { + let erizoPub, erizoSub, publisher, subscriber, client, erizoStream, pubStream; + before(async function() { + client = ctx.client[clientId]; + erizoPub = ctx.erizo[erizoPubId]; + erizoSub = ctx.erizo[erizoSubId]; + pubStream = ctx.clientStreams[streamPubId]; + publisher = erizoPub.connection.getStream(pubStream.id); + const subStream = ctx.erizoStreams[streamSubId]; + subscriber = erizoSub.connection.getStream(subStream.id); + if (!publisher.muxer) { + publisher.muxer = erizoPub.createOneToManyProcessor(); + publisher.setAudioReceiver(publisher.muxer); + publisher.setVideoReceiver(publisher.muxer); + publisher.muxer.setPublisher(publisher); + publisher.muteStream(false, false); + await erizoSub.sleep(300); + } + publisher.muxer.addSubscriber(subscriber, subscriber.id); + subscriber.muteStream(false, false); + await erizoSub.sleep(1000 * wait); + + erizoStream = ctx.erizoStreams[streamSubId]; + }); + + it('should have muxer with publisher', async () => { + expect(publisher.muxer).to.not.be.null; + expect(publisher.muxer.hasPublisher()).to.be.true; + }); + + // it('should be receiving the right stream', async() => { + // if (pubStream.color) { + // const imageData = await client.getImageData(erizoStream); + // const imagePixel = [imageData[0], imageData[1], imageData[2]]; + // const pixelColor = hexToRGB(pubStream.expectedColor); + // const distance = colorDistance(imagePixel, pixelColor); + // expect(distance).to.be.lessThan(100); + // } else { + // const imageData = await client.getImageData(erizoStream); + // expect(imageData).to.be.equals('The associated Track is in an invalid state'); + // } + // }); + + // it('should be receiving video', async() => { + // if (pubStream.color) { + // const imageData = await client.getImageData(erizoStream); + // await erizoSub.sleep(100); + // const imageData2 = await client.getImageData(erizoStream); + // expect(imageData).to.not.be.deep.equals(imageData2); + // } else { + // const imageData = await client.getImageData(erizoStream); + // expect(imageData).to.be.equals('The associated Track is in an invalid state'); + // } + // }); + }); + }; + + ctx.unlinkSubToPub = function(clientId, erizoPubId, erizoSubId, streamPubId, streamSubId) { + describe('Unlink Subscriber to Publisher', function() { + let erizoPub, erizoSub, publisher, subscriber, erizoStream, client; + before(async function() { + client = ctx.client[clientId]; + erizoPub = ctx.erizo[erizoPubId]; + erizoSub = ctx.erizo[erizoSubId]; + const pubStream = ctx.clientStreams[streamPubId]; + publisher = erizoPub.connection.getStream(pubStream.id); + const subStream = ctx.erizoStreams[streamSubId]; + subscriber = erizoSub.connection.getStream(subStream.id); + if (publisher.muxer) { + publisher.muxer.removeSubscriber(subscriber.id); + } + await erizoSub.sleep(0); + + erizoStream = ctx.erizoStreams[streamSubId]; + }); + + it('should have muxer', () => { + expect(publisher.muxer).to.not.be.null; + expect(publisher.muxer.hasPublisher()).to.be.true; + }); + + it('should not be receiving video', async() => { + }); + }); + }; + + ctx.subscribeToStream = function(clientId, erizoId, erizoStreamId, erizoPubId) { + describe('Subscribe One Erizo Stream', function() { + let offer, answer, client, erizo, erizoPub, erizoStream; + before(async function() { + client = ctx.client[clientId]; + erizo = ctx.erizo[erizoId]; + erizoPub = ctx.erizo[erizoPubId]; + erizoStream = ctx.erizoStreams[erizoStreamId]; + const negotiationNeeded = erizo.onceNegotiationIsNeeded(); + await erizo.subscribeStream(erizoStream); + await negotiationNeeded; + + erizo.connection.copySdpInfoFromConnection(erizoPub.connection); + + await erizo.setLocalDescription(); + offer = { type: 'offer', sdp: await erizo.getLocalDescription() }; + await client.setRemoteDescription(offer); + await client.setLocalDescription(); + answer = { type: 'answer', sdp: await client.getLocalDescription() }; + await erizo.setRemoteDescription(answer); + if (!client.candidates) { + await client.waitForCandidates(); + client.candidates = await client.getAllCandidates(); + for (const candidate of client.candidates) { + await erizo.addIceCandidate(candidate); + } + } + + await client.waitForConnected(); + await erizo.waitForReadyMessage(); + await client.showStream(erizoStream); + }); + + ctx.checkErizoSentCorrectOffer(() => offer, erizoId); + ctx.checkClientSentCorrectAnswer(() => answer, clientId); + ctx.checkClientSentCandidates(clientId); + ctx.checkErizoFinishedSuccessfully(erizoId); + ctx.checkClientHasNotFailed(clientId); + }); + }; + + ctx.unsubscribeStreamStep = function(client, erizo, erizoStream) { + describe('Unsubscribe One Erizo Stream', function() { + let offer, answer, erizoStream; + + before(async function() { + const negotiationNeeded = erizo.onceNegotiationIsNeeded(); + await erizo.unsubscribeStream(erizoStream); + await negotiationNeeded; + await erizo.setLocalDescription(); + offer = { type: 'offer', sdp: await erizo.getLocalDescription() }; + + await client.setRemoteDescription(offer); + answer = { type: 'answer', sdp: await client.getLocalDescription() }; + + await erizo.setRemoteDescription(answer); + if (!client.candidates) { + await client.waitForCandidates(); + client.candidates = await client.getAllCandidates(); + for (const candidate of client.candidates) { + await erizo.addIceCandidate(candidate); + } + } + + await client.waitForConnected(); + await erizo.waitForReadyMessage(); + await ctx.deleteErizoStream(erizoStream); + }); + + ctx.checkErizoSentCorrectOffer(() => offer, false); + ctx.checkClientSentCorrectAnswer(() => answer, false); + ctx.checkClientSentCandidates('sub'); + ctx.checkErizoFinishedSuccessfully('sub'); + ctx.checkClientHasNotFailed('sub'); + }); + }; + + test.call(this, ctx); + }); +}; + + + +module.exports = describeStreamSwitchTest;