diff --git a/cobalt/android/apk/app/src/main/java/dev/cobalt/media/MediaCodecBridge.java b/cobalt/android/apk/app/src/main/java/dev/cobalt/media/MediaCodecBridge.java index d10341ad2ad5..389f14bd9205 100644 --- a/cobalt/android/apk/app/src/main/java/dev/cobalt/media/MediaCodecBridge.java +++ b/cobalt/android/apk/app/src/main/java/dev/cobalt/media/MediaCodecBridge.java @@ -481,14 +481,17 @@ public static void createVideoMediaCodecBridge( boolean shouldConfigureHdr = colorInfo != null && MediaCodecUtil.isHdrCapableVideoDecoder(mime, codecCapabilities); if (shouldConfigureHdr) { - Log.d(TAG, "Setting HDR info."); + Log.d(TAG, "Setting color info."); mediaFormat.setInteger(MediaFormat.KEY_COLOR_TRANSFER, colorInfo.colorTransfer); mediaFormat.setInteger(MediaFormat.KEY_COLOR_STANDARD, colorInfo.colorStandard); // If color range is unspecified, don't set it. if (colorInfo.colorRange != 0) { mediaFormat.setInteger(MediaFormat.KEY_COLOR_RANGE, colorInfo.colorRange); } - mediaFormat.setByteBuffer(MediaFormat.KEY_HDR_STATIC_INFO, colorInfo.hdrStaticInfo); + // Only set HDR static metadata for HDR transfer functions (6 = ST2084, 7 = HLG) + if (colorInfo.colorTransfer == 6 || colorInfo.colorTransfer == 7) { + mediaFormat.setByteBuffer(MediaFormat.KEY_HDR_STATIC_INFO, colorInfo.hdrStaticInfo); + } } if (tunnelModeAudioSessionId != TunnelModeAudioSessionId.NONE) { diff --git a/media/base/audio_decoder_config.cc b/media/base/audio_decoder_config.cc index 9a0bffa6255e..9b3bc8765d04 100644 --- a/media/base/audio_decoder_config.cc +++ b/media/base/audio_decoder_config.cc @@ -111,7 +111,12 @@ std::string AudioDecoderConfig::AsHumanReadableString() const { << ", target_output_channel_layout: " << ChannelLayoutToString(target_output_channel_layout()) << ", target_output_sample_format: " +#if BUILDFLAG(USE_STARBOARD_MEDIA) + << SampleFormatToString(target_output_sample_format()) + << ", is_change_type_transition: " << base::ToString(is_change_type_transition()); +#else // BUILDFLAG(USE_STARBOARD_MEDIA) << SampleFormatToString(target_output_sample_format()); +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) return s.str(); } diff --git a/media/base/audio_decoder_config.h b/media/base/audio_decoder_config.h index 147d11fa8278..3d9145199ca7 100644 --- a/media/base/audio_decoder_config.h +++ b/media/base/audio_decoder_config.h @@ -122,6 +122,11 @@ class MEDIA_EXPORT AudioDecoderConfig { return target_output_sample_format_; } +#if BUILDFLAG(USE_STARBOARD_MEDIA) + void set_is_change_type_transition(bool value) { is_change_type_transition_ = value; } + bool is_change_type_transition() const { return is_change_type_transition_; } +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) + private: // WARNING: When modifying or adding any parameters, update the following: // - AudioDecoderConfig::AsHumanReadableString() @@ -171,6 +176,12 @@ class MEDIA_EXPORT AudioDecoderConfig { // be manually set in `SetChannelsForDiscrete()`; int channels_ = 0; +#if BUILDFLAG(USE_STARBOARD_MEDIA) + // Designates whether the received config was caused by a + // SourceBuffer.changeType() call. + bool is_change_type_transition_ = false; +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) + // Not using DISALLOW_COPY_AND_ASSIGN here intentionally to allow the compiler // generated copy constructor and assignment operator. Since the extra data is // typically small, the performance impact is minimal. diff --git a/media/base/video_decoder_config.cc b/media/base/video_decoder_config.cc index 62fae726de64..a51d297f9085 100644 --- a/media/base/video_decoder_config.cc +++ b/media/base/video_decoder_config.cc @@ -115,6 +115,10 @@ std::string VideoDecoderConfig::AsHumanReadableString() const { << ", encryption scheme: " << encryption_scheme() << ", rotation: " << VideoRotationToString(video_transformation().rotation) << ", flipped: " << video_transformation().mirrored +#if BUILDFLAG(USE_STARBOARD_MEDIA) + << ", is_change_type_transition: " + << base::ToString(is_change_type_transition()) +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) << ", color space: " << color_space_info().ToGfxColorSpace().ToString(); if (hdr_metadata().has_value()) { diff --git a/media/base/video_decoder_config.h b/media/base/video_decoder_config.h index b39c21556a69..71da9e1611df 100644 --- a/media/base/video_decoder_config.h +++ b/media/base/video_decoder_config.h @@ -164,6 +164,11 @@ class MEDIA_EXPORT VideoDecoderConfig { // useful for decryptors that decrypts an encrypted stream to a clear stream. void SetIsEncrypted(bool is_encrypted); +#if BUILDFLAG(USE_STARBOARD_MEDIA) + void set_is_change_type_transition(bool value) { is_change_type_transition_ = value; } + bool is_change_type_transition() const { return is_change_type_transition_; } +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) + private: VideoCodec codec_ = VideoCodec::kUnknown; VideoCodecProfile profile_ = VIDEO_CODEC_PROFILE_UNKNOWN; @@ -190,6 +195,12 @@ class MEDIA_EXPORT VideoDecoderConfig { VideoColorSpace color_space_info_; std::optional hdr_metadata_; +#if BUILDFLAG(USE_STARBOARD_MEDIA) + // Designates whether the received config was caused by a + // SourceBuffer.changeType() call. + bool is_change_type_transition_ = false; + +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) // Not using DISALLOW_COPY_AND_ASSIGN here intentionally to allow the compiler // generated copy constructor and assignment operator. Since the extra data is // typically small, the performance impact is minimal. diff --git a/media/filters/BUILD.gn b/media/filters/BUILD.gn index fc6e5de4839b..4337d572e361 100644 --- a/media/filters/BUILD.gn +++ b/media/filters/BUILD.gn @@ -83,6 +83,10 @@ source_set("filters") { "//ui/gfx/geometry:geometry", ] + if (use_starboard_media) { + deps += [ "//starboard:starboard_headers_only" ] + } + libs = [] if (proprietary_codecs) { diff --git a/media/filters/chunk_demuxer.cc b/media/filters/chunk_demuxer.cc index 59b59ad76789..a258291a48bc 100644 --- a/media/filters/chunk_demuxer.cc +++ b/media/filters/chunk_demuxer.cc @@ -32,6 +32,7 @@ #if BUILDFLAG(USE_STARBOARD_MEDIA) #include "base/containers/contains.h" #include "base/strings/string_split.h" +#include "starboard/media.h" // nogncheck #endif // BUILDFLAG(USE_STARBOARD_MEDIA) namespace { @@ -338,8 +339,14 @@ void ChunkDemuxerStream::UnmarkEndOfStream() { // DemuxerStream methods. #if BUILDFLAG(USE_STARBOARD_MEDIA) std::string ChunkDemuxerStream::mime_type() const { + base::AutoLock auto_lock(lock_); return mime_type_; } + +void ChunkDemuxerStream::SetMimeType(std::string_view mime_type) { + base::AutoLock auto_lock(lock_); + mime_type_ = mime_type; +} #endif // BUILDFLAG(USE_STARBOARD_MEDIA) void ChunkDemuxerStream::Read(uint32_t count, ReadCB read_cb) { @@ -1276,6 +1283,58 @@ void ChunkDemuxer::Remove(const std::string& id, host_->OnBufferedTimeRangesChanged(GetBufferedRanges_Locked()); } +#if BUILDFLAG(USE_STARBOARD_MEDIA) +bool ChunkDemuxer::CanChangeType(const std::string& id, + const std::string& new_mime) { + base::AutoLock auto_lock(lock_); + CHECK(IsValidId_Locked(id)); + + if (!supports_change_type_) { + return false; + } + + auto iter = id_to_mime_map_.find(id); + std::string current_mime = iter != id_to_mime_map_.end() ? iter->second : ""; + + if (!SbMediaCanChangeType(current_mime.c_str(), new_mime.c_str())) { + LOG(INFO) << "Codec transition for " << current_mime << " to " + << new_mime << " is not supported." + << " ChunkDemuxer::CanChangeType returns false."; + return false; + } + + std::string type, codecs; + if (!ParseMimeType(new_mime, &type, &codecs)) { + return false; + } + + std::unique_ptr stream_parser( + CreateParserForTypeAndCodecs(type, codecs, media_log_)); + return !!stream_parser; +} + +void ChunkDemuxer::ChangeType(const std::string& id, + const std::string& new_mime) { + DVLOG(1) << __func__ << " id=" << id << " new_mime=" << new_mime; + + base::AutoLock auto_lock(lock_); + + DCHECK(state_ == INITIALIZING || state_ == INITIALIZED) << state_; + CHECK(IsValidId_Locked(id)); + + std::string type, codecs; + CHECK(ParseMimeType(new_mime, &type, &codecs)); + + std::unique_ptr stream_parser( + CreateParserForTypeAndCodecs(type, codecs, media_log_)); + // Caller should query CanChangeType() first to protect from failing this. + DCHECK(stream_parser); + + id_to_mime_map_[id] = new_mime; + source_state_map_[id]->ChangeType( + std::move(stream_parser), ExpectedCodecs(type, codecs), new_mime); +} +#else // BUILDFLAG(USE_STARBOARD_MEDIA) bool ChunkDemuxer::CanChangeType(const std::string& id, const std::string& content_type, const std::string& codecs) { @@ -1324,6 +1383,7 @@ void ChunkDemuxer::ChangeType(const std::string& id, source_state_map_[id]->ChangeType(std::move(stream_parser), ExpectedCodecs(content_type, codecs)); } +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) double ChunkDemuxer::GetDuration() { base::AutoLock auto_lock(lock_); diff --git a/media/filters/chunk_demuxer.h b/media/filters/chunk_demuxer.h index c7289a92ef60..bbfbc128ceb1 100644 --- a/media/filters/chunk_demuxer.h +++ b/media/filters/chunk_demuxer.h @@ -29,6 +29,10 @@ #include "media/filters/source_buffer_stream.h" #include "media/filters/stream_parser_factory.h" +#if BUILDFLAG(USE_STARBOARD_MEDIA) +#include +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) + namespace media { class SourceBufferStream; @@ -88,6 +92,7 @@ class MEDIA_EXPORT ChunkDemuxerStream : public DemuxerStream { // Returns the latest presentation timestamp of the buffers queued in the // stream. base::TimeDelta GetWriteHead() const; + void SetMimeType(std::string_view mime_type); #endif // BUILDFLAG(USE_STARBOARD_MEDIA) void OnMemoryPressure( @@ -191,7 +196,7 @@ class MEDIA_EXPORT ChunkDemuxerStream : public DemuxerStream { GetPendingBuffers_Locked() EXCLUSIVE_LOCKS_REQUIRED(lock_); #if BUILDFLAG(USE_STARBOARD_MEDIA) - const std::string mime_type_; + std::string mime_type_ GUARDED_BY(lock_); base::TimeDelta write_head_ GUARDED_BY(lock_); #endif // BUILDFLAG(USE_STARBOARD_MEDIA) @@ -299,9 +304,12 @@ class MEDIA_EXPORT ChunkDemuxer : public Demuxer { #endif #if BUILDFLAG(USE_STARBOARD_MEDIA) - // Special version of AddId() that retains the |mime_type| from the web app. + // Special versions of AddId(), CanChangeType(), and ChangeType() that retain + // the |mime_type| from the web app. [[nodiscard]] Status AddId(const std::string& id, const std::string& mime_type); + bool CanChangeType(const std::string& id, const std::string& new_mime); + void ChangeType(const std::string& id, const std::string& new_mime); #endif // BUILDFLAG(USE_STARBOARD_MEDIA) // Notifies a caller via `tracks_updated_cb` that the set of media tracks @@ -398,6 +406,7 @@ class MEDIA_EXPORT ChunkDemuxer : public Demuxer { void Remove(const std::string& id, base::TimeDelta start, base::TimeDelta end); +#if !BUILDFLAG(USE_STARBOARD_MEDIA) // Returns whether or not the source buffer associated with |id| can change // its parser type to one which parses |content_type| and |codecs|. // |content_type| indicates the ContentType of the MIME type for the data that @@ -417,6 +426,7 @@ class MEDIA_EXPORT ChunkDemuxer : public Demuxer { void ChangeType(const std::string& id, const std::string& content_type, const std::string& codecs); +#endif // !BUILDFLAG(USE_STARBOARD_MEDIA) // If the buffer is full, attempts to try to free up space, as specified in // the "Coded Frame Eviction Algorithm" in the Media Source Extensions Spec. diff --git a/media/filters/source_buffer_state.cc b/media/filters/source_buffer_state.cc index f16ea4f7cece..789291cff59f 100644 --- a/media/filters/source_buffer_state.cc +++ b/media/filters/source_buffer_state.cc @@ -164,9 +164,16 @@ void SourceBufferState::Init(StreamParser::InitCB init_cb, InitializeParser(expected_codecs); } +#if BUILDFLAG(USE_STARBOARD_MEDIA) +void SourceBufferState::ChangeType( + std::unique_ptr new_stream_parser, + const std::string& new_expected_codecs, + const std::string& new_mime_type) { +#else // BUILDFLAG(USE_STARBOARD_MEDIA) void SourceBufferState::ChangeType( std::unique_ptr new_stream_parser, const std::string& new_expected_codecs) { +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) DCHECK_GE(state_, PENDING_PARSER_CONFIG); DCHECK_NE(state_, PENDING_PARSER_INIT); DCHECK(!parsing_media_segment_); @@ -177,6 +184,14 @@ void SourceBufferState::ChangeType( state_ = PENDING_PARSER_RECONFIG; stream_parser_ = std::move(new_stream_parser); +#if BUILDFLAG(USE_STARBOARD_MEDIA) + for (auto& stream : audio_streams_) { + stream.second->SetMimeType(new_mime_type); + } + for (auto& stream : video_streams_) { + stream.second->SetMimeType(new_mime_type); + } +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) InitializeParser(new_expected_codecs); } @@ -666,6 +681,13 @@ bool SourceBufferState::OnNewConfigs(std::unique_ptr tracks) { } track->set_id(stream->media_track_id()); +#if BUILDFLAG(USE_STARBOARD_MEDIA) + // Currently, |PENDING_PARSER_RECONFIG| is only set when a changeType + // call occurs, so we're safe to update the config's value. + if (state_ == PENDING_PARSER_RECONFIG) { + audio_config.set_is_change_type_transition(true); + } +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) frame_processor_->OnPossibleAudioConfigUpdate(audio_config); success &= stream->UpdateAudioConfig(audio_config, allow_codec_changes, media_log_); @@ -751,6 +773,13 @@ bool SourceBufferState::OnNewConfigs(std::unique_ptr tracks) { } track->set_id(stream->media_track_id()); +#if BUILDFLAG(USE_STARBOARD_MEDIA) + // Currently, |PENDING_PARSER_RECONFIG| is only set when a changeType + // call occurs, so we're safe to update the config's value. + if (state_ == PENDING_PARSER_RECONFIG) { + video_config.set_is_change_type_transition(true); + } +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) success &= stream->UpdateVideoConfig(video_config, allow_codec_changes, media_log_); } else { diff --git a/media/filters/source_buffer_state.h b/media/filters/source_buffer_state.h index 8a43817ee455..46ca53413be3 100644 --- a/media/filters/source_buffer_state.h +++ b/media/filters/source_buffer_state.h @@ -57,8 +57,14 @@ class MEDIA_EXPORT SourceBufferState { // Reconfigures this source buffer to use |new_stream_parser|. Caller must // first ensure that ResetParserState() was done to flush any pending frames // from the old stream parser. +#if BUILDFLAG(USE_STARBOARD_MEDIA) + void ChangeType(std::unique_ptr new_stream_parser, + const std::string& new_expected_codecs, + const std::string& new_mime_type); +#else // BUILDFLAG(USE_STARBOARD_MEDIA) void ChangeType(std::unique_ptr new_stream_parser, const std::string& new_expected_codecs); +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) // Appends media data to the StreamParser, but no parsing is done of it yet, // just buffering the media data for future parsing via RunSegmentParserLoop() diff --git a/media/mojo/clients/mojo_demuxer_stream_impl.cc b/media/mojo/clients/mojo_demuxer_stream_impl.cc index ec00cf94640d..324f002c5eac 100644 --- a/media/mojo/clients/mojo_demuxer_stream_impl.cc +++ b/media/mojo/clients/mojo_demuxer_stream_impl.cc @@ -93,13 +93,23 @@ void MojoDemuxerStreamImpl::OnBufferReady( NOTREACHED() << "Unsupported config change encountered for type: " << stream_->type(); } +#if BUILDFLAG(USE_STARBOARD_MEDIA) + std::optional mime_type = stream_->mime_type(); + std::move(callback).Run(Status::kConfigChanged, {}, audio_config, + video_config, mime_type); +#else // BUILDFLAG(USE_STARBOARD_MEDIA) std::move(callback).Run(Status::kConfigChanged, {}, audio_config, video_config); +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) return; } if (status == Status::kAborted) { +#if BUILDFLAG(USE_STARBOARD_MEDIA) + std::move(callback).Run(Status::kAborted, {}, audio_config, video_config, /*mime_type=*/std::nullopt); +#else // BUILDFLAG(USE_STARBOARD_MEDIA) std::move(callback).Run(Status::kAborted, {}, audio_config, video_config); +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) return; } @@ -110,7 +120,11 @@ void MojoDemuxerStreamImpl::OnBufferReady( mojom::DecoderBufferPtr mojo_buffer = mojo_decoder_buffer_writer_->WriteDecoderBuffer(std::move(buffer)); if (!mojo_buffer) { +#if BUILDFLAG(USE_STARBOARD_MEDIA) + std::move(callback).Run(Status::kAborted, {}, audio_config, video_config, /*mime_type=*/std::nullopt); +#else // BUILDFLAG(USE_STARBOARD_MEDIA) std::move(callback).Run(Status::kAborted, {}, audio_config, video_config); +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) return; } output_mojo_buffers.emplace_back(std::move(mojo_buffer)); @@ -119,8 +133,13 @@ void MojoDemuxerStreamImpl::OnBufferReady( // TODO(dalecurtis): Once we can write framed data to the DataPipe, fill via // the producer handle and then read more to keep the pipe full. Waiting for // space can be accomplished using an AsyncWaiter. +#if BUILDFLAG(USE_STARBOARD_MEDIA) + std::move(callback).Run(status, std::move(output_mojo_buffers), audio_config, + video_config, /*mime_type=*/std::nullopt); +#else // BUILDFLAG(USE_STARBOARD_MEDIA) std::move(callback).Run(status, std::move(output_mojo_buffers), audio_config, video_config); +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) } } // namespace media diff --git a/media/mojo/mojom/audio_decoder_config_mojom_traits.cc b/media/mojo/mojom/audio_decoder_config_mojom_traits.cc index d148083ec560..a4639d9d2f5f 100644 --- a/media/mojo/mojom/audio_decoder_config_mojom_traits.cc +++ b/media/mojo/mojom/audio_decoder_config_mojom_traits.cc @@ -53,6 +53,9 @@ bool StructTraitsInitialize(codec, sample_format, channel_layout, input.samples_per_second(), std::move(extra_data), encryption_scheme, seek_preroll, input.codec_delay()); +#if BUILDFLAG(USE_STARBOARD_MEDIA) + output->set_is_change_type_transition(input.is_change_type_transition()); +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) output->set_profile(profile); output->set_target_output_channel_layout(target_output_channel_layout); output->set_target_output_sample_format(target_output_sample_format); diff --git a/media/mojo/mojom/audio_decoder_config_mojom_traits.h b/media/mojo/mojom/audio_decoder_config_mojom_traits.h index efec4db7d753..d472d76fbceb 100644 --- a/media/mojo/mojom/audio_decoder_config_mojom_traits.h +++ b/media/mojo/mojom/audio_decoder_config_mojom_traits.h @@ -70,6 +70,12 @@ struct StructTraits (Status status, array batch_buffers, AudioDecoderConfig? audio_config, VideoDecoderConfig? video_config); + [EnableIf=use_starboard_media] + // Passes updated stream mime_type during kConfigChanged events for Starboard. + Read(uint32 count) => (Status status, + array batch_buffers, + AudioDecoderConfig? audio_config, + VideoDecoderConfig? video_config, + string? mime_type); + // Enables converting bitstream to a format that is expected by the decoder. // For example, H.264/AAC bitstream based packets into H.264 Annex B format. EnableBitstreamConverter(); diff --git a/media/mojo/mojom/media_types.mojom b/media/mojo/mojom/media_types.mojom index d649d9e890a2..f7e4ed84d422 100644 --- a/media/mojo/mojom/media_types.mojom +++ b/media/mojo/mojom/media_types.mojom @@ -206,6 +206,8 @@ struct AudioDecoderConfig { ChannelLayout target_output_channel_layout; SampleFormat target_output_sample_format; bool should_discard_decoder_delay; + [EnableIf=use_starboard_media] + bool is_change_type_transition; }; // This defines a mojo transport format for media::VideoDecoderConfig. @@ -224,6 +226,8 @@ struct VideoDecoderConfig { EncryptionScheme encryption_scheme; VideoColorSpace color_space_info; gfx.mojom.HDRMetadata? hdr_metadata; + [EnableIf=use_starboard_media] + bool is_change_type_transition; }; // Native struct media::SubsampleEntry; diff --git a/media/mojo/mojom/video_decoder_config_mojom_traits.cc b/media/mojo/mojom/video_decoder_config_mojom_traits.cc index d1b2a06ab444..4b9cd0976cbc 100644 --- a/media/mojo/mojom/video_decoder_config_mojom_traits.cc +++ b/media/mojo/mojom/video_decoder_config_mojom_traits.cc @@ -70,6 +70,10 @@ bool StructTraitsset_hdr_metadata(hdr_metadata.value()); +#if BUILDFLAG(USE_STARBOARD_MEDIA) + output->set_is_change_type_transition(input.is_change_type_transition()); +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) + if (!output->IsValidConfig()) return false; diff --git a/media/mojo/mojom/video_decoder_config_mojom_traits.h b/media/mojo/mojom/video_decoder_config_mojom_traits.h index e313c5a78a10..2b91eef69ad6 100644 --- a/media/mojo/mojom/video_decoder_config_mojom_traits.h +++ b/media/mojo/mojom/video_decoder_config_mojom_traits.h @@ -81,6 +81,12 @@ struct StructTraits batch_buffers, const std::optional& audio_config, +#if BUILDFLAG(USE_STARBOARD_MEDIA) + const std::optional& video_config, + const std::optional& mime_type) { +#else // BUILDFLAG(USE_STARBOARD_MEDIA) const std::optional& video_config) { +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) DVLOG(3) << __func__ << ": status=" << ::media::DemuxerStream::GetStatusName(status) << ", batch_buffers.size=" << batch_buffers.size(); @@ -114,6 +119,11 @@ void MojoDemuxerStreamAdapter::OnBufferReady( DCHECK_NE(type_, UNKNOWN); if (status == kConfigChanged) { +#if BUILDFLAG(USE_STARBOARD_MEDIA) + if (mime_type.has_value()) { + mime_type_ = mime_type.value(); + } +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) UpdateConfig(std::move(audio_config), std::move(video_config)); std::move(read_cb_).Run(kConfigChanged, {}); return; diff --git a/media/mojo/services/mojo_demuxer_stream_adapter.h b/media/mojo/services/mojo_demuxer_stream_adapter.h index 8ecd8934f262..aae361c0d2e1 100644 --- a/media/mojo/services/mojo_demuxer_stream_adapter.h +++ b/media/mojo/services/mojo_demuxer_stream_adapter.h @@ -69,7 +69,12 @@ class MEDIA_MOJO_EXPORT MojoDemuxerStreamAdapter : public DemuxerStream { void OnBufferReady(Status status, std::vector batch_buffers, const std::optional& audio_config, +#if BUILDFLAG(USE_STARBOARD_MEDIA) + const std::optional& video_config, + const std::optional& mime_type); +#else // BUILDFLAG(USE_STARBOARD_MEDIA) const std::optional& video_config); +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) void OnBufferRead(scoped_refptr buffer); diff --git a/media/starboard/starboard_renderer.cc b/media/starboard/starboard_renderer.cc index e9d26c540503..1cd401ffc385 100644 --- a/media/starboard/starboard_renderer.cc +++ b/media/starboard/starboard_renderer.cc @@ -752,6 +752,39 @@ void StarboardRenderer::UpdateDecoderConfig(DemuxerStream* stream) { } } +void StarboardRenderer::ApplyPendingVideoConfig() { + DCHECK(task_runner_->RunsTasksInCurrentSequence()); + DCHECK(pending_video_config_.has_value()); + DCHECK(video_stream_); + + LOG(INFO) + << "Applying pending Video config change from changeType transition."; + + client_->OnVideoConfigChange(*pending_video_config_); + client_->OnVideoNaturalSizeChange( + pending_video_config_->visible_rect().size()); + paint_video_hole_frame_cb_.Run(pending_video_config_->visible_rect().size()); + + UpdateDecoderConfig(video_stream_); + + pending_video_config_.reset(); +} + +void StarboardRenderer::ApplyPendingAudioConfig() { + DCHECK(task_runner_->RunsTasksInCurrentSequence()); + DCHECK(pending_audio_config_.has_value()); + DCHECK(audio_stream_); + + LOG(INFO) + << "Applying pending Audio config change from changeType transition."; + + client_->OnAudioConfigChange(*pending_audio_config_); + + UpdateDecoderConfig(audio_stream_); + + pending_audio_config_.reset(); +} + void StarboardRenderer::OnDemuxerStreamRead( DemuxerStream* stream, int max_buffers, @@ -788,6 +821,9 @@ void StarboardRenderer::OnDemuxerStreamRead( if (stream == audio_stream_) { DCHECK(audio_read_in_progress_); audio_read_in_progress_ = false; + if (pending_audio_config_.has_value()) { + ApplyPendingAudioConfig(); + } for (const auto& buffer : buffers) { if (!buffer->end_of_stream()) { last_audio_sample_interval_ = @@ -799,6 +835,9 @@ void StarboardRenderer::OnDemuxerStreamRead( } else { DCHECK(video_read_in_progress_); video_read_in_progress_ = false; + if (pending_video_config_.has_value()) { + ApplyPendingVideoConfig(); + } for (const auto& buffer : buffers) { if (buffer->end_of_stream()) { is_video_eos_written_ = true; @@ -822,17 +861,32 @@ void StarboardRenderer::OnDemuxerStreamRead( } } else if (status == DemuxerStream::kConfigChanged) { if (stream == audio_stream_) { - client_->OnAudioConfigChange(stream->audio_decoder_config()); + const AudioDecoderConfig& config = stream->audio_decoder_config(); + if (config.is_change_type_transition()) { + pending_audio_config_ = config; + LOG(INFO) + << "Pending Audio config change stored due to a changeType call."; + } else { + pending_audio_config_.reset(); + client_->OnAudioConfigChange(config); + UpdateDecoderConfig(stream); + } } else { DCHECK_EQ(stream, video_stream_); - client_->OnVideoConfigChange(stream->video_decoder_config()); - // TODO(b/375275033): Refine calling to OnVideoNaturalSizeChange(). - client_->OnVideoNaturalSizeChange( - stream->video_decoder_config().visible_rect().size()); - paint_video_hole_frame_cb_.Run( - stream->video_decoder_config().visible_rect().size()); + const VideoDecoderConfig& config = stream->video_decoder_config(); + if (config.is_change_type_transition()) { + pending_video_config_ = config; + LOG(INFO) + << "Pending Video config change stored due to a changeType call."; + } else { + pending_video_config_.reset(); + client_->OnVideoConfigChange(config); + // TODO(b/375275033): Refine calling to OnVideoNaturalSizeChange(). + client_->OnVideoNaturalSizeChange(config.visible_rect().size()); + paint_video_hole_frame_cb_.Run(config.visible_rect().size()); + UpdateDecoderConfig(stream); + } } - UpdateDecoderConfig(stream); stream->Read( max_buffers, base::BindOnce(&StarboardRenderer::OnDemuxerStreamRead, diff --git a/media/starboard/starboard_renderer.h b/media/starboard/starboard_renderer.h index 03ad7cebd201..2653c46857e3 100644 --- a/media/starboard/starboard_renderer.h +++ b/media/starboard/starboard_renderer.h @@ -153,6 +153,8 @@ class MEDIA_EXPORT StarboardRenderer : public Renderer, void CreatePlayerBridge(); void ApplyPendingBounds(); void UpdateDecoderConfig(DemuxerStream* stream); + void ApplyPendingAudioConfig(); + void ApplyPendingVideoConfig(); void OnDemuxerStreamRead(DemuxerStream* stream, int max_buffers, DemuxerStream::Status status, @@ -231,6 +233,15 @@ class MEDIA_EXPORT StarboardRenderer : public Renderer, std::optional output_rect_; + // Stored video/audio configs to store the new config from a + // SourceBuffer.changeType() call. Configs from changeType() are not applied + // to StarboardRenderer when DemuxerStream::kConfigChanged occurs, but when + // the first sample of the updated player config is applied. We do this to + // prevent in-flight samples from being written with the incorrect decoder + // configuration. + std::optional pending_audio_config_; + std::optional pending_video_config_; + // Temporary callback used for Initialize(). PipelineStatusCallback init_cb_; diff --git a/media/test/test_media_source.cc b/media/test/test_media_source.cc index 088c9621bc13..b9c1f3d58b9d 100644 --- a/media/test/test_media_source.cc +++ b/media/test/test_media_source.cc @@ -293,11 +293,16 @@ ChunkDemuxer::Status TestMediaSource::AddId() { void TestMediaSource::ChangeType(const std::string& mimetype) { chunk_demuxer_->ResetParserState(kSourceId, base::TimeDelta(), kInfiniteDuration, &last_timestamp_offset_); +#if BUILDFLAG(USE_STARBOARD_MEDIA) + mimetype_ = mimetype; + chunk_demuxer_->ChangeType(kSourceId, mimetype); +#else // BUILDFLAG(USE_STARBOARD_MEDIA) std::string type; std::string codecs; SplitMime(mimetype, &type, &codecs); mimetype_ = mimetype; chunk_demuxer_->ChangeType(kSourceId, type, codecs); +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) } void TestMediaSource::OnEncryptedMediaInitData( diff --git a/starboard/android/shared/media_codec_video_decoder.cc b/starboard/android/shared/media_codec_video_decoder.cc index 13a2b26fe885..b44ee8dc3618 100644 --- a/starboard/android/shared/media_codec_video_decoder.cc +++ b/starboard/android/shared/media_codec_video_decoder.cc @@ -326,6 +326,7 @@ MediaCodecVideoDecoder::MediaCodecVideoDecoder( std::string* error_message) : JobOwner(job_queue), video_codec_(stream_config.video_stream_info.codec), + video_mime_(stream_config.video_stream_info.mime), drm_system_(static_cast(stream_config.drm_system)), output_mode_(stream_config.output_mode), decode_target_graphics_context_provider_( @@ -517,6 +518,56 @@ void MediaCodecVideoDecoder::WriteInputBuffers( input_buffers.front()->timestamp(), "size", input_buffers.size()); + SbMediaVideoCodec new_codec = + input_buffers.front()->video_stream_info().codec; + const std::string& new_mime = input_buffers.front()->video_stream_info().mime; + const auto& new_color_metadata = + input_buffers.front()->video_stream_info().color_metadata; + + bool color_metadata_changed = false; + if (color_metadata_.has_value()) { + color_metadata_changed = (*color_metadata_ != new_color_metadata); + } else { + color_metadata_changed = !IsIdentity(new_color_metadata); + } + + bool stream_info_changed = (new_codec != video_codec_) || + (!new_mime.empty() && new_mime != video_mime_) || + color_metadata_changed; + + if (just_reset_) { + just_reset_ = false; + if (stream_info_changed) { + SB_LOG(INFO) + << "TEST: Stream info change detected after Reset. Recreating " + "codec immediately without draining..."; + TeardownCodec(); + video_codec_ = new_codec; + video_mime_ = new_mime; + color_metadata_= new_color_metadata; + input_buffer_written_ = 0; + video_fps_ = 0; + } + } + + if (input_buffer_written_ > 0 && stream_info_changed && !draining_codec_) { + draining_codec_ = true; + pending_video_codec_ = new_codec; + pending_video_stream_info_ = input_buffers.front()->video_stream_info(); + + pending_input_buffers_.insert(pending_input_buffers_.end(), + input_buffers.begin(), input_buffers.end()); + + media_decoder_->WriteEndOfStream(); + return; + } + + if (draining_codec_) { + pending_input_buffers_.insert(pending_input_buffers_.end(), + input_buffers.begin(), input_buffers.end()); + return; + } + if (input_buffer_written_ == 0) { SB_DCHECK_EQ(video_fps_, 0); first_buffer_timestamp_ = input_buffers.front()->timestamp(); @@ -977,6 +1028,21 @@ void MediaCodecVideoDecoder::ProcessOutputBuffer( bool is_end_of_stream = dequeue_output_result.flags & MediaCodec::kBufferFlagEndOfStream; + + if (is_end_of_stream && draining_codec_) { + SB_LOG(INFO) << "TEST: EOS received during draining."; + eos_received_in_draining_ = true; + media_codec_bridge->ReleaseOutputBuffer(dequeue_output_result.index, false); + if (buffered_output_frames_ == 0) { + SB_LOG(INFO) << "TEST: Old video fully rendered on screen. Scheduling " + "internal hot-swap on worker thread."; + Schedule(std::bind(&MediaCodecVideoDecoder::PerformInternalDecoderHotSwap, + this)); + } + decoder_status_cb_(kNeedMoreInput, NULL); + return; + } + if (!is_end_of_stream) { ++decoded_output_frames_; if (output_format_) { @@ -1143,6 +1209,14 @@ void MediaCodecVideoDecoder::OnVideoFrameRelease() { --buffered_output_frames_; SB_DCHECK_GE(buffered_output_frames_, 0); } + + if (draining_codec_ && eos_received_in_draining_ && + buffered_output_frames_ == 0) { + SB_LOG(INFO) << "TEST: Old video fully rendered on screen. Scheduling " + "internal hot-swap on worker thread."; + Schedule(std::bind(&MediaCodecVideoDecoder::PerformInternalDecoderHotSwap, + this)); + } } void MediaCodecVideoDecoder::OnSurfaceDestroyed() { @@ -1212,6 +1286,10 @@ void MediaCodecVideoDecoder::ResetInternal(bool skip_flush) { tunnel_mode_prerolled_frames_.store(0); end_of_stream_written_ = false; pending_input_buffers_.clear(); + just_reset_ = true; + draining_codec_ = false; + eos_received_in_draining_ = false; + pending_video_codec_ = kSbMediaVideoCodecNone; // TODO: We rely on VideoRenderAlgorithmTunneled::Seek() to be called inside // VideoRenderer::Seek() after calling MediaCodecVideoDecoder::Reset() @@ -1219,4 +1297,53 @@ void MediaCodecVideoDecoder::ResetInternal(bool skip_flush) { // slightly flaky as it depends on the behavior of the video renderer. } +void MediaCodecVideoDecoder::PerformInternalDecoderHotSwap() { + SB_CHECK(BelongsToCurrentThread()); + SB_LOG(INFO) << "TEST: Performing internal decoder hot-swap from " + << GetMediaVideoCodecName(video_codec_) << " to " + << GetMediaVideoCodecName(pending_video_codec_); + + TeardownCodec(); + + video_codec_ = pending_video_codec_; + video_mime_ = pending_video_stream_info_.mime; + needs_fps_to_initialize_codec_ = + (video_codec_ == kSbMediaVideoCodecAv1 && + MediaCapabilitiesCache::GetInstance()->IsAv18kCappedAt30()); + + input_buffer_written_ = 0; + video_fps_ = 0; + draining_codec_ = false; + eos_received_in_draining_ = false; + + if (IsIdentity(pending_video_stream_info_.color_metadata)) { + SbMediaColorMetadata sdr_metadata = {}; + sdr_metadata.primaries = kSbMediaPrimaryIdBt709; + sdr_metadata.transfer = kSbMediaTransferIdBt709; + sdr_metadata.matrix = kSbMediaMatrixIdBt709; + sdr_metadata.range = kSbMediaRangeIdLimited; + color_metadata_ = sdr_metadata; + } else { + color_metadata_ = pending_video_stream_info_.color_metadata; + } + + auto result = InitializeCodec(pending_video_stream_info_); + if (!result) { + std::string error_message = + "Failed to initialize video decoder internally: " + result.error(); + SB_LOG(ERROR) << error_message; + TeardownCodec(); + ReportError(kSbPlayerErrorDecode, error_message); + return; + } + + if (!pending_input_buffers_.empty()) { + SB_LOG(INFO) << "TEST: Feeding " << pending_input_buffers_.size() + << " queued new samples to the new decoder."; + input_buffer_written_ += pending_input_buffers_.size(); + WriteInputBuffersInternal(pending_input_buffers_); + pending_input_buffers_.clear(); + } +} + } // namespace starboard diff --git a/starboard/android/shared/media_codec_video_decoder.h b/starboard/android/shared/media_codec_video_decoder.h index 2da713c664b4..5a7dfbf0d9ca 100644 --- a/starboard/android/shared/media_codec_video_decoder.h +++ b/starboard/android/shared/media_codec_video_decoder.h @@ -170,7 +170,8 @@ class MediaCodecVideoDecoder : public VideoDecoder, // These variables will be initialized inside ctor or Initialize() and will // not be changed during the life time of this class. - const SbMediaVideoCodec video_codec_; + SbMediaVideoCodec video_codec_; + std::string video_mime_; DecoderStatusCB decoder_status_cb_; ErrorCB error_cb_; DrmSystem* drm_system_; @@ -212,7 +213,7 @@ class MediaCodecVideoDecoder : public VideoDecoder, // Codec initialization will be delayed until the decoder receives enough // inputs to estimate video fps when |needs_fps_to_initialize_codec_| is true. - const bool needs_fps_to_initialize_codec_; + bool needs_fps_to_initialize_codec_; // Workaround for b/506257255, which skips some video frames to make the // effective frame rate no more than 60 fps. This is only used for tunnel // mode. @@ -302,6 +303,15 @@ class MediaCodecVideoDecoder : public VideoDecoder, const PlatformOptions& platform_options); const std::unique_ptr surface_texture_bridge_; + + // Mid-stream seamless codec change draining variables + bool draining_codec_ = false; + bool eos_received_in_draining_ = false; + SbMediaVideoCodec pending_video_codec_ = kSbMediaVideoCodecNone; + VideoStreamInfo pending_video_stream_info_; + bool just_reset_ = false; + + void PerformInternalDecoderHotSwap(); }; } // namespace starboard diff --git a/starboard/shared/stub/media_can_change_type.cc b/starboard/shared/stub/media_can_change_type.cc index 69e2f11dbc3b..fc074330f545 100644 --- a/starboard/shared/stub/media_can_change_type.cc +++ b/starboard/shared/stub/media_can_change_type.cc @@ -15,5 +15,5 @@ #include "starboard/media.h" bool SbMediaCanChangeType(const char* current_mime, const char* new_mime) { - return false; + return true; } diff --git a/third_party/blink/public/platform/web_source_buffer.h b/third_party/blink/public/platform/web_source_buffer.h index a2e493b5d8b0..ea37865099e0 100644 --- a/third_party/blink/public/platform/web_source_buffer.h +++ b/third_party/blink/public/platform/web_source_buffer.h @@ -118,6 +118,12 @@ class WebSourceBuffer { virtual void ResetParserState() = 0; virtual void Remove(double start, double end) = 0; +#if BUILDFLAG(USE_STARBOARD_MEDIA) + // Special Starboard versions of CanChangeType and ChangeType that accept the + // raw mime_type string passed from the webapp. + virtual bool CanChangeType(const WebString& mime_type) = 0; + virtual void ChangeType(const WebString& mime_type) = 0; +#else // BUILDFLAG(USE_STARBOARD_MEDIA) // Returns true iff this SourceBuffer supports changing bytestream and codecs // to |content_type| and |codecs|. |content_type| is the ContentType string // of the bytestream's MIME type, and |codecs| contains the "codecs" parameter @@ -132,6 +138,7 @@ class WebSourceBuffer { // |content_type| and |codecs| parameters. virtual void ChangeType(const WebString& content_type, const WebString& codecs) = 0; +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) virtual bool SetTimestampOffset(double) = 0; diff --git a/third_party/blink/renderer/modules/mediasource/source_buffer.cc b/third_party/blink/renderer/modules/mediasource/source_buffer.cc index 9c5c507d4a58..d0df77fdaefd 100644 --- a/third_party/blink/renderer/modules/mediasource/source_buffer.cc +++ b/third_party/blink/renderer/modules/mediasource/source_buffer.cc @@ -1073,6 +1073,17 @@ void SourceBuffer::ChangeType_Locked( DCHECK(!updating_); source_->AssertAttachmentsMutexHeldIfCrossThreadForDebugging(); +#if BUILDFLAG(USE_STARBOARD_MEDIA) + if (!MediaSource::IsTypeSupportedInternal( + GetExecutionContext(), type, + false /* allow underspecified codecs in |type| */) || + !web_source_buffer_->CanChangeType(type)) { + MediaSource::LogAndThrowDOMException( + *exception_state, DOMExceptionCode::kNotSupportedError, + "Changing to the type provided ('" + type + "') is not supported."); + return; + } +#else // BUILDFLAG(USE_STARBOARD_MEDIA) // 4. If type contains a MIME type that is not supported or contains a MIME // type that is not supported with the types specified (currently or // previously) of SourceBuffer objects in the sourceBuffers attribute of @@ -1093,6 +1104,7 @@ void SourceBuffer::ChangeType_Locked( "Changing to the type provided ('" + type + "') is not supported."); return; } +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) // 5. If the readyState attribute of the parent media source is in the "ended" // state then run the following steps: @@ -1108,7 +1120,11 @@ void SourceBuffer::ChangeType_Locked( // value in the "Generate Timestamps Flag" column of the byte stream format // registry entry that is associated with type. // This call also updates the pipeline to switch bytestream parser and codecs. +#if BUILDFLAG(USE_STARBOARD_MEDIA) + web_source_buffer_->ChangeType(type); +#else // BUILDFLAG(USE_STARBOARD_MEDIA) web_source_buffer_->ChangeType(content_type.GetType(), codecs); +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) // 8. If the generate timestamps flag equals true: Set the mode attribute on // this SourceBuffer object to "sequence", including running the associated diff --git a/third_party/blink/renderer/platform/media/web_source_buffer_impl.cc b/third_party/blink/renderer/platform/media/web_source_buffer_impl.cc index 1fe0b483e732..568622f1b756 100644 --- a/third_party/blink/renderer/platform/media/web_source_buffer_impl.cc +++ b/third_party/blink/renderer/platform/media/web_source_buffer_impl.cc @@ -206,6 +206,18 @@ void WebSourceBufferImpl::Remove(double start, double end) { demuxer_->Remove(id_, timedelta_start, timedelta_end); } +#if BUILDFLAG(USE_STARBOARD_MEDIA) +bool WebSourceBufferImpl::CanChangeType(const WebString& mime_type) { + return demuxer_->CanChangeType(id_, mime_type.Utf8()); +} + +void WebSourceBufferImpl::ChangeType(const WebString& mime_type) { + // Caller must first call ResetParserState() to flush any pending frames. + DCHECK(!demuxer_->IsParsingMediaSegment(id_)); + + demuxer_->ChangeType(id_, mime_type.Utf8()); +} +#else bool WebSourceBufferImpl::CanChangeType(const WebString& content_type, const WebString& codecs) { return demuxer_->CanChangeType(id_, content_type.Utf8(), codecs.Utf8()); @@ -218,6 +230,7 @@ void WebSourceBufferImpl::ChangeType(const WebString& content_type, demuxer_->ChangeType(id_, content_type.Utf8(), codecs.Utf8()); } +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) bool WebSourceBufferImpl::SetTimestampOffset(double offset) { if (demuxer_->IsParsingMediaSegment(id_)) diff --git a/third_party/blink/renderer/platform/media/web_source_buffer_impl.h b/third_party/blink/renderer/platform/media/web_source_buffer_impl.h index 6f27c289760f..ee8391f91a30 100644 --- a/third_party/blink/renderer/platform/media/web_source_buffer_impl.h +++ b/third_party/blink/renderer/platform/media/web_source_buffer_impl.h @@ -53,10 +53,15 @@ class PLATFORM_EXPORT WebSourceBufferImpl : public WebSourceBuffer { double* timestamp_offset) override; void ResetParserState() override; void Remove(double start, double end) override; +#if BUILDFLAG(USE_STARBOARD_MEDIA) + bool CanChangeType(const WebString& mime_type) override; + void ChangeType(const WebString& mime_type) override; +#else // BUILDFLAG(USE_STARBOARD_MEDIA) bool CanChangeType(const WebString& content_type, const WebString& codecs) override; void ChangeType(const WebString& content_type, const WebString& codecs) override; +#endif // BUILDFLAG(USE_STARBOARD_MEDIA) bool SetTimestampOffset(double offset) override; void SetAppendWindowStart(double start) override; void SetAppendWindowEnd(double end) override;