/*************************************************** * VlcMediaEngine.cpp required codec-runtime: ubuntu-restricted-extras (contains intel-media-va-driver) sudo apt install ubuntu-restricted-extras */ #if defined(AX_ENABLE_VLC_MEDIA) # include "VlcMediaEngine.h" # include "VlcMediaEngine.h" NS_AX_BEGIN static constexpr auto VLC_OUTPUT_FORMAT = ax::MEVideoPixelFormat::NV12; void* VlcMediaEngine::libvlc_video_lock(void* data, void** p_pixels) { VlcMediaEngine* mediaEngine = static_cast(data); auto& bufferDim = mediaEngine->_videoDim; auto& outputBuffer = mediaEngine->_frameBuffer1; mediaEngine->_frameBuffer1Mtx.lock(); if constexpr (VLC_OUTPUT_FORMAT == ax::MEVideoPixelFormat::NV12) { outputBuffer.resize(bufferDim.x * bufferDim.y + (bufferDim.x * bufferDim.y >> 1)); // NV12 p_pixels[0] = outputBuffer.data(); p_pixels[1] = outputBuffer.data() + (bufferDim.x * bufferDim.y); } else if constexpr (VLC_OUTPUT_FORMAT == ax::MEVideoPixelFormat::YUY2) { outputBuffer.resize(bufferDim.x * bufferDim.y + ((bufferDim.x >> 1) * bufferDim.y * 4)); // YUY2 p_pixels[0] = outputBuffer.data(); } else { outputBuffer.resize(bufferDim.x * bufferDim.y * 4); // RGBA32 p_pixels[0] = outputBuffer.data(); } return nullptr; } void VlcMediaEngine::libvlc_video_unlock(void* data, void* id, void* const* p_pixels) { VlcMediaEngine* mediaEngine = static_cast(data); mediaEngine->_frameBuffer1Mtx.unlock(); ++mediaEngine->_frameIndex; assert(id == nullptr); } unsigned int VlcMediaEngine::libvlc_video_format_setup(void** opaque, char* chroma, // forcc, refer to:vlc_fourcc.h unsigned* width, unsigned* height, unsigned* pitches, unsigned* lines) { // refer to: vmem.c:Open https://github.com/videolan/vlc/blob/3.0.18/modules/video_output/vmem.c#L150 // future: 4.0: will be widths, heights: // https://github.com/videolan/vlc/blob/master/modules/video_output/vmem.c#L156 VlcMediaEngine* mediaEngine = static_cast(*opaque); // vlc tell us the original codecDim(ALIGNED) mediaEngine->_codecDim.set(width[0], height[0]); // tell vlc we want render as video size width[0] = mediaEngine->_videoDim.x; height[0] = mediaEngine->_videoDim.y; // plane0 pitches[0] = width[0]; // bytesPerRow lines[0] = height[0]; // rows # if LIBVLC_VERSION_MAJOR >= 4 mediaEngine->_videoDim.set(width[1], height[1]); # endif int num_of_plane = 1; if constexpr (VLC_OUTPUT_FORMAT == ax::MEVideoPixelFormat::NV12) { memcpy(chroma, "NV12", 4); // plane1 pitches[1] = mediaEngine->_videoDim.x; // bytesPerRow lines[1] = mediaEngine->_videoDim.y >> 1; // rows num_of_plane = 2; } else if constexpr (VLC_OUTPUT_FORMAT == ax::MEVideoPixelFormat::YUY2) { memcpy(chroma, "YUY2", 4); pitches[0] = width[0] * 2; // bytesPerRow } else { memcpy(chroma, "RGBA", 4); pitches[0] = width[0] * 4; // bytesPerRow } // return the number of picture buffers allocated, 0 indicates failure return num_of_plane; } void VlcMediaEngine::libvlc_handle_event(const libvlc_event_t* event, void* userData) { if (!event) return; VlcMediaEngine* mediaEngine = static_cast(userData); switch (event->type) { case libvlc_MediaPlayerPlaying: mediaEngine->_playbackEnded = false; mediaEngine->_state = MEMediaState::Playing; mediaEngine->handleEvent(MEMediaEventType::Playing); break; case libvlc_MediaPlayerPaused: mediaEngine->_state = MEMediaState::Paused; mediaEngine->handleEvent(MEMediaEventType::Paused); break; # if LIBVLC_VERSION_MAJOR < 4 case libvlc_MediaPlayerEndReached: mediaEngine->_playbackEnded = true; mediaEngine->_state = MEMediaState::Stopped; mediaEngine->handleEvent(MEMediaEventType::Stopped); break; case libvlc_MediaPlayerStopped: if (!mediaEngine->_playbackEnded) { mediaEngine->_state = MEMediaState::Stopped; mediaEngine->handleEvent(MEMediaEventType::Stopped); } break; # else case libvlc_MediaPlayerStopped: mediaEngine->_playbackEnded = true; mediaEngine->_state = MEMediaState::Stopped; mediaEngine->handleEvent(MEMediaEventType::Stopped); break; case libvlc_MediaListPlayerStopped: if (!mediaEngine->_playbackEnded) { mediaEngine->_state = MEMediaState::Stopped; mediaEngine->handleEvent(MEMediaEventType::Stopped); } break; # endif case libvlc_MediaPlayerEncounteredError: mediaEngine->_state = MEMediaState::Error; mediaEngine->handleEvent(MEMediaEventType::Error); break; case libvlc_MediaPlayerMediaChanged: mediaEngine->_frameIndex = 0; break; default:; } } VlcMediaEngine::VlcMediaEngine() { AXLOG("libvlc-version: %s", libvlc_get_version()); // too late set vlc plugins path at hete, vlc maybe read it at DllMain //_putenv_s("VLC_PLUGIN_PATH", R"(D:\dev\axmol\3rdparty\vlc\win\lib)"); _vlc = libvlc_new(0, nullptr); if (!_vlc) { AXLOGI("VlcMediaEngine: libvlc_new fail, ensure install vlc and ubuntu-restricted-extras"); return; } // media player _mp = libvlc_media_player_new(_vlc); libvlc_video_set_callbacks(_mp, libvlc_video_lock, libvlc_video_unlock, nullptr, this); libvlc_video_set_format_callbacks(_mp, libvlc_video_format_setup, nullptr); // media list # if LIBVLC_VERSION_MAJOR < 4 _ml = libvlc_media_list_new(_vlc); # else _ml = libvlc_media_list_new(); # endif // media list player _mlp = libvlc_media_list_player_new(_vlc); libvlc_media_list_player_set_media_player(_mlp, _mp); libvlc_media_list_player_set_media_list(_mlp, _ml); // register media player events _mp_events = libvlc_media_player_event_manager(_mp); libvlc_event_attach(_mp_events, libvlc_MediaPlayerPlaying, libvlc_handle_event, this); libvlc_event_attach(_mp_events, libvlc_MediaPlayerPaused, libvlc_handle_event, this); libvlc_event_attach(_mp_events, libvlc_MediaPlayerStopped, libvlc_handle_event, this); libvlc_event_attach(_mp_events, libvlc_MediaPlayerMediaChanged, libvlc_handle_event, this); # if LIBVLC_VERSION_MAJOR < 4 libvlc_event_attach(_mp_events, libvlc_MediaPlayerEndReached, libvlc_handle_event, this); # endif libvlc_event_attach(_mp_events, libvlc_MediaPlayerEncounteredError, libvlc_handle_event, this); // register media list player events _mpl_events = libvlc_media_list_player_event_manager(_mlp); libvlc_event_attach(_mpl_events, libvlc_MediaListEndReached, libvlc_handle_event, this); libvlc_event_attach(_mpl_events, libvlc_MediaListPlayerStopped, libvlc_handle_event, this); } VlcMediaEngine::~VlcMediaEngine() { if (_mpl_events) { libvlc_event_detach(_mpl_events, libvlc_MediaListEndReached, libvlc_handle_event, this); libvlc_event_detach(_mpl_events, libvlc_MediaListPlayerStopped, libvlc_handle_event, this); } if (_mp_events) { libvlc_event_detach(_mp_events, libvlc_MediaPlayerPlaying, libvlc_handle_event, this); libvlc_event_detach(_mp_events, libvlc_MediaPlayerPaused, libvlc_handle_event, this); libvlc_event_detach(_mp_events, libvlc_MediaPlayerStopped, libvlc_handle_event, this); libvlc_event_detach(_mp_events, libvlc_MediaPlayerMediaChanged, libvlc_handle_event, this); # if LIBVLC_VERSION_MAJOR < 4 libvlc_event_detach(_mp_events, libvlc_MediaPlayerEndReached, libvlc_handle_event, this); # endif libvlc_event_detach(_mp_events, libvlc_MediaPlayerEncounteredError, libvlc_handle_event, this); } if (_mlp) { # if LIBVLC_VERSION_MAJOR < 4 libvlc_media_list_player_stop(_mlp); # else libvlc_media_list_player_stop_async(_mlp); # endif libvlc_media_list_player_release(_mlp); } if (_ml) libvlc_media_list_release(_ml); if (_mp) libvlc_media_player_release(_mp); if (_vlc) libvlc_release(_vlc); } void VlcMediaEngine::handleEvent(MEMediaEventType event) { if (event == MEMediaEventType::Playing) updatePlaybackProperties(); fireMediaEvent(event); } void VlcMediaEngine::setAutoPlay(bool bAutoPlay) { _bAutoPlay = bAutoPlay; } bool VlcMediaEngine::open(std::string_view sourceUri) { if (_state != MEMediaState::Closed) return false; # if LIBVLC_VERSION_MAJOR < 4 libvlc_media_t* media = libvlc_media_new_location(_vlc, sourceUri.data()); # else libvlc_media_t* media = libvlc_media_new_location(sourceUri.data()); # endif if (!media) return false; _frameIndex = 0; _sourceUri = sourceUri; libvlc_media_list_add_media(_ml, media); // always one media libvlc_media_release(media); if (_bAutoPlay) { _state = MEMediaState::Preparing; libvlc_media_list_player_play(_mlp); } else _state = MEMediaState::Stopped; return true; } static void track_codec_to_mime_type(libvlc_media_track_t* track, std::string& out) { out.clear(); out.reserve(9); // 4 * 2 + 1 out.append(reinterpret_cast(&track->i_codec), 4); out.push_back('/'); out.append(reinterpret_cast(&track->i_original_fourcc), 4); } bool VlcMediaEngine::updatePlaybackProperties() { if (_frameIndex == 0) { auto media = libvlc_media_player_get_media(_mp); # if LIBVLC_VERSION_MAJOR < 4 /* local file, we Get the size of the video with the tracks information of the media. */ libvlc_media_track_t** tracks{}; unsigned int nbTracks = libvlc_media_tracks_get(media, &tracks); if (!nbTracks) return false; for (unsigned int i = 0; i < nbTracks; ++i) { auto track = tracks[i]; if (track->i_type == libvlc_track_video) { track_codec_to_mime_type(track, _videoCodecMimeType); AXLOGD("VlcMediaEngine: sourceUri: {}, codec: {}", _sourceUri, _videoCodecMimeType); auto vdi = track->video; auto vdw = vdi->i_width; auto vdh = vdi->i_height; switch (vdi->i_orientation) { case libvlc_video_orient_left_bottom: /**< Rotated 90 degrees clockwise (or 270 anti-clockwise) */ case libvlc_video_orient_right_top: /**< Rotated 90 degrees anti-clockwise */ std::swap(vdw, vdh); default:; } _videoDim.set(vdw, vdh); break; } } libvlc_media_tracks_release(tracks, nbTracks); # else auto tracks = libvlc_media_get_tracklist(media, libvlc_track_type_t::libvlc_track_video); if (tracks) { auto nTrack = libvlc_media_tracklist_count(tracks); if (nTrack > 0) { auto track = libvlc_media_tracklist_at(tracks, 0); track_codec_to_mime_type(track, _videoCodecMimeType); AXLOGI("VlcMediaEngine: sourceUri: {}, codec: {}", _sourceUri, _videoCodecMimeType); auto vdi = track->video; _videoDim.set(vdi->i_width, vdi->i_height); } libvlc_media_tracklist_delete(tracks); } # endif // set playback mode libvlc_media_list_player_set_playback_mode( _mlp, _looping ? libvlc_playback_mode_repeat : libvlc_playback_mode_default); } return true; } bool VlcMediaEngine::close() { if (libvlc_media_list_count(_ml) > 0) { # if LIBVLC_VERSION_MAJOR < 4 libvlc_media_list_player_stop(_mlp); # else libvlc_media_list_player_stop_async(_mlp); # endif libvlc_media_list_remove_index(_ml, 0); } _state = MEMediaState::Closed; return true; } bool VlcMediaEngine::setLoop(bool bLooping) { _looping = bLooping; if (getState() == MEMediaState::Playing) libvlc_media_list_player_set_playback_mode( _mlp, _looping ? libvlc_playback_mode_repeat : libvlc_playback_mode_default); return true; } bool VlcMediaEngine::setRate(double fRate) { return _mp && libvlc_media_player_set_rate(_mp, static_cast(fRate)) == 0; } bool VlcMediaEngine::setCurrentTime(double fSeekTimeInSec) { if (_mp) # if LIBVLC_VERSION_MAJOR < 4 libvlc_media_player_set_time(_mp, static_cast(fSeekTimeInSec * 1000)); # else libvlc_media_player_set_time(_mp, static_cast(fSeekTimeInSec * 1000), true); # endif return true; } bool VlcMediaEngine::play() { if (_mlp && _state != MEMediaState::Closed) libvlc_media_list_player_play(_mlp); return true; } bool VlcMediaEngine::pause() { if (_mlp && _state != MEMediaState::Closed) libvlc_media_list_player_pause(_mlp); return true; } bool VlcMediaEngine::stop() { if (_mlp && _state != MEMediaState::Closed) # if LIBVLC_VERSION_MAJOR < 4 libvlc_media_list_player_stop(_mlp); # else libvlc_media_list_player_stop_async(_mlp); # endif return true; } MEMediaState VlcMediaEngine::getState() const { return _state; } bool VlcMediaEngine::transferVideoFrame() { if (_frameBuffer1.empty()) return false; std::unique_lock lck(_frameBuffer1Mtx); if (UTILS_LIKELY(!_frameBuffer1.empty())) { _frameBuffer2.swap(_frameBuffer1); lck.unlock(); // unlock immidiately before invoke user callback (maybe upload buffer to GPU) auto& buffer = _frameBuffer2; ax::MEVideoFrame frame{buffer.data(), buffer.data() + _videoDim.x * _videoDim.y, buffer.size(), ax::MEVideoPixelDesc{VLC_OUTPUT_FORMAT, _videoDim}, _videoDim}; // assert(static_cast(frame._dataLen) >= frame._vpd._dim.x * frame._vpd._dim.y * 3 / 2); _onVideoFrame(frame); _frameBuffer2.clear(); return true; } return false; } NS_AX_END #endif