axmol/core/media/VlcMediaEngine.cpp

/***************************************************
* VlcMediaEngine.cpp

required codec-runtime: ubuntu-restricted-extras (contains intel-media-va-driver)
sudo apt install ubuntu-restricted-extras

*/
#if defined(AX_ENABLE_VLC_MEDIA)
#    include "VlcMediaEngine.h"

#    include "VlcMediaEngine.h"

NS_AX_BEGIN

static constexpr auto VLC_OUTPUT_FORMAT = ax::MEVideoPixelFormat::NV12;

void* VlcMediaEngine::libvlc_video_lock(void* data, void** p_pixels)
{
    VlcMediaEngine* mediaEngine = static_cast<VlcMediaEngine*>(data);

    auto& bufferDim    = mediaEngine->_videoDim;
    auto& outputBuffer = mediaEngine->_frameBuffer1;
    mediaEngine->_frameBuffer1Mtx.lock();
    if constexpr (VLC_OUTPUT_FORMAT == ax::MEVideoPixelFormat::NV12)
    {
        outputBuffer.resize(bufferDim.x * bufferDim.y + (bufferDim.x * bufferDim.y >> 1));  // NV12
        p_pixels[0] = outputBuffer.data();
        p_pixels[1] = outputBuffer.data() + (bufferDim.x * bufferDim.y);
    }
    else if constexpr (VLC_OUTPUT_FORMAT == ax::MEVideoPixelFormat::YUY2)
    {
        outputBuffer.resize(bufferDim.x * bufferDim.y + ((bufferDim.x >> 1) * bufferDim.y * 4));  // YUY2
        p_pixels[0] = outputBuffer.data();
    }
    else
    {
        outputBuffer.resize(bufferDim.x * bufferDim.y * 4);  // RGBA32
        p_pixels[0] = outputBuffer.data();
    }
    return nullptr;
}

void VlcMediaEngine::libvlc_video_unlock(void* data, void* id, void* const* p_pixels)
{
    VlcMediaEngine* mediaEngine = static_cast<VlcMediaEngine*>(data);

    mediaEngine->_frameBuffer1Mtx.unlock();

    ++mediaEngine->_frameIndex;

    assert(id == nullptr);
}

unsigned int VlcMediaEngine::libvlc_video_format_setup(void** opaque,
                                                       char* chroma,  // forcc, refer to:vlc_fourcc.h
                                                       unsigned* width,
                                                       unsigned* height,
                                                       unsigned* pitches,
                                                       unsigned* lines)
{
    // refer to: vmem.c:Open https://github.com/videolan/vlc/blob/3.0.18/modules/video_output/vmem.c#L150
    // future: 4.0: will be widths, heights:
    // https://github.com/videolan/vlc/blob/master/modules/video_output/vmem.c#L156
    VlcMediaEngine* mediaEngine = static_cast<VlcMediaEngine*>(*opaque);

    // vlc tell us the original codecDim(ALIGNED)
    mediaEngine->_codecDim.set(width[0], height[0]);

    // tell vlc we want render as video size
    width[0]  = mediaEngine->_videoDim.x;
    height[0] = mediaEngine->_videoDim.y;

    // plane0
    pitches[0] = width[0];   // bytesPerRow
    lines[0]   = height[0];  // rows

#    if LIBVLC_VERSION_MAJOR >= 4
    mediaEngine->_videoDim.set(width[1], height[1]);
#    endif

    int num_of_plane = 1;

    if constexpr (VLC_OUTPUT_FORMAT == ax::MEVideoPixelFormat::NV12)
    {
        memcpy(chroma, "NV12", 4);

        // plane1
        pitches[1] = mediaEngine->_videoDim.x;       // bytesPerRow
        lines[1]   = mediaEngine->_videoDim.y >> 1;  // rows

        num_of_plane = 2;
    }
    else if constexpr (VLC_OUTPUT_FORMAT == ax::MEVideoPixelFormat::YUY2)
    {
        memcpy(chroma, "YUY2", 4);
        pitches[0] = width[0] * 2;  // bytesPerRow
    }
    else
    {
        memcpy(chroma, "RGBA", 4);
        pitches[0] = width[0] * 4;  // bytesPerRow
    }

    // return the number of picture buffers allocated, 0 indicates failure
    return num_of_plane;
}

void VlcMediaEngine::libvlc_handle_event(const libvlc_event_t* event, void* userData)
{
    if (!event)
        return;
    VlcMediaEngine* mediaEngine = static_cast<VlcMediaEngine*>(userData);
    switch (event->type)
    {
    case libvlc_MediaPlayerPlaying:
        mediaEngine->_playbackEnded = false;
        mediaEngine->_state         = MEMediaState::Playing;
        mediaEngine->handleEvent(MEMediaEventType::Playing);
        break;
    case libvlc_MediaPlayerPaused:
        mediaEngine->_state = MEMediaState::Paused;
        mediaEngine->handleEvent(MEMediaEventType::Paused);
        break;
#    if LIBVLC_VERSION_MAJOR < 4
    case libvlc_MediaPlayerEndReached:
        mediaEngine->_playbackEnded = true;
        mediaEngine->_state         = MEMediaState::Stopped;
        mediaEngine->handleEvent(MEMediaEventType::Stopped);
        break;
    case libvlc_MediaPlayerStopped:
        if (!mediaEngine->_playbackEnded)
        {
            mediaEngine->_state = MEMediaState::Stopped;
            mediaEngine->handleEvent(MEMediaEventType::Stopped);
        }
        break;
#    else
    case libvlc_MediaPlayerStopped:
        mediaEngine->_playbackEnded = true;
        mediaEngine->_state         = MEMediaState::Stopped;
        mediaEngine->handleEvent(MEMediaEventType::Stopped);
        break;
    case libvlc_MediaListPlayerStopped:
        if (!mediaEngine->_playbackEnded)
        {
            mediaEngine->_state = MEMediaState::Stopped;
            mediaEngine->handleEvent(MEMediaEventType::Stopped);
        }
        break;
#    endif
    case libvlc_MediaPlayerEncounteredError:
        mediaEngine->_state = MEMediaState::Error;
        mediaEngine->handleEvent(MEMediaEventType::Error);
        break;
    case libvlc_MediaPlayerMediaChanged:
        mediaEngine->_frameIndex = 0;
        break;
    default:;
    }
}

VlcMediaEngine::VlcMediaEngine()
{
    AXLOG("libvlc-version: %s", libvlc_get_version());

    // too late set vlc plugins path at hete, vlc maybe read it at DllMain
    //_putenv_s("VLC_PLUGIN_PATH", R"(D:\dev\axmol\3rdparty\vlc\win\lib)");
    _vlc = libvlc_new(0, nullptr);
    if (!_vlc)
    {
        AXLOGI("VlcMediaEngine: libvlc_new fail, ensure install vlc and ubuntu-restricted-extras");
        return;
    }

    // media player
    _mp = libvlc_media_player_new(_vlc);
    libvlc_video_set_callbacks(_mp, libvlc_video_lock, libvlc_video_unlock, nullptr, this);
    libvlc_video_set_format_callbacks(_mp, libvlc_video_format_setup, nullptr);

    // media list
#    if LIBVLC_VERSION_MAJOR < 4
    _ml = libvlc_media_list_new(_vlc);
#    else
    _ml = libvlc_media_list_new();
#    endif

    // media list player
    _mlp = libvlc_media_list_player_new(_vlc);
    libvlc_media_list_player_set_media_player(_mlp, _mp);
    libvlc_media_list_player_set_media_list(_mlp, _ml);

    // register media player events
    _mp_events = libvlc_media_player_event_manager(_mp);
    libvlc_event_attach(_mp_events, libvlc_MediaPlayerPlaying, libvlc_handle_event, this);
    libvlc_event_attach(_mp_events, libvlc_MediaPlayerPaused, libvlc_handle_event, this);
    libvlc_event_attach(_mp_events, libvlc_MediaPlayerStopped, libvlc_handle_event, this);
    libvlc_event_attach(_mp_events, libvlc_MediaPlayerMediaChanged, libvlc_handle_event, this);
#    if LIBVLC_VERSION_MAJOR < 4
    libvlc_event_attach(_mp_events, libvlc_MediaPlayerEndReached, libvlc_handle_event, this);
#    endif
    libvlc_event_attach(_mp_events, libvlc_MediaPlayerEncounteredError, libvlc_handle_event, this);

    // register media list player events
    _mpl_events = libvlc_media_list_player_event_manager(_mlp);
    libvlc_event_attach(_mpl_events, libvlc_MediaListEndReached, libvlc_handle_event, this);
    libvlc_event_attach(_mpl_events, libvlc_MediaListPlayerStopped, libvlc_handle_event, this);
}
VlcMediaEngine::~VlcMediaEngine()
{
    if (_mpl_events)
    {
        libvlc_event_detach(_mpl_events, libvlc_MediaListEndReached, libvlc_handle_event, this);
        libvlc_event_detach(_mpl_events, libvlc_MediaListPlayerStopped, libvlc_handle_event, this);
    }

    if (_mp_events)
    {
        libvlc_event_detach(_mp_events, libvlc_MediaPlayerPlaying, libvlc_handle_event, this);
        libvlc_event_detach(_mp_events, libvlc_MediaPlayerPaused, libvlc_handle_event, this);
        libvlc_event_detach(_mp_events, libvlc_MediaPlayerStopped, libvlc_handle_event, this);
        libvlc_event_detach(_mp_events, libvlc_MediaPlayerMediaChanged, libvlc_handle_event, this);
#    if LIBVLC_VERSION_MAJOR < 4
        libvlc_event_detach(_mp_events, libvlc_MediaPlayerEndReached, libvlc_handle_event, this);
#    endif
        libvlc_event_detach(_mp_events, libvlc_MediaPlayerEncounteredError, libvlc_handle_event, this);
    }

    if (_mlp)
    {
#    if LIBVLC_VERSION_MAJOR < 4
        libvlc_media_list_player_stop(_mlp);
#    else
        libvlc_media_list_player_stop_async(_mlp);
#    endif
        libvlc_media_list_player_release(_mlp);
    }

    if (_ml)
        libvlc_media_list_release(_ml);
    if (_mp)
        libvlc_media_player_release(_mp);
    if (_vlc)
        libvlc_release(_vlc);
}
void VlcMediaEngine::handleEvent(MEMediaEventType event)
{
    if (event == MEMediaEventType::Playing)
        updatePlaybackProperties();

    fireMediaEvent(event);
}

void VlcMediaEngine::setAutoPlay(bool bAutoPlay)
{
    _bAutoPlay = bAutoPlay;
}
bool VlcMediaEngine::open(std::string_view sourceUri)
{
    if (_state != MEMediaState::Closed)
        return false;

#    if LIBVLC_VERSION_MAJOR < 4
    libvlc_media_t* media = libvlc_media_new_location(_vlc, sourceUri.data());
#    else
    libvlc_media_t* media = libvlc_media_new_location(sourceUri.data());
#    endif
    if (!media)
        return false;

    _frameIndex = 0;
    _sourceUri  = sourceUri;

    libvlc_media_list_add_media(_ml, media);  // always one media
    libvlc_media_release(media);

    if (_bAutoPlay)
    {
        _state = MEMediaState::Preparing;
        libvlc_media_list_player_play(_mlp);
    }
    else
        _state = MEMediaState::Stopped;

    return true;
}

static void track_codec_to_mime_type(libvlc_media_track_t* track, std::string& out)

{
    out.clear();
    out.reserve(9);  // 4 * 2 + 1
    out.append(reinterpret_cast<const char*>(&track->i_codec), 4);
    out.push_back('/');
    out.append(reinterpret_cast<const char*>(&track->i_original_fourcc), 4);
}

bool VlcMediaEngine::updatePlaybackProperties()
{
    if (_frameIndex == 0)
    {
        auto media = libvlc_media_player_get_media(_mp);
#    if LIBVLC_VERSION_MAJOR < 4
        /* local file, we Get the size of the video with the tracks information of the media. */
        libvlc_media_track_t** tracks{};
        unsigned int nbTracks = libvlc_media_tracks_get(media, &tracks);
        if (!nbTracks)
            return false;

        for (unsigned int i = 0; i < nbTracks; ++i)
        {
            auto track = tracks[i];
            if (track->i_type == libvlc_track_video)
            {
                track_codec_to_mime_type(track, _videoCodecMimeType);
                AXLOGD("VlcMediaEngine: sourceUri: {}, codec: {}", _sourceUri, _videoCodecMimeType);

                auto vdi = track->video;
                auto vdw = vdi->i_width;
                auto vdh = vdi->i_height;
                switch (vdi->i_orientation)
                {
                case libvlc_video_orient_left_bottom: /**< Rotated 90 degrees clockwise (or 270 anti-clockwise) */
                case libvlc_video_orient_right_top:   /**< Rotated 90 degrees anti-clockwise */
                    std::swap(vdw, vdh);
                default:;
                }
                _videoDim.set(vdw, vdh);
                break;
            }
        }
        libvlc_media_tracks_release(tracks, nbTracks);
#    else
        auto tracks = libvlc_media_get_tracklist(media, libvlc_track_type_t::libvlc_track_video);
        if (tracks)
        {
            auto nTrack = libvlc_media_tracklist_count(tracks);
            if (nTrack > 0)
            {
                auto track = libvlc_media_tracklist_at(tracks, 0);
                track_codec_to_mime_type(track, _videoCodecMimeType);
                AXLOGI("VlcMediaEngine: sourceUri: {}, codec: {}", _sourceUri, _videoCodecMimeType);

                auto vdi = track->video;
                _videoDim.set(vdi->i_width, vdi->i_height);
            }
            libvlc_media_tracklist_delete(tracks);
        }
#    endif

        // set playback mode
        libvlc_media_list_player_set_playback_mode(
            _mlp, _looping ? libvlc_playback_mode_repeat : libvlc_playback_mode_default);
    }
    return true;
}
bool VlcMediaEngine::close()
{
    if (libvlc_media_list_count(_ml) > 0)
    {
#    if LIBVLC_VERSION_MAJOR < 4
        libvlc_media_list_player_stop(_mlp);
#    else
        libvlc_media_list_player_stop_async(_mlp);
#    endif
        libvlc_media_list_remove_index(_ml, 0);
    }
    _state = MEMediaState::Closed;
    return true;
}
bool VlcMediaEngine::setLoop(bool bLooping)
{
    _looping = bLooping;
    if (getState() == MEMediaState::Playing)
        libvlc_media_list_player_set_playback_mode(
            _mlp, _looping ? libvlc_playback_mode_repeat : libvlc_playback_mode_default);
    return true;
}
bool VlcMediaEngine::setRate(double fRate)
{
    return _mp && libvlc_media_player_set_rate(_mp, static_cast<float>(fRate)) == 0;
}
bool VlcMediaEngine::setCurrentTime(double fSeekTimeInSec)
{
    if (_mp)
#    if LIBVLC_VERSION_MAJOR < 4
        libvlc_media_player_set_time(_mp, static_cast<libvlc_time_t>(fSeekTimeInSec * 1000));
#    else
        libvlc_media_player_set_time(_mp, static_cast<libvlc_time_t>(fSeekTimeInSec * 1000), true);
#    endif
    return true;
}
bool VlcMediaEngine::play()
{
    if (_mlp && _state != MEMediaState::Closed)
        libvlc_media_list_player_play(_mlp);
    return true;
}
bool VlcMediaEngine::pause()
{
    if (_mlp && _state != MEMediaState::Closed)
        libvlc_media_list_player_pause(_mlp);
    return true;
}
bool VlcMediaEngine::stop()
{
    if (_mlp && _state != MEMediaState::Closed)
#    if LIBVLC_VERSION_MAJOR < 4
        libvlc_media_list_player_stop(_mlp);
#    else
        libvlc_media_list_player_stop_async(_mlp);
#    endif
    return true;
}
MEMediaState VlcMediaEngine::getState() const
{
    return _state;
}

bool VlcMediaEngine::transferVideoFrame()
{
    if (_frameBuffer1.empty())
        return false;

    std::unique_lock<std::mutex> lck(_frameBuffer1Mtx);
    if (UTILS_LIKELY(!_frameBuffer1.empty()))
    {
        _frameBuffer2.swap(_frameBuffer1);
        lck.unlock();  // unlock immidiately before invoke user callback (maybe upload buffer to GPU)

        auto& buffer = _frameBuffer2;

        ax::MEVideoFrame frame{buffer.data(), buffer.data() + _videoDim.x * _videoDim.y, buffer.size(),
                               ax::MEVideoPixelDesc{VLC_OUTPUT_FORMAT, _videoDim}, _videoDim};
        // assert(static_cast<int>(frame._dataLen) >= frame._vpd._dim.x * frame._vpd._dim.y * 3 / 2);
        _onVideoFrame(frame);
        _frameBuffer2.clear();
        return true;
    }

    return false;
}

NS_AX_END

#endif