Lothario
/
axmol
mirror of https://github.com/axmolengine/axmol.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
axmol/core/media/AvfMediaEngine.mm

507 lines
16 KiB
Plaintext
Raw Blame History

#include "AvfMediaEngine.h"
#if defined(__APPLE__)
# include <TargetConditionals.h>
# include <assert.h>
# include "yasio/detail/endian_portable.hpp"
# include "yasio/stl/string_view.hpp"
# include "yasio/detail/sz.hpp"
#if TARGET_OS_IPHONE
# import <UIKit/UIKit.h>
#endif
USING_NS_AX;
@interface AVMediaSessionHandler : NSObject
- (AVMediaSessionHandler*)initWithMediaEngine:(AvfMediaEngine*)me;
- (void)dealloc;
- (void)playerItemDidPlayToEndTime:(NSNotification*)notification;
@property AvfMediaEngine* _me;
@end
@implementation AVMediaSessionHandler
@synthesize _me;
- (AVMediaSessionHandler*)initWithMediaEngine:(AvfMediaEngine*)me
{
self = [super init];
if (self)
_me = me;
return self;
}
- registerUINotifications
{
#if TARGET_OS_IPHONE
auto nc = [NSNotificationCenter defaultCenter];
[nc addObserver:self
selector:@selector(handleAudioRouteChange:)
name:AVAudioSessionRouteChangeNotification
object:[AVAudioSession sharedInstance]];
[nc addObserver:self
selector:@selector(handleActive:)
name:UIApplicationDidBecomeActiveNotification
object:nil];
[nc addObserver:self
selector:@selector(handleDeactive:)
name:UIApplicationWillResignActiveNotification
object:nil];
[nc addObserver:self
selector:@selector(handleEnterBackround:)
name:UIApplicationDidEnterBackgroundNotification
object:nil];
[nc addObserver:self
selector:@selector(handleEnterForground:)
name:UIApplicationWillEnterForegroundNotification
object:nil];
#endif
}
#if TARGET_OS_IPHONE
- (void)handleAudioRouteChange:(NSNotification*)notification
{
if (_me->isPlaying())
_me->internalPlay(true);
}
- (void)handleActive:(NSNotification*)notification
{
if (_me->isPlaying())
_me->internalPlay();
}
- (void)handleDeactive:(NSNotification*)notification
{
if (_me->isPlaying())
_me->internalPause();
}
- (void)handleEnterForground:(NSNotification*)notification
{
if (_me->isPlaying())
_me->internalPlay();
}
- (void)handleEnterBackround:(NSNotification*)notification
{
if (_me->isPlaying())
_me->internalPause();
}
#endif
- deregisterUINotifications
{
#if TARGET_OS_IPHONE
auto nc = [NSNotificationCenter defaultCenter];
[nc removeObserver:self
name:AVAudioSessionRouteChangeNotification
object:nil];
[nc removeObserver:self
name:UIApplicationDidBecomeActiveNotification
object:nil];
[nc removeObserver:self
name:UIApplicationWillResignActiveNotification
object:nil];
[nc removeObserver:self
name:UIApplicationDidEnterBackgroundNotification
object:nil];
[nc removeObserver:self
name:UIApplicationWillEnterForegroundNotification
object:nil];
#endif
}
- (void)dealloc
{
[super dealloc];
}
- (void)playerItemDidPlayToEndTime:(NSNotification*)notification
{
_me->onPlayerEnd();
}
- (void)observeValueForKeyPath:(NSString*)keyPath
ofObject:(id)object
change:(NSDictionary<NSKeyValueChangeKey, id>*)change
context:(void*)context
{
if ((id)context == object && [keyPath isEqualToString:@"status"])
_me->onStatusNotification(context);
}
@end
NS_AX_BEGIN
void AvfMediaEngine::onPlayerEnd()
{
_state = MEMediaState::Completed;
FireEvent(MEMediaEventType::Completed);
if (_repeatEnabled) {
this->SetCurrentTime(0);
this->Play();
}
}
void AvfMediaEngine::SetMediaEventCallback(MEMediaEventCallback cb)
{
_eventCallback = cb;
}
void AvfMediaEngine::SetAutoPlay(bool bAutoPlay)
{
_bAutoPlay = bAutoPlay;
}
bool AvfMediaEngine::Open(std::string_view sourceUri)
{
Close();
NSURL* nsMediaUrl = nil;
std::string_view Path;
if (cxx20::starts_with(sourceUri, "file://"sv))
{
// Media Framework doesn't percent encode the URL, so the path portion is just a native file path.
// Extract it and then use it create a proper URL.
Path = sourceUri.substr(7);
NSString* nsPath = [NSString stringWithUTF8String:Path.data()];
nsMediaUrl = [NSURL fileURLWithPath:nsPath isDirectory:NO];
}
else
{
// Assume that this has been percent encoded for now - when we support HTTP Live Streaming we will need to check
// for that.
NSString* nsUri = [NSString stringWithUTF8String:sourceUri.data()];
nsMediaUrl = [NSURL URLWithString:nsUri];
}
// open media file
if (nsMediaUrl == nil)
{
AXME_TRACE("Failed to open Media file: %s", sourceUri.data());
return false;
}
// create player instance
_player = [[AVPlayer alloc] init];
if (!_player)
{
AXME_TRACE("Failed to create instance of an AVPlayer: %s", sourceUri.data());
return false;
}
_player.actionAtItemEnd = AVPlayerActionAtItemEndPause;
// create player item
_sessionHandler = [[AVMediaSessionHandler alloc] initWithMediaEngine:this];
assert(_sessionHandler != nil);
// Use URL asset which gives us resource loading ability if system can't handle the scheme
AVURLAsset* urlAsset = [[AVURLAsset alloc] initWithURL:nsMediaUrl options:nil];
_playerItem = [[AVPlayerItem playerItemWithAsset:urlAsset] retain];
[urlAsset release];
if (_playerItem == nil)
{
AXME_TRACE("Failed to open player item with Url: %s", sourceUri.data());
return false;
}
_state = MEMediaState::Preparing;
// load tracks
[[_playerItem asset] loadValuesAsynchronouslyForKeys:@[ @"tracks" ]
completionHandler:^{
NSError* nsError = nil;
if ([[_playerItem asset] statusOfValueForKey:@"tracks" error:&nsError] ==
AVKeyValueStatusLoaded)
{
// File movies will be ready now
if (_playerItem.status == AVPlayerItemStatusReadyToPlay)
{
onStatusNotification(_playerItem);
}
}
else if (nsError != nullptr)
{
NSDictionary* errDetail = [nsError userInfo];
NSString* errStr =
[[errDetail objectForKey:NSUnderlyingErrorKey] localizedDescription];
AXME_TRACE("Load media asset failed, %s", errStr.UTF8String);
}
}];
[[NSNotificationCenter defaultCenter] addObserver:_sessionHandler
selector:@selector(playerItemDidPlayToEndTime:)
name:AVPlayerItemDidPlayToEndTimeNotification
object:_playerItem];
[_playerItem addObserver:_sessionHandler forKeyPath:@"status" options:0 context:_playerItem];
_player.rate = 0.0;
[_player replaceCurrentItemWithPlayerItem:_playerItem];
// TODO: handle EnterForground, EnterBackground, Active, Deactive, AudioRouteChanged
[_sessionHandler registerUINotifications];
return true;
}
void AvfMediaEngine::onStatusNotification(void* context)
{
if (!_playerItem || context != _playerItem)
return;
if (_playerItem.status == AVPlayerItemStatusFailed)
{
FireEvent(MEMediaEventType::Error);
return;
}
if (_playerItem.status != AVPlayerItemStatusReadyToPlay)
return;
for (AVPlayerItemTrack* playerTrack in _playerItem.tracks)
{
AVAssetTrack* assetTrack = playerTrack.assetTrack;
NSString* mediaType = assetTrack.mediaType;
if ([mediaType isEqualToString:AVMediaTypeVideo])
{ // we only care about video
auto naturalSize = [assetTrack naturalSize];
_videoExtent.x = naturalSize.width;
_videoExtent.y = naturalSize.height;
NSMutableDictionary* outputAttrs = [NSMutableDictionary dictionary];
CMFormatDescriptionRef DescRef = (CMFormatDescriptionRef)[assetTrack.formatDescriptions objectAtIndex:0];
CMVideoCodecType codecType = CMFormatDescriptionGetMediaSubType(DescRef);
int videoOutputPF = kCVPixelFormatType_32BGRA;
if (kCMVideoCodecType_H264 == codecType || kCMVideoCodecType_HEVC == codecType)
{
videoOutputPF = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
CFDictionaryRef formatExtensions = CMFormatDescriptionGetExtensions(DescRef);
if (formatExtensions)
{
CFBooleanRef bFullRange = (CFBooleanRef)CFDictionaryGetValue(
formatExtensions, kCMFormatDescriptionExtension_FullRangeVideo);
if (bFullRange && (bool)CFBooleanGetValue(bFullRange))
{
videoOutputPF = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange;
}
}
}
_bFullColorRange = false;
switch (videoOutputPF)
{
case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
_bFullColorRange = true;
case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
_videoPF = MEVideoPixelFormat::NV12;
break;
default: // kCVPixelFormatType_32BGRA
_videoPF = MEVideoPixelFormat::BGR32;
}
[outputAttrs setObject:[NSNumber numberWithInt:videoOutputPF]
forKey:(NSString*)kCVPixelBufferPixelFormatTypeKey];
[outputAttrs setObject:[NSNumber numberWithInteger:1]
forKey:(NSString*)kCVPixelBufferBytesPerRowAlignmentKey];
[outputAttrs setObject:[NSNumber numberWithBool:YES] forKey:(NSString*)kCVPixelBufferMetalCompatibilityKey];
AVPlayerItemVideoOutput* videoOutput =
[[AVPlayerItemVideoOutput alloc] initWithPixelBufferAttributes:outputAttrs];
// Only decode for us
videoOutput.suppressesPlayerRendering = YES;
[_playerItem addOutput:videoOutput];
_playerOutput = videoOutput;
break;
}
}
if (_bAutoPlay)
this->Play();
}
bool AvfMediaEngine::TransferVideoFrame(std::function<void(const MEVideoFrame&)> callback)
{
auto videoOutput = static_cast<AVPlayerItemVideoOutput*>(this->_playerOutput);
if (!videoOutput)
return false;
CMTime currentTime = [videoOutput itemTimeForHostTime:CACurrentMediaTime()];
if (![videoOutput hasNewPixelBufferForItemTime:currentTime])
return false;
CVPixelBufferRef videoFrame = [videoOutput copyPixelBufferForItemTime:currentTime itemTimeForDisplay:nullptr];
if (!videoFrame)
return false;
auto& videoDim = _videoExtent;
MEIntPoint bufferDim;
CVPixelBufferLockBaseAddress(videoFrame, kCVPixelBufferLock_ReadOnly);
if (CVPixelBufferIsPlanar(videoFrame))
{ // NV12('420v' or '420f' expected
assert(CVPixelBufferGetPlaneCount(videoFrame) == 2);
auto YWidth = static_cast<int>(CVPixelBufferGetWidthOfPlane(videoFrame, 0)); // 1920
auto YHeight = static_cast<int>(CVPixelBufferGetHeightOfPlane(videoFrame, 0)); // 1080
auto UVWidth = static_cast<int>(CVPixelBufferGetWidthOfPlane(videoFrame, 1)); // 960
auto UVHeight = static_cast<int>(CVPixelBufferGetHeightOfPlane(videoFrame, 1)); // 540
auto YPitch = static_cast<int>(CVPixelBufferGetBytesPerRowOfPlane(videoFrame, 0));
auto UVPitch = static_cast<int>(CVPixelBufferGetBytesPerRowOfPlane(videoFrame, 1));
auto YDataLen = YPitch * YHeight; // 1920x1080: YDataLen=2073600
auto UVDataLen = UVPitch * UVHeight; // 1920x1080: UVDataLen=1036800
auto frameYData = (uint8_t*)CVPixelBufferGetBaseAddressOfPlane(videoFrame, 0);
auto frameCbCrData = (uint8_t*)CVPixelBufferGetBaseAddressOfPlane(videoFrame, 1);
assert(YASIO_SZ_ALIGN(videoDim.x, 32) * videoDim.y * 3 / 2 == YDataLen + UVDataLen);
// Apple: both H264, HEVC(H265) bufferDimX=ALIGN(videoDim.x, 32), bufferDimY=videoDim.y
// Windows:
// - H264: BufferDimX align videoDim.x with 16, BufferDimY as-is
// - HEVC(H265): BufferDim(X,Y) align videoDim(X,Y) with 32
MEVideoFrame frame{frameYData, frameCbCrData, static_cast<size_t>(YDataLen + UVDataLen), MEVideoPixelDesc{_videoPF, MEIntPoint{YPitch, YHeight}}, videoDim};
#if defined(_DEBUG) || !defined(_NDEBUG)
auto& ycbcrDesc = frame._ycbcrDesc;
ycbcrDesc.YDim.x = YWidth;
ycbcrDesc.YDim.y = YHeight;
ycbcrDesc.CbCrDim.x = UVWidth;
ycbcrDesc.CbCrDim.y = UVHeight;
ycbcrDesc.YPitch = YPitch;
ycbcrDesc.CbCrPitch = UVPitch;
#endif
callback(frame);
}
else
{ // BGRA
auto frameData = (uint8_t*)CVPixelBufferGetBaseAddress(videoFrame);
size_t frameDataSize = CVPixelBufferGetDataSize(videoFrame);
callback(MEVideoFrame{frameData, nullptr, frameDataSize, MEVideoPixelDesc{_videoPF, videoDim}, videoDim});
}
CVPixelBufferUnlockBaseAddress(videoFrame, kCVPixelBufferLock_ReadOnly);
CVPixelBufferRelease(videoFrame);
}
bool AvfMediaEngine::Close()
{
if (_state == MEMediaState::Closed)
return true;
if (_playerItem != nil)
{
if (_player != nil)
{
[_sessionHandler deregisterUINotifications];
[[NSNotificationCenter defaultCenter] removeObserver:_sessionHandler
name:AVPlayerItemDidPlayToEndTimeNotification
object:_playerItem];
[_playerItem removeObserver:_sessionHandler forKeyPath:@"status"];
}
[_playerItem release];
_playerItem = nil;
}
if (_player != nil)
{
[_player release];
_player = nil;
}
_state = MEMediaState::Closed;
return true;
}
bool AvfMediaEngine::SetLoop(bool bLooping)
{
_repeatEnabled = bLooping;
if (bLooping)
_player.actionAtItemEnd = AVPlayerActionAtItemEndNone;
else
_player.actionAtItemEnd = AVPlayerActionAtItemEndPause;
return true;
}
bool AvfMediaEngine::SetRate(double fRate)
{
if (_player)
{
[_player setRate:fRate];
// TODO:
_player.muted = fRate < 0 ? YES : NO;
}
return true;
}
bool AvfMediaEngine::SetCurrentTime(double fSeekTimeInSec)
{
if (_player != nil)
[_player seekToTime:CMTimeMake(fSeekTimeInSec, 1)];
return true;
}
bool AvfMediaEngine::Play()
{
if (_state != MEMediaState::Playing)
{
[_player play];
_state = MEMediaState::Playing;
FireEvent(MEMediaEventType::Playing);
}
return true;
}
void AvfMediaEngine::internalPlay(bool replay)
{
if (_player != nil) {
if (replay)
[_player pause];
[_player play];
}
}
void AvfMediaEngine::internalPause()
{
if (_player != nil)
[_player pause];
}
bool AvfMediaEngine::Pause()
{
if (_state == MEMediaState::Playing)
{
[_player pause];
_state = MEMediaState::Paused;
FireEvent(MEMediaEventType::Paused);
}
return true;
}
bool AvfMediaEngine::Stop()
{
if (_state != MEMediaState::Stopped)
{
SetCurrentTime(0);
[_player pause];
_state = MEMediaState::Stopped;
// stop() will be invoked in dealloc, which is invoked by _videoPlayer's destructor,
// so do't send the message when _videoPlayer is being deleted.
}
return true;
}
MEMediaState AvfMediaEngine::GetState() const
{
return _state;
}
NS_AX_END
#endif
Reference in New Issue View Git Blame Copy Permalink