railbird/react-native-video
9
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

PR fixes for iOS sideloaded captions

Browse Source
This commit is contained in:
Ash Mishra 2018-07-09 11:20:32 -07:00
parent fa63a9ef87
commit c9b752454a
1 changed files with 92 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

170
ios/RCTVideo.m
View File

@ -20,6 +20,7 @@ static NSString *const timedMetadata = @"timedMetadata";
BOOL _playerItemObserversSet; BOOL _playerItemObserversSet;
BOOL _playerBufferEmpty; BOOL _playerBufferEmpty;
AVPlayerLayer *_playerLayer; AVPlayerLayer *_playerLayer;
BOOL _playerLayerObserverSet;
AVPlayerViewController *_playerViewController; AVPlayerViewController *_playerViewController;
NSURL *_videoURL; NSURL *_videoURL;
@ -46,6 +47,7 @@ static NSString *const timedMetadata = @"timedMetadata";
BOOL _allowsExternalPlayback; BOOL _allowsExternalPlayback;
NSArray * _textTracks; NSArray * _textTracks;
NSDictionary * _selectedTextTrack; NSDictionary * _selectedTextTrack;
NSDictionary * _selectedText;
BOOL _playbackStalled; BOOL _playbackStalled;
BOOL _playInBackground; BOOL _playInBackground;
BOOL _playWhenInactive; BOOL _playWhenInactive;
@ -350,7 +352,7 @@ static NSString *const timedMetadata = @"timedMetadata";
bool isNetwork = [RCTConvert BOOL:[source objectForKey:@"isNetwork"]]; bool isNetwork = [RCTConvert BOOL:[source objectForKey:@"isNetwork"]];
bool isAsset = [RCTConvert BOOL:[source objectForKey:@"isAsset"]]; bool isAsset = [RCTConvert BOOL:[source objectForKey:@"isAsset"]];
NSString *uri = [source objectForKey:@"uri"]; NSString *uri = [source objectForKey:@"uri"];
NSString *subtitleUri = self.selectedTextTrack[@"uri"]; NSString *subtitleUri = _selectedText[@"uri"];
NSString *type = [source objectForKey:@"type"]; NSString *type = [source objectForKey:@"type"];
NSDictionary *headers = [source objectForKey:@"requestHeaders"]; NSDictionary *headers = [source objectForKey:@"requestHeaders"];
@ -376,19 +378,19 @@ static NSString *const timedMetadata = @"timedMetadata";
asset = [AVURLAsset URLAssetWithURL:[self urlFilePath:uri] options:nil]; asset = [AVURLAsset URLAssetWithURL:[self urlFilePath:uri] options:nil];
subAsset = [AVURLAsset URLAssetWithURL:[self urlFilePath:subtitleUri] options:nil]; subAsset = [AVURLAsset URLAssetWithURL:[self urlFilePath:subtitleUri] options:nil];
} }
else if (!isNetwork && !isAsset) // this is a relative file hardcoded in the app? else // file passed in through JS, or an asset in the Xcode project
{ {
asset = [AVURLAsset URLAssetWithURL:[[NSURL alloc] initFileURLWithPath:[[NSBundle mainBundle] pathForResource:uri ofType:type]] options:nil]; asset = [AVURLAsset URLAssetWithURL:[[NSURL alloc] initFileURLWithPath:[[NSBundle mainBundle] pathForResource:uri ofType:type]] options:nil];
} }
if (!_textTracks || !self.selectedTextTrack) return [AVPlayerItem playerItemWithAsset:asset]; if (!_textTracks || !_selectedText) return [AVPlayerItem playerItemWithAsset:asset];
// otherwise sideload text tracks // otherwise sideload text tracks
AVAssetTrack *videoAsset = [asset tracksWithMediaType:AVMediaTypeVideo].firstObject; AVAssetTrack *videoAsset = [asset tracksWithMediaType:AVMediaTypeVideo].firstObject;
AVAssetTrack *audioAsset = [asset tracksWithMediaType:AVMediaTypeAudio].firstObject; AVAssetTrack *audioAsset = [asset tracksWithMediaType:AVMediaTypeAudio].firstObject;
AVAssetTrack *subtitleAsset = [subAsset tracksWithMediaType:AVMediaTypeText].firstObject; AVAssetTrack *textAsset = [subAsset tracksWithMediaType:AVMediaTypeText].firstObject;
NSLog(@"assets: %@,%@,%@", videoAsset, audioAsset, subtitleAsset); NSLog(@"assets: %@,%@,%@", videoAsset, audioAsset, textAsset);
AVMutableComposition *mixComposition = [[AVMutableComposition alloc] init]; AVMutableComposition *mixComposition = [[AVMutableComposition alloc] init];
@ -407,7 +409,7 @@ static NSString *const timedMetadata = @"timedMetadata";
error:nil]; error:nil];
[subtitleCompTrack insertTimeRange:CMTimeRangeMake(kCMTimeZero, videoAsset.timeRange.duration) [subtitleCompTrack insertTimeRange:CMTimeRangeMake(kCMTimeZero, videoAsset.timeRange.duration)
ofTrack:subtitleAsset ofTrack:textAsset
atTime:kCMTimeZero atTime:kCMTimeZero
error:nil]; error:nil];
@ -454,7 +456,6 @@ static NSString *const timedMetadata = @"timedMetadata";
NSString *orientation = @"undefined"; NSString *orientation = @"undefined";
if ([_playerItem.asset tracksWithMediaType:AVMediaTypeVideo].count > 0) { if ([_playerItem.asset tracksWithMediaType:AVMediaTypeVideo].count > 0) {
AVAssetTrack *videoTrack = [[_playerItem.asset tracksWithMediaType:AVMediaTypeVideo] objectAtIndex:0]; AVAssetTrack *videoTrack = [[_playerItem.asset tracksWithMediaType:AVMediaTypeVideo] objectAtIndex:0];
width = [NSNumber numberWithFloat:videoTrack.naturalSize.width]; width = [NSNumber numberWithFloat:videoTrack.naturalSize.width];
height = [NSNumber numberWithFloat:videoTrack.naturalSize.height]; height = [NSNumber numberWithFloat:videoTrack.naturalSize.height];
@ -463,7 +464,6 @@ static NSString *const timedMetadata = @"timedMetadata";
if ((videoTrack.naturalSize.width == preferredTransform.tx if ((videoTrack.naturalSize.width == preferredTransform.tx
&& videoTrack.naturalSize.height == preferredTransform.ty) && videoTrack.naturalSize.height == preferredTransform.ty)
|| (preferredTransform.tx == 0 && preferredTransform.ty == 0)) || (preferredTransform.tx == 0 && preferredTransform.ty == 0))
{ {
orientation = @"landscape"; orientation = @"landscape";
} else { } else {
@ -638,23 +638,30 @@ static NSString *const timedMetadata = @"timedMetadata";
- (void)setCurrentTime:(float)currentTime - (void)setCurrentTime:(float)currentTime
{ {
[self setSeek: currentTime]; NSDictionary *info = @{
@"time": [NSNumber numberWithFloat:currentTime],
@"tolerance": [NSNumber numberWithInt:100]
};
[self setSeek:info];
} }
- (void)setSeek:(float)seekTime - (void)setSeek:(NSDictionary *)info
{ {
int timeScale = 10000; NSNumber *seekTime = info[@"time"];
NSNumber *seekTolerance = info[@"tolerance"];
int timeScale = 1000;
AVPlayerItem *item = _player.currentItem; AVPlayerItem *item = _player.currentItem;
if (item && item.status == AVPlayerItemStatusReadyToPlay) { if (item && item.status == AVPlayerItemStatusReadyToPlay) {
// TODO check loadedTimeRanges // TODO check loadedTimeRanges
CMTime cmSeekTime = CMTimeMakeWithSeconds(seekTime, timeScale); CMTime cmSeekTime = CMTimeMakeWithSeconds([seekTime floatValue], timeScale);
CMTime current = item.currentTime; CMTime current = item.currentTime;
// TODO figure out a good tolerance level // TODO figure out a good tolerance level
CMTime tolerance = CMTimeMake(1000, timeScale); CMTime tolerance = CMTimeMake([seekTolerance floatValue], timeScale);
BOOL wasPaused = _paused; BOOL wasPaused = _paused;
if (CMTimeCompare(current, cmSeekTime) != 0) { if (CMTimeCompare(current, cmSeekTime) != 0) {
if (!wasPaused) [_player pause]; if (!wasPaused) [_player pause];
[_player seekToTime:cmSeekTime toleranceBefore:tolerance toleranceAfter:tolerance completionHandler:^(BOOL finished) { [_player seekToTime:cmSeekTime toleranceBefore:tolerance toleranceAfter:tolerance completionHandler:^(BOOL finished) {
@ -662,22 +669,22 @@ static NSString *const timedMetadata = @"timedMetadata";
[self addPlayerTimeObserver]; [self addPlayerTimeObserver];
} }
if (!wasPaused) { if (!wasPaused) {
[self setPaused:false]; [self setPaused:false];
} }
if(self.onVideoSeek) { if(self.onVideoSeek) {
self.onVideoSeek(@{@"currentTime": [NSNumber numberWithFloat:CMTimeGetSeconds(item.currentTime)], self.onVideoSeek(@{@"currentTime": [NSNumber numberWithFloat:CMTimeGetSeconds(item.currentTime)],
@"seekTime": [NSNumber numberWithFloat:seekTime], @"seekTime": seekTime,
@"target": self.reactTag}); @"target": self.reactTag});
} }
}]; }];
_pendingSeek = false; _pendingSeek = false;
} }
} else { } else {
// TODO: See if this makes sense and if so, actually implement it // TODO: See if this makes sense and if so, actually implement it
_pendingSeek = true; _pendingSeek = true;
_pendingSeekTime = seekTime; _pendingSeekTime = [seekTime floatValue];
} }
} }
@ -721,32 +728,20 @@ static NSString *const timedMetadata = @"timedMetadata";
_repeat = repeat; _repeat = repeat;
} }
- (NSDictionary*) selectedTextTrack {
if (_textTracks) {
[self setSideloadedTextTrack];
}
else [self setStreamingTextTrack];
return _selectedTextTrack;
}
- (void)setSelectedTextTrack:(NSDictionary *)selectedTextTrack { - (void)setSelectedTextTrack:(NSDictionary *)selectedTextTrack {
_selectedTextTrack = selectedTextTrack;
_selectedTextTrack = selectedTextTrack; if (_textTracks) {
[self setSideloadedText];
}
else [self setStreamingText];
} }
- (void)setSideloadedTextTrack { - (void) setSideloadedText {
NSString *type = _selectedTextTrack[@"type"]; NSString *type = _selectedTextTrack[@"type"];
NSArray* textTracks = [self getTextTrackInfo]; NSArray* textTracks = [self getTextTrackInfo];
if (textTracks==nil) { _selectedText = nil;
_selectedTextTrack = nil;
return;
}
NSDictionary* selectedCaption = nil;
if ([type isEqualToString:@"disabled"]) { if ([type isEqualToString:@"disabled"]) {
// Do nothing. We want to ensure option is nil // Do nothing. We want to ensure option is nil
@ -754,20 +749,20 @@ static NSString *const timedMetadata = @"timedMetadata";
NSString *selectedValue = _selectedTextTrack[@"value"]; NSString *selectedValue = _selectedTextTrack[@"value"];
for (int i = 0; i < textTracks.count; ++i) { for (int i = 0; i < textTracks.count; ++i) {
NSDictionary *currentCaption = [textTracks objectAtIndex:i]; NSDictionary *currentTextTrack = [textTracks objectAtIndex:i];
if ([selectedValue isEqualToString:currentCaption[@"language"]]) { if ([selectedValue isEqualToString:currentTextTrack[@"language"]]) {
selectedCaption = currentCaption; _selectedText = currentTextTrack;
break; break;
} }
} }
} else if ([type isEqualToString:@"title"]) { } else if ([type isEqualToString:@"title"]) {
NSString *selectedValue = _selectedTextTrack[@"value"]; NSString *selectedValue = _selectedTextTrack[@"value"];
for (int i = 0; i < textTracks.count; ++i) { for (int i = 0; i < textTracks.count; ++i) {
NSDictionary *currentCaption = [textTracks objectAtIndex:i]; NSDictionary *currentTextTrack = [textTracks objectAtIndex:i];
if ([selectedValue isEqualToString:currentCaption[@"title"]]) { if ([selectedValue isEqualToString:currentTextTrack[@"title"]]) {
selectedCaption = currentCaption; _selectedText = currentTextTrack;
break; break;
} }
} }
@ -775,28 +770,37 @@ static NSString *const timedMetadata = @"timedMetadata";
if ([_selectedTextTrack[@"value"] isKindOfClass:[NSNumber class]]) { if ([_selectedTextTrack[@"value"] isKindOfClass:[NSNumber class]]) {
int index = [_selectedTextTrack[@"value"] intValue]; int index = [_selectedTextTrack[@"value"] intValue];
if (textTracks.count > index) { if (textTracks.count > index) {
selectedCaption = [textTracks objectAtIndex:index]; _selectedText = [textTracks objectAtIndex:index];
} }
} }
} }
// user's selected language might not be available, or system defaults have captions enabled // user's selected language might not be available, or system defaults have captions enabled
if (selectedCaption==nil || [type isEqualToString:@"default"]) { if (_selectedText==nil || [type isEqualToString:@"default"]) {
CFArrayRef captioningMediaCharacteristics = MACaptionAppearanceCopyPreferredCaptioningMediaCharacteristics(kMACaptionAppearanceDomainUser); CFArrayRef captioningMediaCharacteristics = MACaptionAppearanceCopyPreferredCaptioningMediaCharacteristics(kMACaptionAppearanceDomainUser);
NSArray *captionSettings = (__bridge NSArray*)captioningMediaCharacteristics; NSArray *captionSettings = (__bridge NSArray*)captioningMediaCharacteristics;
if ([captionSettings containsObject: AVMediaCharacteristicTranscribesSpokenDialogForAccessibility]) { if ([captionSettings containsObject: AVMediaCharacteristicTranscribesSpokenDialogForAccessibility]) {
selectedCaption = textTracks.firstObject; // iterate through the textTracks to find a matching option, or default to the first object.
_selectedText = textTracks.firstObject;
NSString * systemLanguage = [[NSLocale preferredLanguages] firstObject];
for (int i = 0; i < textTracks.count; ++i) {
NSDictionary *currentTextTrack = [textTracks objectAtIndex:i];
if ([systemLanguage isEqualToString:currentTextTrack[@"language"]]) {
_selectedText = currentTextTrack;
break;
}
}
} }
} }
_selectedTextTrack = selectedCaption;
} }
-(void) setStreamingTextTrack { -(void) setStreamingText {
NSString *type = _selectedTextTrack[@"type"]; NSString *type = _selectedTextTrack[@"type"];
AVMediaSelectionGroup *group = [_player.currentItem.asset AVMediaSelectionGroup *group = [_player.currentItem.asset
mediaSelectionGroupForMediaCharacteristic:AVMediaCharacteristicLegible]; mediaSelectionGroupForMediaCharacteristic:AVMediaCharacteristicLegible];
AVMediaSelectionOption *option; AVMediaSelectionOption *mediaOption;
if ([type isEqualToString:@"disabled"]) { if ([type isEqualToString:@"disabled"]) {
// Do nothing. We want to ensure option is nil // Do nothing. We want to ensure option is nil
@ -813,7 +817,7 @@ static NSString *const timedMetadata = @"timedMetadata";
objectAtIndex:0]; objectAtIndex:0];
} }
if ([value isEqualToString:optionValue]) { if ([value isEqualToString:optionValue]) {
option = currentOption; mediaOption = currentOption;
break; break;
} }
} }
@ -823,7 +827,7 @@ static NSString *const timedMetadata = @"timedMetadata";
if ([_selectedTextTrack[@"value"] isKindOfClass:[NSNumber class]]) { if ([_selectedTextTrack[@"value"] isKindOfClass:[NSNumber class]]) {
int index = [_selectedTextTrack[@"value"] intValue]; int index = [_selectedTextTrack[@"value"] intValue];
if (group.options.count > index) { if (group.options.count > index) {
option = [group.options objectAtIndex:index]; mediaOption = [group.options objectAtIndex:index];
} }
} }
} else { // default. invalid type or "system" } else { // default. invalid type or "system"
@ -832,12 +836,15 @@ static NSString *const timedMetadata = @"timedMetadata";
} }
// If a match isn't found, option will be nil and text tracks will be disabled // If a match isn't found, option will be nil and text tracks will be disabled
[_player.currentItem selectMediaOption:option inMediaSelectionGroup:group]; [_player.currentItem selectMediaOption:mediaOption inMediaSelectionGroup:group];
} }
- (void)setTextTracks:(NSArray*) textTracks; - (void)setTextTracks:(NSArray*) textTracks;
{ {
_textTracks = textTracks; _textTracks = textTracks;
// in case textTracks was set after selectedTextTrack
if (_selectedTextTrack) [self setSelectedTextTrack:_selectedTextTrack];
} }
- (NSArray *)getTextTrackInfo - (NSArray *)getTextTrackInfo
@ -852,13 +859,16 @@ static NSString *const timedMetadata = @"timedMetadata";
mediaSelectionGroupForMediaCharacteristic:AVMediaCharacteristicLegible]; mediaSelectionGroupForMediaCharacteristic:AVMediaCharacteristicLegible];
for (int i = 0; i < group.options.count; ++i) { for (int i = 0; i < group.options.count; ++i) {
AVMediaSelectionOption *currentOption = [group.options objectAtIndex:i]; AVMediaSelectionOption *currentOption = [group.options objectAtIndex:i];
NSString *title = [[[currentOption commonMetadata] NSString *title = @"";
valueForKey:@"value"] NSArray *values = [[currentOption commonMetadata] valueForKey:@"value"];
objectAtIndex:0]; if (values.count > 0) {
title = [values objectAtIndex:0];
}
NSString *language = [currentOption extendedLanguageTag] ? [currentOption extendedLanguageTag] : @"";
NSDictionary *textTrack = @{ NSDictionary *textTrack = @{
@"index": [NSNumber numberWithInt:i], @"index": [NSNumber numberWithInt:i],
@"title": title, @"title": title,
@"language": [currentOption extendedLanguageTag] @"language": language
}; };
[textTracks addObject:textTrack]; [textTracks addObject:textTrack];
} }
@ -931,20 +941,21 @@ static NSString *const timedMetadata = @"timedMetadata";
- (void)usePlayerLayer - (void)usePlayerLayer
{ {
if( _player ) if( _player )
{ {
_playerLayer = [AVPlayerLayer playerLayerWithPlayer:_player]; _playerLayer = [AVPlayerLayer playerLayerWithPlayer:_player];
_playerLayer.frame = self.bounds; _playerLayer.frame = self.bounds;
_playerLayer.needsDisplayOnBoundsChange = YES; _playerLayer.needsDisplayOnBoundsChange = YES;
// to prevent video from being animated when resizeMode is 'cover' // to prevent video from being animated when resizeMode is 'cover'
// resize mode must be set before layer is added // resize mode must be set before layer is added
[self setResizeMode:_resizeMode]; [self setResizeMode:_resizeMode];
[_playerLayer addObserver:self forKeyPath:readyForDisplayKeyPath options:NSKeyValueObservingOptionNew context:nil]; [_playerLayer addObserver:self forKeyPath:readyForDisplayKeyPath options:NSKeyValueObservingOptionNew context:nil];
_playerLayerObserverSet = YES;
[self.layer addSublayer:_playerLayer];
self.layer.needsDisplayOnBoundsChange = YES; [self.layer addSublayer:_playerLayer];
} self.layer.needsDisplayOnBoundsChange = YES;
}
} }
- (void)setControls:(BOOL)controls - (void)setControls:(BOOL)controls
@ -978,9 +989,12 @@ static NSString *const timedMetadata = @"timedMetadata";
- (void)removePlayerLayer - (void)removePlayerLayer
{ {
[_playerLayer removeFromSuperlayer]; [_playerLayer removeFromSuperlayer];
if (_playerLayerObserverSet) {
[_playerLayer removeObserver:self forKeyPath:readyForDisplayKeyPath]; [_playerLayer removeObserver:self forKeyPath:readyForDisplayKeyPath];
_playerLayer = nil; _playerLayerObserverSet = NO;
}
_playerLayer = nil;
} }
#pragma mark - RCTVideoPlayerViewControllerDelegate #pragma mark - RCTVideoPlayerViewControllerDelegate