Merge pull request #1063 from react-native-community/feature/sidecar-text-tracks

Sideloaded text tracks & include text track info in onLoad

README.md

@@ -191,8 +191,6 @@ using System.Collections.Generic;
        onFullscreenPlayerDidPresent={this.fullScreenPlayerDidPresent}   // Callback after fullscreen started
        onFullscreenPlayerWillDismiss={this.fullScreenPlayerWillDismiss} // Callback before fullscreen stops
        onFullscreenPlayerDidDismiss={this.fullScreenPlayerDidDismiss}  // Callback after fullscreen stopped
-       onLoadStart={this.loadStart}            // Callback when video starts to load
-       onLoad={this.setDuration}               // Callback when video loads
        onProgress={this.setTime}               // Callback every ~250ms with currentTime
        onTimedMetadata={this.onTimedMetadata}  // Callback when the stream receive some metadata
        style={styles.backgroundVideo} />
@@ -233,9 +231,14 @@ var styles = StyleSheet.create({
 * [resizeMode](#resizemode)
 * [selectedTextTrack](#selectedtexttrack)
 * [stereoPan](#stereopan)
+* [textTracks](#texttracks)
 * [useTextureView](#usetextureview)
 * [volume](#volume)
 
+### Event props
+* [onLoad](#onload)
+* [onLoadStart](#onloadstart)
+
 #### allowsExternalPlayback
 Indicates whether the player allows switching to external playback mode such as AirPlay or HDMI.
 * **true (default)** - allow switching to external playback mode
@@ -359,7 +362,7 @@ Type | Value | Description
 "language" | string | Display the text track with the language specified as the Value, e.g. "fr"
 "index" | number | Display the text track with the index specified as the value, e.g. 0
 
-Both iOS & Android offer Settings to enable Captions for hearing impaired people. If "system" is selected and the Captions Setting is enabled, iOS/Android will look for a caption that matches that customer's language and display it.
+Both iOS & Android (only 4.4 and higher) offer Settings to enable Captions for hearing impaired people. If "system" is selected and the Captions Setting is enabled, iOS/Android will look for a caption that matches that customer's language and display it. 
 
 If a track matching the specified Type (and Value if appropriate) is unavailable, no text track will be displayed. If multiple tracks match the criteria, the first match will be used.
 
@@ -373,6 +376,40 @@ Adjust the balance of the left and right audio channels.  Any value between –1
 
 Platforms: Android MediaPlayer
 
+#### textTracks
+Load one or more "sidecar" text tracks. This takes an array of objects representing each track. Each object should have the format:
+
+Property | Description
+--- | ---
+title | Descriptive name for the track
+language | 2 letter [ISO 639-1 code](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) representing the language
+type | Mime type of the track<br> * TextTrackType.SRT - .srt SubRip Subtitle<br> * TextTrackType.TTML - .ttml TTML<br> * TextTrackType.VTT - .vtt WebVTT
+uri | URL for the text track. Currently, only tracks hosted on a webserver are supported
+
+Example:
+```
+import { TextTrackType }, Video from 'react-native-video';
+
+textTracks={[
+  {
+    title: "English CC",
+    language: "en",
+    type: "text/vtt", TextTrackType.VTT,
+    uri: "https://bitdash-a.akamaihd.net/content/sintel/subtitles/subtitles_en.vtt"
+  },
+  {
+    title: "Spanish Subtitles",
+    language: "es",
+    type: "application/x-subrip", TextTrackType.SRT,
+    uri: "https://durian.blender.org/wp-content/content/subtitles/sintel_es.srt"
+  }
+]}
+```
+
+This isn't support on iOS because AVPlayer doesn't support it. Text tracks must be loaded as part of an HLS playlist.
+
+Platforms: Android ExoPlayer
+
 #### useTextureView
 Output to a TextureView instead of the default SurfaceView. In general, you will want to use SurfaceView because it is more efficient and provides better performance. However, SurfaceViews has two limitations:
 * It can't be animated, transformed or scaled
@@ -393,6 +430,68 @@ Adjust the volume.
 
 Platforms: all
 
+### Event props
+
+#### onLoad
+Callback function that is called when the media is loaded and ready to play.
+
+Payload:
+
+Property | Type | Description
+--- | --- | ---
+currentPosition | number | Time in seconds where the media will start
+duration | number | Length of the media in seconds
+naturalSize | object | Properties:<br> * width - Width in pixels that the video was encoded at<br> * height - Height in pixels that the video was encoded at<br> * orientation - "portrait" or "landscape"
+textTracks | array | An array of text track info objects with the following properties:<br> * index - Index number<br> * title - Description of the track<br> * language - 2 letter [ISO 639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) language code<br> * type - Mime type of track
+
+Example:
+```
+{ 
+  canPlaySlowForward: true,
+  canPlayReverse: false,
+  canPlaySlowReverse: false,
+  canPlayFastForward: false,
+  canStepForward: false,
+  canStepBackward: false,
+  currentTime: 0,
+  duration: 5910.208984375,
+  naturalSize: {
+     height: 1080
+     orientation: 'landscape'
+     width: '1920'
+  },
+  textTracks: [
+    { title: '#1 French', language: 'fr', index: 0, type: 'text/vtt' },
+    { title: '#2 English CC', language: 'en', index: 1, type: 'text/vtt' },
+    { title: '#3 English Director Commentary', language: 'en', index: 2, type: 'text/vtt' }
+  ]
+}
+```
+
+Platforms: all
+
+#### onLoadStart
+Callback function that is called when the media starts loading.
+
+Payload:
+
+Property | Description
+--- | ---
+isNetwork | Boolean indicating if the media is being loaded from the network
+type | Type of the media. Not available on Windows
+uri | URI for the media source. Not available on Windows
+
+Example:
+```
+{
+  isNetwork: true,
+  type: '',
+  uri: 'https://bitdash-a.akamaihd.net/content/sintel/hls/playlist.m3u8'
+}
+```
+
+Platforms: all
+
 ### Additional props
 
 To see the full list of available props, you can check the [propTypes](https://github.com/react-native-community/react-native-video/blob/master/Video.js#L246) of the Video.js component.

TextTrackType.js

@@ -0,0 +1,7 @@
+import keyMirror from 'keymirror';
+
+export default {
+  SRT: 'application/x-subrip',
+  TTML: 'application/ttml+xml',
+  VTT: 'text/vtt'
+};

Video.js

@@ -2,6 +2,7 @@ import React, {Component} from 'react';
 import PropTypes from 'prop-types';
 import {StyleSheet, requireNativeComponent, NativeModules, View, ViewPropTypes, Image} from 'react-native';
 import resolveAssetSource from 'react-native/Libraries/Image/resolveAssetSource';
+import TextTrackType from './TextTrackType';
 import VideoResizeMode from './VideoResizeMode.js';
 
 const styles = StyleSheet.create({
@@ -10,6 +11,8 @@ const styles = StyleSheet.create({
   },
 });
 
+export { TextTrackType };
+
 export default class Video extends Component {
 
   constructor(props) {
@@ -282,6 +285,18 @@ Video.propTypes = {
       PropTypes.number
     ])
   }),
+  textTracks: PropTypes.arrayOf(
+    PropTypes.shape({
+      title: PropTypes.string,
+      uri: PropTypes.string.isRequired,
+      type: PropTypes.oneOf([
+        TextTrackType.SRT,
+        TextTrackType.TTML,
+        TextTrackType.VTT,
+      ]),
+      language: PropTypes.string.isRequired
+    })
+  ),
   paused: PropTypes.bool,
   muted: PropTypes.bool,
   volume: PropTypes.number,

android-exoplayer/src/main/java/com/brentvatne/exoplayer/ReactExoplayerView.java

@@ -16,8 +16,13 @@ import android.widget.FrameLayout;
 import com.brentvatne.react.R;
 import com.brentvatne.receiver.AudioBecomingNoisyReceiver;
 import com.brentvatne.receiver.BecomingNoisyListener;
+import com.facebook.react.bridge.Arguments;
 import com.facebook.react.bridge.Dynamic;
 import com.facebook.react.bridge.LifecycleEventListener;
+import com.facebook.react.bridge.ReadableArray;
+import com.facebook.react.bridge.ReadableMap;
+import com.facebook.react.bridge.WritableArray;
+import com.facebook.react.bridge.WritableMap;
 import com.facebook.react.uimanager.ThemedReactContext;
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.DefaultLoadControl;
@@ -37,6 +42,8 @@ import com.google.android.exoplayer2.metadata.MetadataRenderer;
 import com.google.android.exoplayer2.source.BehindLiveWindowException;
 import com.google.android.exoplayer2.source.ExtractorMediaSource;
 import com.google.android.exoplayer2.source.MediaSource;
+import com.google.android.exoplayer2.source.MergingMediaSource;
+import com.google.android.exoplayer2.source.SingleSampleMediaSource;
 import com.google.android.exoplayer2.source.TrackGroupArray;
 import com.google.android.exoplayer2.source.dash.DashMediaSource;
 import com.google.android.exoplayer2.source.dash.DefaultDashChunkSource;
@@ -51,6 +58,7 @@ import com.google.android.exoplayer2.trackselection.TrackSelection;
 import com.google.android.exoplayer2.trackselection.TrackSelectionArray;
 import com.google.android.exoplayer2.upstream.DataSource;
 import com.google.android.exoplayer2.upstream.DefaultBandwidthMeter;
+import com.google.android.exoplayer2.util.MimeTypes;
 import com.google.android.exoplayer2.util.Util;
 
 import java.net.CookieHandler;
@@ -58,6 +66,7 @@ import java.net.CookieManager;
 import java.net.CookiePolicy;
 import java.lang.Math;
 import java.lang.Object;
+import java.util.ArrayList;
 
 @SuppressLint("ViewConstructor")
 class ReactExoplayerView extends FrameLayout implements
@@ -103,6 +112,7 @@ class ReactExoplayerView extends FrameLayout implements
     private boolean repeat;
     private String textTrackType;
     private Dynamic textTrackValue;
+    private ReadableArray textTracks;
     private boolean disableFocus;
     private float mProgressUpdateInterval = 250.0f;
     private boolean playInBackground = false;
@@ -229,7 +239,19 @@ class ReactExoplayerView extends FrameLayout implements
             player.setPlaybackParameters(params);
         }
         if (playerNeedsSource && srcUri != null) {
-            MediaSource mediaSource = buildMediaSource(srcUri, extension);
+            ArrayList<MediaSource> mediaSourceList = buildTextSources();
+            MediaSource videoSource = buildMediaSource(srcUri, extension);
+            MediaSource mediaSource;
+            if (mediaSourceList.size() == 0) {
+                mediaSource = videoSource;
+            } else {
+                mediaSourceList.add(0, videoSource);
+                MediaSource[] textSourceArray = mediaSourceList.toArray(
+                        new MediaSource[mediaSourceList.size()]
+                );
+                mediaSource = new MergingMediaSource(textSourceArray);
+            }
+
             boolean haveResumePosition = resumeWindow != C.INDEX_UNSET;
             if (haveResumePosition) {
                 player.seekTo(resumeWindow, resumePosition);
@@ -263,6 +285,32 @@ class ReactExoplayerView extends FrameLayout implements
         }
     }
 
+    private ArrayList<MediaSource> buildTextSources() {
+        ArrayList<MediaSource> textSources = new ArrayList<>();
+        if (textTracks == null) {
+            return textSources;
+        }
+
+        for (int i = 0; i < textTracks.size(); ++i) {
+            ReadableMap textTrack = textTracks.getMap(i);
+            String language = textTrack.getString("language");
+            String title = textTrack.hasKey("title")
+                    ? textTrack.getString("title") : language + " " + i;
+            Uri uri = Uri.parse(textTrack.getString("uri"));
+            MediaSource textSource = buildTextSource(title, uri, textTrack.getString("type"),
+                    language);
+            if (textSource != null) {
+                textSources.add(textSource);
+            }
+        }
+        return textSources;
+    }
+
+    private MediaSource buildTextSource(String title, Uri uri, String mimeType, String language) {
+        Format textFormat = Format.createTextSampleFormat(title, mimeType, Format.NO_VALUE, language);
+        return new SingleSampleMediaSource(uri, mediaDataSourceFactory, textFormat, C.TIME_UNSET);
+    }
+
     private void releasePlayer() {
         if (player != null) {
             isPaused = player.getPlayWhenReady();
@@ -453,10 +501,33 @@ class ReactExoplayerView extends FrameLayout implements
             Format videoFormat = player.getVideoFormat();
             int width = videoFormat != null ? videoFormat.width : 0;
             int height = videoFormat != null ? videoFormat.height : 0;
-            eventEmitter.load(player.getDuration(), player.getCurrentPosition(), width, height);
+            eventEmitter.load(player.getDuration(), player.getCurrentPosition(), width, height,
+                    getTextTrackInfo());
         }
     }
 
+    private WritableArray getTextTrackInfo() {
+        WritableArray textTracks = Arguments.createArray();
+
+        MappingTrackSelector.MappedTrackInfo info = trackSelector.getCurrentMappedTrackInfo();
+        int index = getTextTrackRendererIndex();
+        if (info == null || index == C.INDEX_UNSET) {
+            return textTracks;
+        }
+
+        TrackGroupArray groups = info.getTrackGroups(index);
+        for (int i = 0; i < groups.length; ++i) {
+             Format format = groups.get(i).getFormat(0);
+             WritableMap textTrack = Arguments.createMap();
+             textTrack.putInt("index", i);
+             textTrack.putString("title", format.id);
+             textTrack.putString("type", format.sampleMimeType);
+             textTrack.putString("language", format.language);
+             textTracks.pushMap(textTrack);
+        }
+        return textTracks;
+    }
+
     private void onBuffering(boolean buffering) {
         if (isBuffering == buffering) {
             return;
@@ -623,6 +694,11 @@ class ReactExoplayerView extends FrameLayout implements
         }
     }
 
+    public void setTextTracks(ReadableArray textTracks) {
+        this.textTracks = textTracks;
+        reloadSource();
+    }
+
     private void reloadSource() {
         playerNeedsSource = true;
         initializePlayer();

android-exoplayer/src/main/java/com/brentvatne/exoplayer/ReactExoplayerViewManager.java

@@ -5,6 +5,7 @@ import android.net.Uri;
 import android.text.TextUtils;
 
 import com.facebook.react.bridge.Dynamic;
+import com.facebook.react.bridge.ReadableArray;
 import com.facebook.react.bridge.ReadableMap;
 import com.facebook.react.common.MapBuilder;
 import com.facebook.react.uimanager.ThemedReactContext;
@@ -28,6 +29,7 @@ public class ReactExoplayerViewManager extends ViewGroupManager<ReactExoplayerVi
     private static final String PROP_SELECTED_TEXT_TRACK = "selectedTextTrack";
     private static final String PROP_SELECTED_TEXT_TRACK_TYPE = "type";
     private static final String PROP_SELECTED_TEXT_TRACK_VALUE = "value";
+    private static final String PROP_TEXT_TRACKS = "textTracks";
     private static final String PROP_PAUSED = "paused";
     private static final String PROP_MUTED = "muted";
     private static final String PROP_VOLUME = "volume";
@@ -131,6 +133,12 @@ public class ReactExoplayerViewManager extends ViewGroupManager<ReactExoplayerVi
         videoView.setSelectedTextTrack(typeString, value);
     }
 
+    @ReactProp(name = PROP_TEXT_TRACKS)
+    public void setPropTextTracks(final ReactExoplayerView videoView,
+                                  @Nullable ReadableArray textTracks) {
+        videoView.setTextTracks(textTracks);
+    }
+
     @ReactProp(name = PROP_PAUSED, defaultBoolean = false)
     public void setPaused(final ReactExoplayerView videoView, final boolean paused) {
         videoView.setPausedModifier(paused);

android-exoplayer/src/main/java/com/brentvatne/exoplayer/VideoEventEmitter.java

@@ -109,6 +109,7 @@ class VideoEventEmitter {
     private static final String EVENT_PROP_WIDTH = "width";
     private static final String EVENT_PROP_HEIGHT = "height";
     private static final String EVENT_PROP_ORIENTATION = "orientation";
+    private static final String EVENT_PROP_TEXT_TRACKS = "textTracks";
     private static final String EVENT_PROP_HAS_AUDIO_FOCUS = "hasAudioFocus";
     private static final String EVENT_PROP_IS_BUFFERING = "isBuffering";
     private static final String EVENT_PROP_PLAYBACK_RATE = "playbackRate";
@@ -128,7 +129,8 @@ class VideoEventEmitter {
         receiveEvent(EVENT_LOAD_START, null);
     }
 
-    void load(double duration, double currentPosition, int videoWidth, int videoHeight) {
+    void load(double duration, double currentPosition, int videoWidth, int videoHeight,
+              WritableArray textTracks) {
         WritableMap event = Arguments.createMap();
         event.putDouble(EVENT_PROP_DURATION, duration / 1000D);
         event.putDouble(EVENT_PROP_CURRENT_TIME, currentPosition / 1000D);
@@ -143,6 +145,8 @@ class VideoEventEmitter {
         }
         event.putMap(EVENT_PROP_NATURAL_SIZE, naturalSize);
 
+        event.putArray(EVENT_PROP_TEXT_TRACKS, textTracks);
+
         // TODO: Actually check if you can.
         event.putBoolean(EVENT_PROP_FAST_FORWARD, true);
         event.putBoolean(EVENT_PROP_SLOW_FORWARD, true);

ios/RCTVideo.m

@@ -415,6 +415,7 @@ static NSString *const timedMetadata = @"timedMetadata";
                                      @"height": height,
                                      @"orientation": orientation
                                      },
+                             @"textTracks": [self getTextTrackInfo],
                              @"target": self.reactTag});
       }
 
@@ -694,6 +695,26 @@ static NSString *const timedMetadata = @"timedMetadata";
   [_player.currentItem selectMediaOption:option inMediaSelectionGroup:group];
 }
 
+- (NSArray *)getTextTrackInfo
+{
+  NSMutableArray *textTracks = [[NSMutableArray alloc] init];
+  AVMediaSelectionGroup *group = [_player.currentItem.asset
+                                  mediaSelectionGroupForMediaCharacteristic:AVMediaCharacteristicLegible];
+  for (int i = 0; i < group.options.count; ++i) {
+    AVMediaSelectionOption *currentOption = [group.options objectAtIndex:i];
+    NSString *title = [[[currentOption commonMetadata]
+                        valueForKey:@"value"]
+                       objectAtIndex:0];
+    NSDictionary *textTrack = @{
+                                @"index": [NSNumber numberWithInt:i],
+                                @"title": title,
+                                @"language": [currentOption extendedLanguageTag]
+                                };
+    [textTracks addObject:textTrack];
+  }
+  return textTracks;
+}
+
 - (BOOL)getFullscreen
 {
     return _fullscreenPlayerPresented;