Cue Tags (#791)

* Add text track cues for #ext-x-cue-out, #ext-x-cue-in, and #ext-x-cue-cont segment tags
* Change tags track to keep its reference and add documentation

README.md

@@ -24,6 +24,7 @@ Play back HLS with video.js, even where it's not natively supported.
       - [Source](#source)
     - [List](#list)
       - [withCredentials](#withcredentials)
+      - [useCueTags](#usecuetags)
   - [Runtime Properties](#runtime-properties)
     - [hls.playlists.master](#hlsplaylistsmaster)
     - [hls.playlists.media](#hlsplaylistsmedia)
@@ -212,6 +213,43 @@ is set to `true`.
 See html5rocks's [article](http://www.html5rocks.com/en/tutorials/cors/)
 for more info.
 
+##### useCueTags
+* Type: `boolean`
+* can be used as an initialization option
+
+When the `useCueTags` property is set to `true,` a text track is created with
+label 'hls-segment-metadata' and kind 'metadata'. The track is then added to
+`player.textTracks()`. Whenever a segment associated with a cue tag is playing,
+the cue tags will be listed as a properties inside of a stringified JSON object
+under its active cue's `text` property. The properties that are currently
+supported are cueOut, cueOutCont, and cueIn. Changes in active cue may be
+tracked by following the Video.js cue points API for text tracks. For example:
+
+```javascript
+let textTracks = player.textTracks();
+let cuesTrack;
+
+for (let i = 0; i < textTracks.length; i++) {
+  if (textTracks[i].label === 'hls-segment-metadata') {
+    cuesTrack = textTracks[i];
+  }
+}
+
+cuesTrack.addEventListener('cuechange', function() {
+  let activeCues = cuesTrack.activeCues;
+
+  for (let i = 0; i < activeCues.length; i++) {
+    let activeCue = activeCues[i];
+    let cueData = JSON.parse(activeCue.text);
+
+    console.log('Cue runs from ' + activeCue.startTime +
+                ' to ' + activeCue.endTime +
+                ' with cue tag contents ' +
+                (cueData.cueOut || cueData.cueOutCont || cueData.cueIn));
+  }
+});
+```
+
 ### Runtime Properties
 Runtime properties are attached to the tech object when HLS is in
 use. You can get a reference to the HLS source handler like this:

src/master-playlist-controller.js

@@ -6,6 +6,7 @@ import SegmentLoader from './segment-loader';
 import Ranges from './ranges';
 import videojs from 'video.js';
 import HlsAudioTrack from './hls-audio-track';
+import window from 'global/window';
 
 // 5 minute blacklist
 const BLACKLIST_DURATION = 5 * 60 * 1000;
@@ -48,7 +49,8 @@ export default class MasterPlaylistController extends videojs.EventTarget {
     mode,
     tech,
     bandwidth,
-    externHls
+    externHls,
+    useCueTags
   }) {
     super();
 
@@ -58,6 +60,13 @@ export default class MasterPlaylistController extends videojs.EventTarget {
     this.tech_ = tech;
     this.hls_ = tech.hls;
     this.mode_ = mode;
+    this.useCueTags_ = useCueTags;
+    if (this.useCueTags_) {
+      this.cueTagsTrack_ = this.tech_.addTextTrack('metadata', 'hls-segment-metadata');
+      this.cueTagsTrack_.inBandMetadataTrackDispatchType = '';
+      this.tech_.textTracks().addTrack_(this.cueTagsTrack_);
+    }
+
     this.audioTracks_ = [];
     this.requestOptions_ = {
       withCredentials: this.withCredentials,
@@ -124,6 +133,8 @@ export default class MasterPlaylistController extends videojs.EventTarget {
         return;
       }
 
+      this.updateCues_(updatedPlaylist);
+
       // TODO: Create a new event on the PlaylistLoader that signals
       // that the segments have changed in some way and use that to
       // update the SegmentLoader instead of doing it twice here and
@@ -819,4 +830,44 @@ export default class MasterPlaylistController extends videojs.EventTarget {
       }
     });
   }
+
+  updateCues_(media) {
+    if (!this.useCueTags_ || !media.segments) {
+      return;
+    }
+
+    while (this.cueTagsTrack_.cues.length) {
+      this.cueTagsTrack_.removeCue(this.cueTagsTrack_.cues[0]);
+    }
+
+    let mediaTime = 0;
+
+    for (let i = 0; i < media.segments.length; i++) {
+      let segment = media.segments[i];
+
+      if ('cueOut' in segment || 'cueOutCont' in segment || 'cueIn' in segment) {
+        let cueJson = {};
+
+        if ('cueOut' in segment) {
+          cueJson.cueOut = segment.cueOut;
+        }
+        if ('cueOutCont' in segment) {
+          cueJson.cueOutCont = segment.cueOutCont;
+        }
+        if ('cueIn' in segment) {
+          cueJson.cueIn = segment.cueIn;
+        }
+
+        // Use a short duration for the cue point, as it should trigger for a segment
+        // transition (in this case, defined as the beginning of the segment that the tag
+        // precedes), but keep it for a minimum of 0.5 seconds to remain usable (won't
+        // lose it as an active cue by the time a user retrieves the active cues).
+        this.cueTagsTrack_.addCue(new window.VTTCue(mediaTime,
+                                                    mediaTime + 0.5,
+                                                    JSON.stringify(cueJson)));
+      }
+
+      mediaTime += segment.duration;
+    }
+  }
 }

test/master-playlist-controller.test.js

@@ -13,6 +13,7 @@ import MasterPlaylistController from '../src/master-playlist-controller';
 import { Hls } from '../src/videojs-contrib-hls';
 /* eslint-enable no-unused-vars */
 import Playlist from '../src/playlist';
+import window from 'global/window';
 
 QUnit.module('MasterPlaylistController', {
   beforeEach() {
@@ -617,3 +618,184 @@ function() {
 
   Playlist.seekable = origSeekable;
 });
+
+QUnit.test('calls to update cues on new media', function() {
+  let callCount = 0;
+
+  this.masterPlaylistController.updateCues_ = (media) => callCount++;
+
+  // master
+  standardXHRResponse(this.requests.shift());
+
+  QUnit.equal(callCount, 0, 'no call to update cues on master');
+
+  // media
+  standardXHRResponse(this.requests.shift());
+
+  QUnit.equal(callCount, 1, 'calls to update cues on first media');
+
+  this.masterPlaylistController.masterPlaylistLoader_.trigger('loadedplaylist');
+
+  QUnit.equal(callCount, 2, 'calls to update cues on subsequent media');
+});
+
+QUnit.test('calls to update cues on media when no master', function() {
+  this.requests.length = 0;
+  this.player.src({
+    src: 'manifest/media.m3u8',
+    type: 'application/vnd.apple.mpegurl'
+  });
+  this.masterPlaylistController = this.player.tech_.hls.masterPlaylistController_;
+
+  let callCount = 0;
+
+  this.masterPlaylistController.updateCues_ = (media) => callCount++;
+
+  // media
+  standardXHRResponse(this.requests.shift());
+
+  QUnit.equal(callCount, 1, 'calls to update cues on first media');
+
+  this.masterPlaylistController.masterPlaylistLoader_.trigger('loadedplaylist');
+
+  QUnit.equal(callCount, 2, 'calls to update cues on subsequent media');
+});
+
+QUnit.test('respects useCueTags option', function() {
+  this.masterPlaylistController.updateCues_({
+    segments: [{
+      duration: 10,
+      tags: ['test']
+    }]
+  });
+
+  QUnit.ok(!this.masterPlaylistController.cueTagsTrack_,
+           'does not create cueTagsTrack_ if useCueTags is falsy');
+  QUnit.equal(this.player.textTracks().length,
+              0,
+              'does not create a text track if useCueTags is falsy');
+
+  this.player.dispose();
+
+  let origHlsOptions = videojs.options.hls;
+
+  videojs.options.hls = {
+    useCueTags: true
+  };
+
+  this.player = createPlayer();
+  this.player.src({
+    src: 'manifest/media.m3u8',
+    type: 'application/vnd.apple.mpegurl'
+  });
+  this.masterPlaylistController = this.player.tech_.hls.masterPlaylistController_;
+
+  QUnit.ok(this.masterPlaylistController.cueTagsTrack_,
+           'creates cueTagsTrack_ if useCueTags is truthy');
+  QUnit.equal(this.masterPlaylistController.cueTagsTrack_.label,
+              'hls-segment-metadata',
+              'cueTagsTrack_ has label of hls-segment-metadata');
+  QUnit.equal(this.player.textTracks()[0], this.masterPlaylistController.cueTagsTrack_,
+           'adds cueTagsTrack as a text track if useCueTags is truthy');
+
+  this.masterPlaylistController.updateCues_({
+    segments: [{
+      duration: 10,
+      cueOut: 'test'
+    }]
+  });
+
+  let cue = this.masterPlaylistController.cueTagsTrack_.cues[0];
+
+  QUnit.equal(cue.startTime,
+              0,
+              'adds cue with correct start time if useCueTags is truthy');
+  QUnit.equal(cue.endTime,
+              0.5,
+              'adds cue with correct end time if useCueTags is truthy');
+  QUnit.equal(cue.text,
+              JSON.stringify({ cueOut: 'test' }),
+              'adds cue with correct text if useCueTags is truthy');
+
+  videojs.options.hls = origHlsOptions;
+});
+
+QUnit.test('update tag cues', function() {
+  let origHlsOptions = videojs.options.hls;
+
+  videojs.options.hls = {
+    useCueTags: true
+  };
+
+  this.player = createPlayer();
+  this.player.src({
+    src: 'manifest/master.m3u8',
+    type: 'application/vnd.apple.mpegurl'
+  });
+  this.masterPlaylistController = this.player.tech_.hls.masterPlaylistController_;
+
+  let cueTagsTrack = this.masterPlaylistController.cueTagsTrack_;
+  let testCue = new window.VTTCue(0, 10, 'test');
+
+  cueTagsTrack.addCue(testCue);
+
+  this.masterPlaylistController.updateCues_({});
+
+  QUnit.equal(cueTagsTrack.cues.length,
+              1,
+              'does not change cues if media does not have segment property');
+  QUnit.equal(cueTagsTrack.cues[0],
+              testCue,
+              'does not change cues if media does not have segment property');
+
+  this.masterPlaylistController.updateCues_({
+    segments: []
+  });
+
+  QUnit.equal(cueTagsTrack.cues.length,
+              0,
+              'removes cues even if no segments in playlist');
+
+  this.masterPlaylistController.updateCues_({
+    segments: [{
+      duration: 5.1,
+      cueOut: '11.5'
+    }, {
+      duration: 6.4,
+      cueOutCont: '5.1/11.5'
+    }, {
+      duration: 6,
+      cueIn: ''
+    }]
+  });
+
+  QUnit.equal(cueTagsTrack.cues.length, 3, 'adds a cue for each segment');
+
+  QUnit.equal(cueTagsTrack.cues[0].startTime, 0, 'cue starts at 0');
+  QUnit.equal(cueTagsTrack.cues[0].endTime, 0.5, 'cue ends at start time plus duration');
+  QUnit.equal(JSON.parse(cueTagsTrack.cues[0].text).cueOut, '11.5', 'cueOut matches');
+  QUnit.ok(!('cueOutCont' in JSON.parse(cueTagsTrack.cues[0].text)),
+           'cueOutCont not in cue');
+  QUnit.ok(!('cueIn' in JSON.parse(cueTagsTrack.cues[0].text)), 'cueIn not in cue');
+  QUnit.equal(cueTagsTrack.cues[1].startTime, 5.1, 'cue starts at 5.1');
+  QUnit.equal(cueTagsTrack.cues[1].endTime, 5.6, 'cue ends at start time plus duration');
+  QUnit.equal(JSON.parse(cueTagsTrack.cues[1].text).cueOutCont,
+              '5.1/11.5',
+              'cueOutCont matches');
+  QUnit.ok(!('cueOut' in JSON.parse(cueTagsTrack.cues[1].text)), 'cueOut not in cue');
+  QUnit.ok(!('cueIn' in JSON.parse(cueTagsTrack.cues[1].text)), 'cueIn not in cue');
+  QUnit.equal(cueTagsTrack.cues[2].startTime, 11.5, 'cue starts at 11.5');
+  QUnit.equal(cueTagsTrack.cues[2].endTime, 12, 'cue ends at start time plus duration');
+  QUnit.equal(JSON.parse(cueTagsTrack.cues[2].text).cueIn, '', 'cueIn matches');
+  QUnit.ok(!('cueOut' in JSON.parse(cueTagsTrack.cues[2].text)), 'cueOut not in cue');
+  QUnit.ok(!('cueOutCont' in JSON.parse(cueTagsTrack.cues[2].text)),
+           'cueOutCont not in cue');
+
+  this.masterPlaylistController.updateCues_({
+    segments: []
+  });
+
+  QUnit.equal(cueTagsTrack.cues.length, 0, 'removes old cues on update');
+
+  videojs.options.hls = origHlsOptions;
+});

test/videojs-contrib-hls.test.js

@@ -2275,6 +2275,34 @@ QUnit.test('Allows overriding the global beforeRequest function', function() {
   QUnit.equal(this.player.tech_.hls.stats.mediaRequests, 1, 'one segment request');
 });
 
+QUnit.test('passes useCueTags hls option to master playlist controller', function() {
+  this.player.src({
+    src: 'master.m3u8',
+    type: 'application/vnd.apple.mpegurl'
+  });
+
+  QUnit.ok(!this.player.tech_.hls.masterPlaylistController_.useCueTags_,
+           'useCueTags is falsy by default');
+
+  let origHlsOptions = videojs.options.hls;
+
+  videojs.options.hls = {
+    useCueTags: true
+  };
+
+  this.player.dispose();
+  this.player = createPlayer();
+  this.player.src({
+    src: 'http://example.com/media.m3u8',
+    type: 'application/vnd.apple.mpegurl'
+  });
+
+  QUnit.ok(this.player.tech_.hls.masterPlaylistController_.useCueTags_,
+           'useCueTags passed to master playlist controller');
+
+  videojs.options.hls = origHlsOptions;
+});
+
 QUnit.module('HLS Integration', {
   beforeEach() {
     this.env = useFakeEnvironment();