HLS -> FLV translation working. Fairly extensive FLV validation testing.

Fix up a couple remaining issues with the HLS->FLV translation. At this point, we've validated that the generated file can be played back in VLC if you download it to your computer. Added another ts segment for testing purposes. Added unit testing that traverses the generated FLV and validates the tags are constructed correctly and seem consistent.
12 years ago · b2ed4313f3
13 changed files with 19064 additions and 52 deletions
--- a/example.html
+++ b/example.html
@ -45,8 +45,12 @@
  <script src="src/h264-stream.js"></script>
  <script src="src/aac-stream.js"></script>
  <script src="src/segment-parser.js"></script>
-  <!-- an example MPEG2-TS segment -->
-  <script src="test/tsSegment.js"></script>
+  
+  <!-- example MPEG2-TS segments -->
+  <!-- bipbop -->
+  <!-- <script src="test/tsSegment.js"></script> -->
+  <!-- bunnies -->
+  <script src="test/tsSegment-bc.js"></script>

 </head>
 <body>
@ -67,20 +71,31 @@
    mediaSource = new videojs.MediaSource();

    mediaSource.addEventListener('sourceopen', function(event){
-      var tag, bytes, parser;
+      var tag, bytes, parser, i, feedBytes, everything, old;

      // feed parsed bytes into the player
      var sourceBuffer = mediaSource.addSourceBuffer('video/flv; codecs="vp6,aac"');
      parser = new videojs.hls.SegmentParser();
-      var header = parser.getFlvHeader();
-      sourceBuffer.appendBuffer(header);
+      // var header = parser.getFlvHeader();
+      everything = parser.getFlvHeader();
+      // sourceBuffer.appendBuffer(header);
+
+      parser.parseSegmentBinaryData(window.bcSegment);

-      parser.parseSegmentBinaryData(window.testSegment);
      while (parser.tagsAvailable()) {
        tag = parser.getNextTag();
-        console.log('sending bytes', tag.length);
-        sourceBuffer.appendBuffer(tag.bytes.subarray(0, tag.length), video);
+        old = everything;
+        everything = new Uint8Array(old.byteLength + tag.bytes.byteLength);
+        everything.set(old);
+        everything.set(tag.bytes, old.byteLength);
      }
+      console.log('sending ' + everything.byteLength + 'B');
+      var iframe = document.createElement('iframe');
+      iframe.src = 'data:video/x-flv;base64,' + window.btoa((Array.prototype.map.call(everything, function(byte) {
+        return String.fromCharCode(byte);
+      })).join(''));
+      document.body.appendChild(iframe);
+      // sourceBuffer.appendBuffer(everything, video);
    }, false);

    url = videojs.URL.createObjectURL(mediaSource);
--- a/hls-example.html
+++ b/hls-example.html
@ -0,0 +1,40 @@
+<!doctype html>
+<html>
+  <head><title></title></head>
+  <body>
+<!-- Start of Brightcove Player -->
+
+<div style="display:none">
+
+</div>
+
+<!--
+By use of this code snippet, I agree to the Brightcove Publisher T and C 
+found at https://accounts.brightcove.com/en/terms-and-conditions/. 
+-->
+
+<script language="JavaScript" type="text/javascript" src="http://admin.brightcove.com/js/BrightcoveExperiences.js"></script>
+
+<object id="myExperience791331688001" class="BrightcoveExperience">
+  <param name="bgcolor" value="#FFFFFF" />
+  <param name="width" value="500" />
+  <param name="height" value="470" />
+  <param name="playerID" value="2498485115001" />
+  <param name="playerKey" value="AQ~~,AAAAdgygTQk~,cL85eN46vuSRabEOn8tmRIgEmJahevxf" />
+  <param name="isVid" value="true" />
+  <param name="isUI" value="true" />
+  <param name="dynamicStreaming" value="true" />
+  
+  <param name="@videoPlayer" value="791331688001" />
+</object>
+
+<!-- 
+This script tag will cause the Brightcove Players defined above it to be created as soon
+as the line is read by the browser. If you wish to have the player instantiated only after
+the rest of the HTML is processed and the page load is complete, remove the line.
+-->
+<script type="text/javascript">brightcove.createExperiences();</script>
+
+<!-- End of Brightcove Player -->
+</body>
+</html>
--- a/src/aac-stream.js
+++ b/src/aac-stream.js
@ -53,8 +53,8 @@ window.videojs.hls.AacStream = function() {
    }
  };

-  // (pData:ByteArray, o:int = 0, l:int = 0):void
-  this.writeBytes = function(pData, o, l) {
+  // (data:ByteArray, o:int = 0, l:int = 0):void
+  this.writeBytes = function(data, o, l) {
    var
      e, // :int
      newExtraData, // :uint
@ -64,11 +64,11 @@ window.videojs.hls.AacStream = function() {
    o = o || 0;
    l = l || 0;

-    // Do not allow more that 'pes_length' bytes to be written
+    // Do not allow more than 'pes_length' bytes to be written
    l = (pes_length < l ? pes_length : l);
    pes_length -= l;
    e = o + l;
-    while(o < e) {
+    while (o < e) {
      switch (state) {
      default:
        state = 0;
@ -77,8 +77,8 @@ window.videojs.hls.AacStream = function() {
        if (o >= e) {
          return;
        }
-        if (0xFF !== pData[o]) {
-          console.log("Error no ATDS header found");
+        if (0xFF !== data[o]) {
+          console.assert(false, 'Error no ATDS header found');
          o += 1;
          state = 0;
          return;
@ -91,14 +91,14 @@ window.videojs.hls.AacStream = function() {
        if (o >= e) {
          return;
        }
-        if (0xF0 !== (pData[o] & 0xF0)) {
-          console.log("Error no ATDS header found");
+        if (0xF0 !== (data[o] & 0xF0)) {
+          console.assert(false, 'Error no ATDS header found');
          o +=1;
          state = 0;
          return;
        }

-        adtsProtectionAbsent = !!(pData[o] & 0x01);
+        adtsProtectionAbsent = !!(data[o] & 0x01);

        o += 1;
        state = 2;
@ -107,9 +107,9 @@ window.videojs.hls.AacStream = function() {
        if (o >= e) {
          return;
        }
-        adtsObjectType = ((pData[o] & 0xC0) >>> 6) + 1;
-        adtsSampleingIndex = ((pData[o] & 0x3C) >>> 2);
-        adtsChanelConfig = ((pData[o] & 0x01) << 2);
+        adtsObjectType = ((data[o] & 0xC0) >>> 6) + 1;
+        adtsSampleingIndex = ((data[o] & 0x3C) >>> 2);
+        adtsChanelConfig = ((data[o] & 0x01) << 2);

        o += 1;
        state = 3;
@ -118,8 +118,8 @@ window.videojs.hls.AacStream = function() {
        if (o >= e) {
          return;
        }
-        adtsChanelConfig |= ((pData[o] & 0xC0) >>> 6);
-        adtsFrameSize = ((pData[o] & 0x03) << 11);
+        adtsChanelConfig |= ((data[o] & 0xC0) >>> 6);
+        adtsFrameSize = ((data[o] & 0x03) << 11);

        o += 1;
        state = 4;
@ -128,7 +128,7 @@ window.videojs.hls.AacStream = function() {
        if (o >= e) {
          return;
        }
-        adtsFrameSize |= (pData[o] << 3);
+        adtsFrameSize |= (data[o] << 3);

        o += 1;
        state = 5;
@ -137,7 +137,7 @@ window.videojs.hls.AacStream = function() {
        if(o >= e) {
          return;
        }
-        adtsFrameSize |= ((pData[o] & 0xE0) >>> 5);
+        adtsFrameSize |= ((data[o] & 0xE0) >>> 5);
        adtsFrameSize -= (adtsProtectionAbsent ? 7 : 9);

        o += 1;
@ -147,7 +147,7 @@ window.videojs.hls.AacStream = function() {
        if (o >= e) {
          return;
        }
-        adtsSampleCount = ((pData[o] & 0x03) + 1) * 1024;
+        adtsSampleCount = ((data[o] & 0x03) + 1) * 1024;
        adtsDuration = (adtsSampleCount * 1000) / adtsSampleingRates[adtsSampleingIndex];

        newExtraData = (adtsObjectType << 11) |
@ -176,6 +176,7 @@ window.videojs.hls.AacStream = function() {
          aacFrame.pts = next_pts; 
          aacFrame.view.setUint16(aacFrame.position, newExtraData);
          aacFrame.position += 2;
+          aacFrame.length = Math.max(aacFrame.length, aacFrame.position);

          this.tags.push(aacFrame);
        }
@ -204,7 +205,7 @@ window.videojs.hls.AacStream = function() {
            return;
          }
          bytesToCopy = (e - o) < adtsFrameSize ? (e - o) : adtsFrameSize;
-          aacFrame.writeBytes( pData, o, bytesToCopy );
+          aacFrame.writeBytes(data, o, bytesToCopy);
          o += bytesToCopy;
          adtsFrameSize -= bytesToCopy;
        }
--- a/src/bin-utils.js
+++ b/src/bin-utils.js
@ -0,0 +1,29 @@
+(function(window) {
+  var module = {
+    hexDump: function(data) {
+      var
+        bytes = Array.prototype.slice.call(data),
+        step = 16,
+        hex,
+        ascii;
+      for (var j = 0; j < bytes.length / step; j++) {
+        hex = bytes.slice(j * step, j * step + step).map(function(e) {
+          var value = e.toString(16);
+          return "00".substring(0, 2 - value.length) + value;
+        }).join(' ');
+        ascii = bytes.slice(j * step, j * step + step).map(function(e) {
+          if (e > 32 && e < 125) {
+            return String.fromCharCode(e);
+          }
+          return '.';
+        }).join('');
+        return hex + '  ' + ascii;
+      }
+    },
+    tagDump: function(tag) {
+      return module.hexDump(tag.bytes);
+    }
+  };
+
+  window.videojs.hls.utils = module;
+})(this);
--- a/src/flv-tag.js
+++ b/src/flv-tag.js
@ -136,7 +136,8 @@ hls.FlvTag = function(type, extraData) {
    this.view.setUint8(this.position, 0x00);
    this.position++;
    this.view.setFloat64(this.position, val);
-    this.position += 2;
+    this.position += 8;
+    this.length = Math.max(this.length, this.position);
    ++adHoc;
  };

@ -154,6 +155,7 @@ hls.FlvTag = function(type, extraData) {
    this.position++;
    this.view.setUint8(this.position, val ? 0x01 : 0x00);
    this.position++;
+    this.length = Math.max(this.length, this.position);
    ++adHoc;
  };

@ -176,7 +178,7 @@ hls.FlvTag = function(type, extraData) {
      break;

    case hls.FlvTag.AUDIO_TAG:
-      this.bytes[11] = 0xAF;
+      this.bytes[11] = 0xAF; // 44 kHz, 16-bit stereo
      this.bytes[12] = extraData ? 0x00 : 0x01;
      break;

@ -191,26 +193,27 @@ hls.FlvTag = function(type, extraData) {
                      0x74, 0x61, 0x44, 0x61,
                      0x74, 0x61], this.position);
      this.position += 10;
-      this.view.setUint8(this.position,
-                         this.bytes[this.position] | 0x08); // Array type
+      this.bytes[this.position] = 0x08; // Array type
      this.position++;
      this.view.setUint32(this.position, adHoc);
      this.position = this.length;
-      this.view.setUint32(this.position, 0x09); // End Data Tag
-      this.position += 4;
+      this.bytes.set([0, 0, 9], this.position);
+      this.position += 3;
+      // this.view.setUint32(this.position, 0x09); // End Data Tag
+      // this.position += 4;
      this.length = this.position;
      break;
    }

    len = this.length - 11;

-    this.bytes[ 1] = ( len & 0x00FF0000 ) >>> 16;
-    this.bytes[ 2] = ( len & 0x0000FF00 ) >>>  8;
-    this.bytes[ 3] = ( len & 0x000000FF ) >>>  0;
-    this.bytes[ 4] = ( this.pts & 0x00FF0000 ) >>> 16;
-    this.bytes[ 5] = ( this.pts & 0x0000FF00 ) >>>  8;
-    this.bytes[ 6] = ( this.pts & 0x000000FF ) >>>  0;
-    this.bytes[ 7] = ( this.pts & 0xFF000000 ) >>> 24;
+    this.bytes[ 1] = (len & 0x00FF0000) >>> 16;
+    this.bytes[ 2] = (len & 0x0000FF00) >>>  8;
+    this.bytes[ 3] = (len & 0x000000FF) >>>  0;
+    this.bytes[ 4] = (this.pts & 0x00FF0000) >>> 16;
+    this.bytes[ 5] = (this.pts & 0x0000FF00) >>>  8;
+    this.bytes[ 6] = (this.pts & 0x000000FF) >>>  0;
+    this.bytes[ 7] = (this.pts & 0xFF000000) >>> 24;
    this.bytes[ 8] = 0;
    this.bytes[ 9] = 0;
    this.bytes[10] = 0;
@ -218,6 +221,11 @@ hls.FlvTag = function(type, extraData) {
    this.view.setUint32(this.length, this.length);
    this.length += 4;
    this.position += 4;
+
+    // trim down the byte buffer to what is actually being used
+    this.bytes = this.bytes.subarray(0, this.length);
+    console.assert(this.bytes.byteLength === this.length);
+
    return this;
  };
 };
--- a/src/h264-stream.js
+++ b/src/h264-stream.js
@ -17,7 +17,6 @@
      this.pps = []; // :Array

      this.addSPS = function(size) { // :ByteArray
-        console.log('come on, you fucker');
        console.assert(size > 0);
        var tmp = new Uint8Array(size); // :ByteArray
        this.sps.push(tmp);
@ -261,8 +260,6 @@

  window.videojs.hls.H264Stream = function() {
    var
-      tags = [],
-
      next_pts, // :uint;
      next_dts, // :uint;
      pts_delta = -1, // :int
@ -307,12 +304,12 @@

        if (h264Frame.keyFrame) {
          // Push extra data on every IDR frame in case we did a stream change + seek
-          tags.push(oldExtraData.metaDataTag(h264Frame.pts));
-          tags.push(oldExtraData.extraDataTag(h264Frame.pts));
+          this.tags.push(oldExtraData.metaDataTag(h264Frame.pts));
+          this.tags.push(oldExtraData.extraDataTag(h264Frame.pts));
        }

        h264Frame.endNalUnit();
-        tags.push(h264Frame);
+        this.tags.push(h264Frame);
      }

      h264Frame = null;
--- a/src/segment-parser.js
+++ b/src/segment-parser.js
@ -386,5 +386,14 @@

      return true;
    };
+
+    self.stats = {
+      h264Tags: function() {
+        return h264Stream.tags.length;
+      },
+      aacTags: function() {
+        return aacStream.tags.length;
+      }
+    };
  };
 })(this);
--- a/test/bc0.ts
+++ b/test/bc0.ts
--- a/test/bipbop1.flv
+++ b/test/bipbop1.flv
--- a/test/bipbop1.ts
+++ b/test/bipbop1.ts
--- a/test/tsSegment-bc.js
+++ b/test/tsSegment-bc.js
--- a/test/video-js-hls.html
+++ b/test/video-js-hls.html
@ -45,7 +45,10 @@
  <script src="../src/aac-stream.js"></script>
  <script src="../src/segment-parser.js"></script>
  <!-- an example MPEG2-TS segment -->
-  <script src="tsSegment.js"></script>
+  <!-- <script src="tsSegment.js"></script> -->
+  <script src="tsSegment-bc.js"></script>
+
+ <script src="../src/bin-utils.js"></script>

  <script src="video-js-hls_test.js"></script>
 </head>
--- a/test/video-js-hls_test.js
+++ b/test/video-js-hls_test.js
@ -25,7 +25,13 @@
    expectedHeader = [
      0x46, 0x4c, 0x56, 0x01, 0x05, 0x00, 0x00, 0x00,
      0x09, 0x00, 0x00, 0x00, 0x00
-    ];
+    ],
+    testAudioTag,
+    testVideoTag,
+    testScriptTag,
+    asciiFromBytes,
+    testScriptString,
+    testScriptEcmaArray;

  module('environment');

@ -44,17 +50,175 @@
    var header = Array.prototype.slice.call(parser.getFlvHeader());
    ok(header, 'the header is truthy');
    equal(9 + 4, header.length, 'the header length is correct');
-    equal(header[0], 'F'.charCodeAt(0), 'the signature is correct');
-    equal(header[1], 'L'.charCodeAt(0), 'the signature is correct');
-    equal(header[2], 'V'.charCodeAt(0), 'the signature is correct');
+    equal(header[0], 'F'.charCodeAt(0), 'the first character is "F"');
+    equal(header[1], 'L'.charCodeAt(0), 'the second character is "L"');
+    equal(header[2], 'V'.charCodeAt(0), 'the third character is "V"');

    deepEqual(expectedHeader, header, 'the rest of the header is correct');
  });

  test('parses the first bipbop segment', function() {
-    var tag, bytes;
-    parser.parseSegmentBinaryData(window.testSegment);
+    var tag, bytes, i;
+    parser.parseSegmentBinaryData(window.bcSegment);
    
    ok(parser.tagsAvailable(), 'tags are available');
+
+    console.log('h264 tags:', parser.stats.h264Tags(),
+                'aac tags:', parser.stats.aacTags());
+
+    console.log(videojs.hls.utils.hexDump(parser.getFlvHeader()));
+    for (i = 0; i < 4; ++i) {
+      parser.getNextTag();
+    }
+    console.log(videojs.hls.utils.tagDump(parser.getNextTag()));
+    console.log('bad tag:');
+    for (i = 0; i < 3; ++i) {
+      console.log(videojs.hls.utils.tagDump(parser.getNextTag()));
+    }
+  });
+
+  testAudioTag = function(tag) {
+    var
+      byte = tag.bytes[11],
+      format = (byte & 0xF0) >>> 4,
+      soundRate = byte & 0x03,
+      soundSize = (byte & 0x2) >>> 1,
+      soundType = byte & 0x1,
+      aacPacketType = tag.bytes[12];
+
+    equal(10, format, 'the audio format is aac');
+    equal(3, soundRate, 'the sound rate is 44kHhz');
+    equal(1, soundSize, 'the sound size is 16-bit samples');
+    equal(1, soundType, 'the sound type is stereo');
+
+    ok(aacPacketType === 0 || aacPacketType === 1, 'aac packets should have a valid type');
+  };
+
+  testVideoTag = function(tag) {
+    var
+      byte = tag.bytes[11],
+      frameType = (byte & 0xF0) >>> 4,
+      codecId = byte & 0x0F,
+      packetType = tag.bytes[12],
+      compositionTime = (tag.view.getInt32(13) & 0xFFFFFF00) >> 8;
+
+    console.log(frameType);
+
+    // XXX: I'm not sure that frame types 3-5 are invalid
+    ok(frameType === 1 || frameType === 2,
+       'the frame type should be valid');
+
+    equal(7, codecId, 'the codec ID is AVC for h264');
+    ok(packetType <=2 && packetType >= 0, 'the packet type is within [0, 2]');
+    if (packetType !== 1) {
+      equal(0,
+            compositionTime,
+            'the composition time is zero for non-NALU packets');
+    }
+    
+    // TODO: the rest of the bytes are an NLU unit
+  };
+
+  asciiFromBytes = function(bytes) {
+    var
+      string = [],
+      i = bytes.byteLength;
+
+    while (i--) {
+      string[i] = String.fromCharCode(bytes[i]);
+    }
+    return string.join('');
+  };
+
+  testScriptString = function(tag, offset, expected) {
+    var type = tag.bytes[offset],
+        stringLength = tag.view.getUint16(offset + 1),
+        string,
+        i = expected.length;
+
+    equal(2, type, 'the script element is of string type');
+    equal(stringLength, expected.length, 'the script string length is correct');
+    string = asciiFromBytes(tag.bytes.subarray(offset + 3,
+                                               offset + 3 + stringLength));
+    equal(expected, string, 'the string value is "' + expected + '"');
+  };
+
+  testScriptEcmaArray = function(tag, start) {
+    var
+      numItems = tag.view.getUint32(start),
+      i = numItems,
+      offset = start + 4,
+      length,
+      type;
+
+    while (i--) {
+      length = tag.view.getUint16(offset);
+
+      // advance offset to the property value
+      offset += 2 + length;
+
+      type = tag.bytes[offset];
+      ok(type === 1 || type === 0,
+         'the ecma array property value type is number or boolean');
+      offset++;
+      if (type) {
+        // boolean
+        ok(tag.bytes[offset] === 0 || tag.bytes[offset] === 1,
+           'the script boolean value is 0 or 1');
+        offset++;
+      } else {
+        // number
+        offset += 8;
+      }
+    }
+    equal(tag.bytes[offset], 0, 'the property array terminator is valid');
+    equal(tag.bytes[offset + 1], 0, 'the property array terminator is valid');
+    equal(tag.bytes[offset + 2], 9, 'the property array terminator is valid');
+  };
+
+  testScriptTag = function(tag) {
+    testScriptString(tag, 11, 'onMetaData');
+
+    // the onMetaData object is stored as an 'ecma array', an array with non-
+    // integer indices (i.e. a dictionary or hash-map).
+    equal(8, tag.bytes[24], 'onMetaData is of ecma array type');
+    testScriptEcmaArray(tag, 25);
+  };
+
+  test('the flv tags are well-formed', function() {
+    var
+      tag,
+      byte,
+      type,
+      lastTime = 0;
+    parser.parseSegmentBinaryData(window.bcSegment);
+
+    while (parser.tagsAvailable()) {
+      tag = parser.getNextTag();
+      type = tag.bytes[0];
+
+      // generic flv headers
+      ok(type === 8 || type === 9 || type === 18,
+         'the type field specifies audio, video or script');
+      
+      byte = (tag.view.getUint32(1) & 0xFFFFFF00) >>> 8;
+      equal(tag.bytes.byteLength - 11 - 4, byte, 'the size field is correct');
+
+      byte = tag.view.getUint32(5) & 0xFFFFFF00;
+      ok(byte >= lastTime, 'the timestamp for the tag is greater than zero');
+      lastTime = byte;
+
+      // tag type-specific headers
+      ({
+        8: testAudioTag,
+        9: testVideoTag,
+        18: testScriptTag
+      })[type](tag);
+
+      // previous tag size
+      equal(tag.bytes.byteLength - 4,
+            tag.view.getUint32(tag.bytes.byteLength - 4),
+            'the size of the previous tag is correct');
+    }
  });
 })(this);