Skip to content

Commit

Permalink
Merge pull request #6248 from ant-media/seiUserData
Browse files Browse the repository at this point in the history 
Support SEI in HLS
  • Loading branch information
@mekya
mekya committed May 15, 2024
2 parents 96e8363 + cd457c3 commit 6a0e440
Show file tree
Hide file tree
Showing 10 changed files with 391 additions and 255 deletions.
14 changes: 14 additions & 0 deletions src/main/java/io/antmedia/AppSettings.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2042,6 +2042,12 @@ public boolean isWriteStatsToDatastore() {
*/
@Value("${id3TagEnabled:false}")
private boolean id3TagEnabled = false;

/**
* Enables the SEI data for HLS
*/
@Value("${seiEnabled:false}")
private boolean seiEnabled = false;

/**
* Ant Media Server can get the audio level from incoming RTP Header in WebRTC streaming and send to the viewers.
Expand Down Expand Up @@ -3584,6 +3590,14 @@ public void setId3TagEnabled(boolean id3TagEnabled) {
this.id3TagEnabled = id3TagEnabled;
}

public boolean isSeiEnabled() {
return seiEnabled;
}

public void setSeiEnabled(boolean seiEnabled) {
this.seiEnabled = seiEnabled;
}

public boolean isSendAudioLevelToViewers() {
return sendAudioLevelToViewers;
}
Expand Down
75 changes: 62 additions & 13 deletions src/main/java/io/antmedia/muxer/HLSMuxer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,20 +2,18 @@

import static org.bytedeco.ffmpeg.global.avcodec.*;
import static org.bytedeco.ffmpeg.global.avformat.avformat_alloc_output_context2;
import static org.bytedeco.ffmpeg.global.avutil.AVMEDIA_TYPE_DATA;
import static org.bytedeco.ffmpeg.global.avutil.av_rescale_q;
import static org.bytedeco.ffmpeg.global.avutil.*;
import static org.bytedeco.ffmpeg.global.avutil.AV_OPT_SEARCH_CHILDREN;

import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.Files;

import org.apache.commons.lang3.StringUtils;
import org.bytedeco.ffmpeg.avcodec.AVCodec;
import org.bytedeco.ffmpeg.avcodec.AVCodecContext;
import org.bytedeco.ffmpeg.avcodec.AVCodecParameters;
import org.bytedeco.ffmpeg.avcodec.AVPacket;
import org.bytedeco.ffmpeg.avcodec.*;
import org.bytedeco.ffmpeg.avformat.AVFormatContext;
import org.bytedeco.ffmpeg.avformat.AVStream;
import org.bytedeco.ffmpeg.avutil.AVRational;
import org.bytedeco.ffmpeg.global.avcodec;
import org.bytedeco.javacpp.BytePointer;
Expand All @@ -28,6 +26,10 @@

public class HLSMuxer extends Muxer {

public static final String SEI_USER_DATA = "sei_user_data";

private static final String SEI_UUID = "086f3693-b7b3-4f2c-9653-21492feee5b8+";

private static final String SEGMENT_SUFFIX_TS = "%0"+SEGMENT_INDEX_LENGTH+"d.ts";

protected static Logger logger = LoggerFactory.getLogger(HLSMuxer.class);
Expand Down Expand Up @@ -68,6 +70,9 @@ public class HLSMuxer extends Muxer {

private boolean id3Enabled = false;

private boolean seiEnabled = false;


public HLSMuxer(Vertx vertx, StorageClient storageClient, String s3StreamsFolderPath, int uploadExtensionsToS3, String httpEndpoint, boolean addDateTimeToResourceName) {
super(vertx);
this.storageClient = storageClient;
Expand Down Expand Up @@ -351,19 +356,58 @@ public synchronized boolean addStream(AVCodec codec, AVCodecContext codecContext
logger.error("Cannot get codec parameters for {}", streamId);
}

//call super directly because no need to add bit stream filter
//call super directly because no need to add bit stream filter
return super.addStream(codecParameter, codecContext.time_base(), streamIndex);
}

@Override
public synchronized boolean addVideoStream(int width, int height, AVRational timebase, int codecId, int streamIndex,
boolean isAVC, AVCodecParameters codecpar) {

boolean result = super.addVideoStream(width, height, timebase, codecId, streamIndex, isAVC, codecpar);
if (result && seiEnabled)
{
AVStream outStream = getOutputFormatContext().streams(inputOutputStreamIndexMap.get(streamIndex));

setBitstreamFilter("h264_metadata");

AVBSFContext avbsfContext = initVideoBitstreamFilter(getBitStreamFilter(), outStream.codecpar(), inputTimeBaseMap.get(streamIndex));

if (avbsfContext != null) {
int ret = avcodec_parameters_copy(outStream.codecpar(), avbsfContext.par_out());
result = ret == 0;
}

setSeiData("initial sei data");

logger.info("Adding video stream index:{} for stream:", streamIndex);
}

return result;
}

public void setSeiData(String data) {
int ret = av_opt_set(bsfFilterContextList.get(0).priv_data(), SEI_USER_DATA, SEI_UUID+data, AV_OPT_SEARCH_CHILDREN);
logError(ret, "Cannot set sei_user_data for {} and error is {}", streamId);


ret = av_bsf_init(bsfFilterContextList.get(0));
logError(ret, "Cannot update sei_user_data for {} and error is {}", streamId);

}

public static void logError(int ret, String message, String streamId) {
if (ret < 0 && logger.isErrorEnabled()) {
logger.error(message, streamId, Muxer.getErrorDefinition(ret));
}
}


@Override
public synchronized boolean addStream(AVCodecParameters codecParameters, AVRational timebase, int streamIndex)
{
bsfVideoName = "h264_mp4toannexb";

boolean ret = super.addStream(codecParameters, timebase, streamIndex);

return ret;
setBitstreamFilter("h264_mp4toannexb");
return super.addStream(codecParameters, timebase, streamIndex);
}

public boolean addID3Stream() {
Expand Down Expand Up @@ -419,7 +463,12 @@ public String getSegmentFilename() {
public void setId3Enabled(boolean id3Enabled) {
this.id3Enabled = id3Enabled;
}


public void setSeiEnabled(boolean seiEnabled) {
this.seiEnabled = seiEnabled;
}


@Override
protected synchronized void clearResource() {
super.clearResource();
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/io/antmedia/muxer/Mp4Muxer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,8 +143,8 @@ public synchronized boolean addVideoStream(int width, int height, AVRational tim
*/
@Override
protected boolean prepareAudioOutStream(AVStream inStream, AVStream outStream) {
if (bsfVideoName != null) {
AVBitStreamFilter adtsToAscBsf = av_bsf_get_by_name(this.bsfVideoName);
if (getBitStreamFilter() != null) {
AVBitStreamFilter adtsToAscBsf = av_bsf_get_by_name(this.getBitStreamFilter());
bsfContext = new AVBSFContext(null);

int ret = av_bsf_alloc(adtsToAscBsf, bsfContext);
Expand Down
11 changes: 11 additions & 0 deletions src/main/java/io/antmedia/muxer/MuxAdaptor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,16 @@ public boolean addID3Data(String data) {
return false;
}

public boolean addSEIData(String data) {
for (Muxer muxer : muxerList) {
if(muxer instanceof HLSMuxer) {
((HLSMuxer)muxer).setSeiData(data);
return true;
}
}
return false;
}

public static class PacketTime {
public final long packetTimeMs;
public final long systemTimeMs;
Expand Down Expand Up @@ -433,6 +443,7 @@ public boolean init(IScope scope, String streamId, boolean isAppend) {
hlsMuxer.setHlsParameters( hlsListSize, hlsTime, hlsPlayListType, getAppSettings().getHlsflags(), getAppSettings().getHlsEncryptionKeyInfoFile(), getAppSettings().getHlsSegmentType());
hlsMuxer.setDeleteFileOnExit(deleteHLSFilesOnExit);
hlsMuxer.setId3Enabled(appSettings.isId3TagEnabled());
hlsMuxer.setSeiEnabled(appSettings.isSeiEnabled());
addMuxer(hlsMuxer);
logger.info("adding HLS Muxer for {}", streamId);
}
Expand Down
84 changes: 35 additions & 49 deletions src/main/java/io/antmedia/muxer/Muxer.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,8 @@
import static org.bytedeco.ffmpeg.global.avformat.avformat_open_input;
import static org.bytedeco.ffmpeg.global.avformat.avformat_write_header;
import static org.bytedeco.ffmpeg.global.avformat.avio_closep;
import static org.bytedeco.ffmpeg.global.avutil.AVMEDIA_TYPE_AUDIO;
import static org.bytedeco.ffmpeg.global.avutil.AVMEDIA_TYPE_DATA;
import static org.bytedeco.ffmpeg.global.avutil.AVMEDIA_TYPE_VIDEO;
import static org.bytedeco.ffmpeg.global.avutil.AV_NOPTS_VALUE;
import static org.bytedeco.ffmpeg.global.avutil.AV_PIX_FMT_YUV420P;
import static org.bytedeco.ffmpeg.global.avutil.AV_ROUND_NEAR_INF;
import static org.bytedeco.ffmpeg.global.avutil.AV_ROUND_PASS_MINMAX;
import static org.bytedeco.ffmpeg.global.avutil.av_dict_free;
import static org.bytedeco.ffmpeg.global.avutil.av_dict_set;
import static org.bytedeco.ffmpeg.global.avutil.av_rescale_q;
import static org.bytedeco.ffmpeg.global.avutil.av_rescale_q_rnd;
import static org.bytedeco.ffmpeg.global.avutil.av_strerror;
import static org.bytedeco.ffmpeg.global.avutil.*;
import static org.bytedeco.ffmpeg.global.avutil.AV_OPT_SEARCH_CHILDREN;

import java.io.File;
import java.io.IOException;
Expand All @@ -38,8 +28,6 @@
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import io.antmedia.FFmpegUtilities;
import io.antmedia.rest.RestServiceBase;
Expand Down Expand Up @@ -133,14 +121,14 @@ public abstract class Muxer {
/**
* Bitstream filter name that will be applied to packets
*/
protected String bsfVideoName = null;
protected List<String> bsfVideoNames = new ArrayList<>();

protected String streamId = null;

protected Map<Integer, AVRational> inputTimeBaseMap = new ConcurrentHashMap<>();


protected AVBSFContext videoBsfFilterContext = null;
protected List<AVBSFContext> bsfFilterContextList = new ArrayList<>();

protected int videoWidth;
protected int videoHeight;
Expand Down Expand Up @@ -464,10 +452,10 @@ protected synchronized void clearResource() {
audioPkt = null;
}

if (videoBsfFilterContext != null) {
for (AVBSFContext videoBsfFilterContext: bsfFilterContextList) {
av_bsf_free(videoBsfFilterContext);
videoBsfFilterContext = null;
}
bsfFilterContextList.clear();

/* close output */
if (outputFormatContext != null &&
Expand Down Expand Up @@ -532,7 +520,7 @@ public void logPacketIssue(String format, Object... arguments) {
/**
* Write packets to the output. This function is used in transcoding.
* Previously, It's the replacement of {link {@link #writePacket(AVPacket)}
* @param avpacket
* @param pkt
* @param codecContext
*/
public synchronized void writePacket(AVPacket pkt, AVCodecContext codecContext) {
Expand Down Expand Up @@ -569,11 +557,15 @@ public ByteBuffer getPacketBufferWithExtradata(byte[] extradata, AVPacket pkt){


public void setBitstreamFilter(String bsfName) {
this.bsfVideoName = bsfName;
bsfVideoNames.add(bsfName);
}

public String getBitStreamFilter() {
return bsfVideoName;
if(!bsfVideoNames.isEmpty())
{
return bsfVideoNames.get(0);
}
return null;
}

public File getFile() {
Expand Down Expand Up @@ -836,16 +828,18 @@ && isCodecSupported(codecParameters.codec_id()) &&
//if it's not running add to the list
registeredStreamIndexList.add(streamIndex);

if (bsfVideoName != null && codecParameters.codec_type() == AVMEDIA_TYPE_VIDEO)
if (codecParameters.codec_type() == AVMEDIA_TYPE_VIDEO)
{
AVBSFContext videoBitstreamFilter = initVideoBitstreamFilter(codecParameters, timebase);
if (videoBitstreamFilter != null)
{
codecParameters = videoBitstreamFilter.par_out();
timebase = videoBitstreamFilter.time_base_out();
for (String bsfVideoName: bsfVideoNames) {
AVBSFContext videoBitstreamFilter = initVideoBitstreamFilter(bsfVideoName, codecParameters, timebase);
if (videoBitstreamFilter != null)
{
codecParameters = videoBitstreamFilter.par_out();
timebase = videoBitstreamFilter.time_base_out();
}
}

}

String codecType = "audio";
if (codecParameters.codec_type() == AVMEDIA_TYPE_VIDEO)
{
Expand Down Expand Up @@ -886,9 +880,9 @@ else if (codecParameters.codec_type() == AVMEDIA_TYPE_DATA)
return result;
}

public AVBSFContext initVideoBitstreamFilter(AVCodecParameters codecParameters, AVRational timebase) {
public AVBSFContext initVideoBitstreamFilter(String bsfVideoName, AVCodecParameters codecParameters, AVRational timebase) {
AVBitStreamFilter bsfilter = av_bsf_get_by_name(bsfVideoName);
videoBsfFilterContext = new AVBSFContext(null);
AVBSFContext videoBsfFilterContext = new AVBSFContext(null);
int ret = av_bsf_alloc(bsfilter, videoBsfFilterContext);

if (ret < 0) {
Expand All @@ -909,6 +903,8 @@ public AVBSFContext initVideoBitstreamFilter(AVCodecParameters codecParameters,
return null;
}

bsfFilterContextList.add(videoBsfFilterContext);

return videoBsfFilterContext;
}

Expand Down Expand Up @@ -1180,32 +1176,22 @@ public void addExtradataIfRequired(AVPacket pkt, boolean isKeyFrame)
protected void writeVideoFrame(AVPacket pkt, AVFormatContext context) {
int ret;


if (videoBsfFilterContext != null)
for(AVBSFContext videoBsfFilterContext : bsfFilterContextList)
{
ret = av_bsf_send_packet(videoBsfFilterContext, pkt);
if (ret < 0) {
logger.warn("Cannot send packet to bit stream filter for stream:{}", streamId);
return;
}
while (av_bsf_receive_packet(videoBsfFilterContext, pkt) == 0)
{
logger.trace("write video packet pts:{} dts:{}", pkt.pts(), pkt.dts());
ret = av_write_frame(context, tmpPacket);
if (ret < 0 && logger.isWarnEnabled()) {
logger.warn("cannot write video frame to muxer({}) av_bsf_receive_packet. Error is {} ", file.getName(), getErrorDefinition(ret));
}
}
ret = av_bsf_receive_packet(videoBsfFilterContext, pkt);
}
else
{
logger.trace("write video packet pts:{} dts:{}", pkt.pts(), pkt.dts());
ret = av_write_frame(context, pkt);
if (ret < 0 && logger.isWarnEnabled()) {
//TODO: this is written for some muxers like HLS because normalized video time is coming from WebRTC
//WebRTCVideoForwarder#getVideoTime. Fix this problem when upgrading the webrtc stack
logger.warn("cannot write video frame to muxer({}). Pts: {} dts:{} Error is {} ", file.getName(), pkt.pts(), pkt.dts(), getErrorDefinition(ret));
}

logger.trace("write video packet pts:{} dts:{}", pkt.pts(), pkt.dts());
ret = av_write_frame(context, pkt);
if (ret < 0 && logger.isWarnEnabled()) {
//TODO: this is written for some muxers like HLS because normalized video time is coming from WebRTC
//WebRTCVideoForwarder#getVideoTime. Fix this problem when upgrading the webrtc stack
logger.warn("cannot write video frame to muxer({}). Pts: {} dts:{} Error is {} ", file.getName(), pkt.pts(), pkt.dts(), getErrorDefinition(ret));
}
}

Expand Down
Loading

0 comments on commit 6a0e440

Please sign in to comment.