Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

音频编码bug #252

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
+70 −38
Open

音频编码bug #252

Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
fd7802b
音频编码bug
february29 Sep 29, 2017
46801e2
音频编码注释
february29 Oct 9, 2017
1719505
Update LFHardwareVideoEncoder.m
february29 Oct 9, 2017
cb028c6
视频编码注释
february29 Oct 9, 2017
6241f40
视频编码注释
february29 Oct 9, 2017
7228318
Revert "音频编码注释"
february29 Oct 9, 2017
5cf384f
Revert "Revert "音频编码注释""
february29 Oct 9, 2017
e12868c
视频编码注释完善
february29 Oct 9, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 36 additions & 31 deletions LFLiveKit/coder/LFHardwareAudioEncoder.m
View file
Original file line number Diff line number Diff line change
@@ -60,30 +60,32 @@ - (void)encodeAudioData:(nullable NSData*)audioData timeStamp:(uint64_t)timeStam
return;
}


if(leftLength + audioData.length >= self.configuration.bufferLength){
///<  发送
///<  发送 达到发送大小
NSInteger totalSize = leftLength + audioData.length;
NSInteger encodeCount = totalSize/self.configuration.bufferLength;
NSInteger encodeCount = totalSize/self.configuration.bufferLength;//audioData.length比较大，totalSize可能会是bufferLengthleft多倍
char *totalBuf = malloc(totalSize);
char *p = totalBuf;

memset(totalBuf, (int)totalSize, 0);
memcpy(totalBuf, leftBuf, leftLength);
memcpy(totalBuf + leftLength, audioData.bytes, audioData.length);
memset(totalBuf, 0, (int)totalSize);//初始化totalBuf
memcpy(totalBuf, leftBuf, leftLength);//copy上次剩余
memcpy(totalBuf + leftLength, audioData.bytes, audioData.length);////copy此次数据

for(NSInteger index = 0;index < encodeCount;index++){
//从p的位置开始发送，发送bufferLength大小
[self encodeBuffer:p timeStamp:timeStamp];
p += self.configuration.bufferLength;
}

leftLength = totalSize%self.configuration.bufferLength;
memset(leftBuf, 0, self.configuration.bufferLength);
memcpy(leftBuf, totalBuf + (totalSize -leftLength), leftLength);
leftLength = totalSize%self.configuration.bufferLength;//改变发送剩余数据大小leftLength
memset(leftBuf, 0, self.configuration.bufferLength);//将leftBuf起始位置到bufferLength置0
memcpy(leftBuf, totalBuf + (totalSize -leftLength), leftLength);//将totalBuf剩余的放倒leftBuf当中，下次发送

free(totalBuf);

}else{
///< 积累
///< 积累 未达到发送大小
memcpy(leftBuf+leftLength, audioData.bytes, audioData.length);
leftLength = leftLength + audioData.length;
}
@@ -96,35 +98,36 @@ - (void)encodeBuffer:(char*)buf timeStamp:(uint64_t)timeStamp{
inBuffer.mData = buf;
inBuffer.mDataByteSize = (UInt32)self.configuration.bufferLength;

// 初始化一个输入缓冲列表
AudioBufferList buffers;
buffers.mNumberBuffers = 1;
buffers.mNumberBuffers = 1;//只有一个inBuffer
buffers.mBuffers[0] = inBuffer;


// 初始化一个输出缓冲列表
// 初始化一个输出缓冲列表
AudioBufferList outBufferList;
outBufferList.mNumberBuffers = 1;
outBufferList.mNumberBuffers = 1;//只有一个outBuffer
outBufferList.mBuffers[0].mNumberChannels = inBuffer.mNumberChannels;
outBufferList.mBuffers[0].mDataByteSize = inBuffer.mDataByteSize; // 设置缓冲区大小
outBufferList.mBuffers[0].mData = aacBuf; // 设置AAC缓冲区
outBufferList.mBuffers[0].mData = aacBuf; // 设置AAC缓冲区 编码后数据存放的位置
UInt32 outputDataPacketSize = 1;
if (AudioConverterFillComplexBuffer(m_converter, inputDataProc, &buffers, &outputDataPacketSize, &outBufferList, NULL) != noErr) {
return;
}

//封装为LFAudioFrame方便以后推流使用
LFAudioFrame *audioFrame = [LFAudioFrame new];
audioFrame.timestamp = timeStamp;
audioFrame.data = [NSData dataWithBytes:aacBuf length:outBufferList.mBuffers[0].mDataByteSize];

char exeData[2];
char exeData[2];//flv编码音频头 44100 为0x12 0x10
exeData[0] = _configuration.asc[0];
exeData[1] = _configuration.asc[1];
audioFrame.audioInfo = [NSData dataWithBytes:exeData length:2];
if (self.aacDeleage && [self.aacDeleage respondsToSelector:@selector(audioEncoder:audioFrame:)]) {
[self.aacDeleage audioEncoder:self audioFrame:audioFrame];
[self.aacDeleage audioEncoder:self audioFrame:audioFrame];//调用编码完成后代理
}

if (self->enabledWriteVideoFile) {
if (self->enabledWriteVideoFile) {//写入本地文件中，debug时调用
NSData *adts = [self adtsData:_configuration.numberOfChannels rawDataLength:audioFrame.data.length];
fwrite(adts.bytes, 1, adts.length, self->fp);
fwrite(audioFrame.data.bytes, 1, audioFrame.data.length, self->fp);
@@ -141,45 +144,47 @@ - (BOOL)createAudioConvert { //根据输入样本初始化一个编码转换器
if (m_converter != nil) {
return TRUE;
}

// 音频输入描述
AudioStreamBasicDescription inputFormat = {0};
inputFormat.mSampleRate = _configuration.audioSampleRate;
inputFormat.mFormatID = kAudioFormatLinearPCM;
inputFormat.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagsNativeEndian | kAudioFormatFlagIsPacked;
inputFormat.mChannelsPerFrame = (UInt32)_configuration.numberOfChannels;
inputFormat.mFramesPerPacket = 1;
inputFormat.mBitsPerChannel = 16;
inputFormat.mBytesPerFrame = inputFormat.mBitsPerChannel / 8 * inputFormat.mChannelsPerFrame;
inputFormat.mBytesPerPacket = inputFormat.mBytesPerFrame * inputFormat.mFramesPerPacket;
inputFormat.mSampleRate = _configuration.audioSampleRate;// 采样率
inputFormat.mFormatID = kAudioFormatLinearPCM;// 数据格式
inputFormat.mFormatFlags = kAudioFormatFlagIsSignedInteger | kAudioFormatFlagsNativeEndian | kAudioFormatFlagIsPacked;// 格式标识
inputFormat.mChannelsPerFrame = (UInt32)_configuration.numberOfChannels;// 声道数
inputFormat.mFramesPerPacket = 1;//packet中包含的frame数目，无压缩时为1，可变比特率时，一个大点儿的固定值例如在ACC中1024。
inputFormat.mBitsPerChannel = 16;// 每个声道比特数，语音每采样点占用位数
inputFormat.mBytesPerFrame = inputFormat.mBitsPerChannel / 8 * inputFormat.mChannelsPerFrame;// 每帧多少字节
inputFormat.mBytesPerPacket = inputFormat.mBytesPerFrame * inputFormat.mFramesPerPacket;// 一个packet中的字节数目，如果时可变的packet则为0

// 音频输出描述
AudioStreamBasicDescription outputFormat; // 这里开始是输出音频格式
memset(&outputFormat, 0, sizeof(outputFormat));
memset(&outputFormat, 0, sizeof(outputFormat));// 初始化
outputFormat.mSampleRate = inputFormat.mSampleRate; // 采样率保持一致
outputFormat.mFormatID = kAudioFormatMPEG4AAC; // AAC编码 kAudioFormatMPEG4AAC kAudioFormatMPEG4AAC_HE_V2
outputFormat.mChannelsPerFrame = (UInt32)_configuration.numberOfChannels;;
outputFormat.mFramesPerPacket = 1024; // AAC一帧是1024个字节

const OSType subtype = kAudioFormatMPEG4AAC;
//两种编码方式 软编码 硬编码
AudioClassDescription requestedCodecs[2] = {
{
kAudioEncoderComponentType,
subtype,
kAppleSoftwareAudioCodecManufacturer
kAppleSoftwareAudioCodecManufacturer// 软编码
},
{
kAudioEncoderComponentType,
subtype,
kAppleHardwareAudioCodecManufacturer
kAppleHardwareAudioCodecManufacturer// 硬编码
}
};

OSStatus result = AudioConverterNewSpecific(&inputFormat, &outputFormat, 2, requestedCodecs, &m_converter);;
OSStatus result = AudioConverterNewSpecific(&inputFormat, &outputFormat, 2, requestedCodecs, &m_converter);//创建AudioConverter ：输入描述，输出描述，requestedCodecs的数量，支持的编码方式，AudioConverter
UInt32 outputBitrate = _configuration.audioBitrate;
UInt32 propSize = sizeof(outputBitrate);


if(result == noErr) {
result = AudioConverterSetProperty(m_converter, kAudioConverterEncodeBitRate, propSize, &outputBitrate);
result = AudioConverterSetProperty(m_converter, kAudioConverterEncodeBitRate, propSize, &outputBitrate);//设置码率
}

return YES;
41 changes: 34 additions & 7 deletions LFLiveKit/coder/LFHardwareVideoEncoder.m
View file
Original file line number Diff line number Diff line change
@@ -43,7 +43,7 @@ - (instancetype)initWithVideoStreamConfiguration:(LFLiveVideoConfiguration *)con
return self;
}

- (void)resetCompressionSession {
- (void)resetCompressionSession {//重置VTCompressionSessionRef
if (compressionSession) {
VTCompressionSessionCompleteFrames(compressionSession, kCMTimeInvalid);

@@ -52,22 +52,34 @@ - (void)resetCompressionSession {
compressionSession = NULL;
}

//创建VTCompressionSessionRef用于编码h.264 VideoCompressonOutputCallback为编码完成后回掉
OSStatus status = VTCompressionSessionCreate(NULL, _configuration.videoSize.width, _configuration.videoSize.height, kCMVideoCodecType_H264, NULL, NULL, NULL, VideoCompressonOutputCallback, (__bridge void *)self, &compressionSession);
if (status != noErr) {
return;
}

//设置VTCompressionSessionRef参数
_currentVideoBitRate = _configuration.videoBitRate;
// 设置最大关键帧间隔，即gop size
VTSessionSetProperty(compressionSession, kVTCompressionPropertyKey_MaxKeyFrameInterval, (__bridge CFTypeRef)@(_configuration.videoMaxKeyframeInterval));
//
VTSessionSetProperty(compressionSession, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, (__bridge CFTypeRef)@(_configuration.videoMaxKeyframeInterval/_configuration.videoFrameRate));
// 设置帧率，只用于初始化session，不是实际FPS
VTSessionSetProperty(compressionSession, kVTCompressionPropertyKey_ExpectedFrameRate, (__bridge CFTypeRef)@(_configuration.videoFrameRate));
// 设置编码码率(比特率)，如果不设置，默认将会以很低的码率编码，导致编码出来的视频很模糊
VTSessionSetProperty(compressionSession, kVTCompressionPropertyKey_AverageBitRate, (__bridge CFTypeRef)@(_configuration.videoBitRate));
NSArray *limit = @[@(_configuration.videoBitRate * 1.5/8), @(1)];
// 设置数据速率限制
NSArray *limit = @[@(_configuration.videoBitRate * 1.5/8), @(1)];// CFArray[CFNumber], [bytes, seconds, bytes, seconds...]
VTSessionSetProperty(compressionSession, kVTCompressionPropertyKey_DataRateLimits, (__bridge CFArrayRef)limit);
// 设置实时编码输出，降低编码延迟
VTSessionSetProperty(compressionSession, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue);
// h264 profile, 直播一般使用baseline，可减少由于b帧带来的延时
VTSessionSetProperty(compressionSession, kVTCompressionPropertyKey_ProfileLevel, kVTProfileLevel_H264_Main_AutoLevel);
// 设置允许帧重新排序
VTSessionSetProperty(compressionSession, kVTCompressionPropertyKey_AllowFrameReordering, kCFBooleanTrue);
// 设置编码类型h.264
VTSessionSetProperty(compressionSession, kVTCompressionPropertyKey_H264EntropyMode, kVTH264EntropyMode_CABAC);
// 准备编码
VTCompressionSessionPrepareToEncodeFrames(compressionSession);

}
@@ -99,6 +111,7 @@ - (void)dealloc {
- (void)encodeVideoData:(CVPixelBufferRef)pixelBuffer timeStamp:(uint64_t)timeStamp {
if(_isBackGround) return;
frameCount++;
// fps
CMTime presentationTimeStamp = CMTimeMake(frameCount, (int32_t)_configuration.videoFrameRate);
VTEncodeInfoFlags flags;
CMTime duration = CMTimeMake(1, (int32_t)_configuration.videoFrameRate);
@@ -109,6 +122,7 @@ - (void)encodeVideoData:(CVPixelBufferRef)pixelBuffer timeStamp:(uint64_t)timeSt
}
NSNumber *timeNumber = @(timeStamp);

//开始编码
OSStatus status = VTCompressionSessionEncodeFrame(compressionSession, pixelBuffer, presentationTimeStamp, duration, (__bridge CFDictionaryRef)properties, (__bridge_retained void *)timeNumber, &flags);
if(status != noErr){
[self resetCompressionSession];
@@ -141,6 +155,7 @@ static void VideoCompressonOutputCallback(void *VTref, void *VTFrameRef, OSStatu
CFDictionaryRef dic = (CFDictionaryRef)CFArrayGetValueAtIndex(array, 0);
if (!dic) return;

// 判断当前帧是否为关键帧
BOOL keyframe = !CFDictionaryContainsKey(dic, kCMSampleAttachmentKey_NotSync);
uint64_t timeStamp = [((__bridge_transfer NSNumber *)VTFrameRef) longLongValue];

@@ -149,21 +164,26 @@ static void VideoCompressonOutputCallback(void *VTref, void *VTFrameRef, OSStatu
return;
}

if (keyframe && !videoEncoder->sps) {
if (keyframe && !videoEncoder->sps) {//是关键帧，并且尚未设置sps（序列参数集）
CMFormatDescriptionRef format = CMSampleBufferGetFormatDescription(sampleBuffer);

size_t sparameterSetSize, sparameterSetCount;
//获取sps
const uint8_t *sparameterSet;
OSStatus statusCode = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(format, 0, &sparameterSet, &sparameterSetSize, &sparameterSetCount, 0);
if (statusCode == noErr) {
size_t pparameterSetSize, pparameterSetCount;
//获取pps
const uint8_t *pparameterSet;
OSStatus statusCode = CMVideoFormatDescriptionGetH264ParameterSetAtIndex(format, 1, &pparameterSet, &pparameterSetSize, &pparameterSetCount, 0);
if (statusCode == noErr) {
videoEncoder->sps = [NSData dataWithBytes:sparameterSet length:sparameterSetSize];
videoEncoder->pps = [NSData dataWithBytes:pparameterSet length:pparameterSetSize];
videoEncoder->sps = [NSData dataWithBytes:sparameterSet length:sparameterSetSize];//设置sps
videoEncoder->pps = [NSData dataWithBytes:pparameterSet length:pparameterSetSize];//这只pps
//数据处理时，sps pps 数据可以作为一个普通h264帧，放在h264视频流的最前面。
//如果保存到文件中，需要将此数据前加上 [0 0 0 1] 4个字节，写入到h264文件的最前面。
//如果推流，将此数据放入flv数据区即可。

if (videoEncoder->enabledWriteVideoFile) {
if (videoEncoder->enabledWriteVideoFile) {//debug
NSMutableData *data = [[NSMutableData alloc] init];
uint8_t header[] = {0x00, 0x00, 0x00, 0x01};
[data appendBytes:header length:4];
@@ -178,32 +198,39 @@ static void VideoCompressonOutputCallback(void *VTref, void *VTFrameRef, OSStatu
}


//获取视频数据
CMBlockBufferRef dataBuffer = CMSampleBufferGetDataBuffer(sampleBuffer);
size_t length, totalLength;
char *dataPointer;
//获取视频数据指针 数据大小 总数据大小
OSStatus statusCodeRet = CMBlockBufferGetDataPointer(dataBuffer, 0, &length, &totalLength, &dataPointer);
if (statusCodeRet == noErr) {
size_t bufferOffset = 0;
static const int AVCCHeaderLength = 4;
// 循环获取nalu数据
while (bufferOffset < totalLength - AVCCHeaderLength) {
// Read the NAL unit length
uint32_t NALUnitLength = 0;
memcpy(&NALUnitLength, dataPointer + bufferOffset, AVCCHeaderLength);


//大小端转化，关于大端和小端模式，请参考此网址：http://blog.csdn.net/sunjie886/article/details/54944810
NALUnitLength = CFSwapInt32BigToHost(NALUnitLength);

//封装视频数据LFVideoFrame,方便以后推流
LFVideoFrame *videoFrame = [LFVideoFrame new];
videoFrame.timestamp = timeStamp;
videoFrame.data = [[NSData alloc] initWithBytes:(dataPointer + bufferOffset + AVCCHeaderLength) length:NALUnitLength];
videoFrame.isKeyFrame = keyframe;
videoFrame.sps = videoEncoder->sps;
videoFrame.pps = videoEncoder->pps;

//调用视频编码完成后的代理
if (videoEncoder.h264Delegate && [videoEncoder.h264Delegate respondsToSelector:@selector(videoEncoder:videoFrame:)]) {
[videoEncoder.h264Delegate videoEncoder:videoEncoder videoFrame:videoFrame];
}

if (videoEncoder->enabledWriteVideoFile) {
if (videoEncoder->enabledWriteVideoFile) {//debug
NSMutableData *data = [[NSMutableData alloc] init];
if (keyframe) {
uint8_t header[] = {0x00, 0x00, 0x00, 0x01};