1// 2// libtgvoip is free and unencumbered public domain software. 3// For more information, see http://unlicense.org or the UNLICENSE file 4// you should have received with this source code distribution. 5// 6 7#import <Foundation/Foundation.h> 8#include "VideoToolboxEncoderSource.h" 9#include "../../PrivateDefines.h" 10#include "../../logging.h" 11 12using namespace tgvoip; 13using namespace tgvoip::video; 14 15#define CHECK_ERR(err, msg) if(err!=noErr){LOGE("VideoToolboxEncoder: " msg " failed: %d", err); return;} 16 17VideoToolboxEncoderSource::VideoToolboxEncoderSource(){ 18 19} 20 21VideoToolboxEncoderSource::~VideoToolboxEncoderSource(){ 22 if(session){ 23 CFRelease(session); 24 session=NULL; 25 } 26} 27 28void VideoToolboxEncoderSource::Start(){ 29 30} 31 32void VideoToolboxEncoderSource::Stop(){ 33 34} 35 36void VideoToolboxEncoderSource::Reset(uint32_t codec, int maxResolution){ 37 if(session){ 38 LOGV("Releasing old compression session"); 39 //VTCompressionSessionCompleteFrames(session, kCMTimeInvalid); 40 VTCompressionSessionInvalidate(session); 41 CFRelease(session); 42 session=NULL; 43 LOGV("Released compression session"); 44 } 45 CMVideoCodecType codecType; 46 switch(codec){ 47 case CODEC_AVC: 48 codecType=kCMVideoCodecType_H264; 49 break; 50 case CODEC_HEVC: 51 codecType=kCMVideoCodecType_HEVC; 52 break; 53 default: 54 LOGE("VideoToolboxEncoder: Unsupported codec"); 55 return; 56 } 57 needUpdateStreamParams=true; 58 this->codec=codec; 59 // typedef void (*VTCompressionOutputCallback)(void *outputCallbackRefCon, void *sourceFrameRefCon, OSStatus status, VTEncodeInfoFlags infoFlags, CMSampleBufferRef sampleBuffer); 60 uint32_t width, height; 61 switch(maxResolution){ 62 case INIT_VIDEO_RES_1080: 63 width=1920; 64 height=1080; 65 break; 66 case INIT_VIDEO_RES_720: 67 width=1280; 68 height=720; 69 break; 70 case INIT_VIDEO_RES_480: 71 width=854; 72 height=480; 73 break; 74 case INIT_VIDEO_RES_360: 75 default: 76 width=640; 77 height=360; 78 break; 79 } 80 OSStatus status=VTCompressionSessionCreate(NULL, width, height, codecType, NULL, NULL, NULL, [](void *outputCallbackRefCon, void *sourceFrameRefCon, OSStatus status, VTEncodeInfoFlags infoFlags, CMSampleBufferRef sampleBuffer){ 81 reinterpret_cast<VideoToolboxEncoderSource*>(outputCallbackRefCon)->EncoderCallback(status, sampleBuffer, infoFlags); 82 }, this, &session); 83 if(status!=noErr){ 84 LOGE("VTCompressionSessionCreate failed: %d", status); 85 return; 86 } 87 LOGD("Created VTCompressionSession"); 88 status=VTSessionSetProperty(session, kVTCompressionPropertyKey_AllowFrameReordering, kCFBooleanFalse); 89 CHECK_ERR(status, "VTSessionSetProperty(AllowFrameReordering)"); 90 int64_t interval=15; 91 status=VTSessionSetProperty(session, kVTCompressionPropertyKey_MaxKeyFrameIntervalDuration, (__bridge CFTypeRef)@(interval)); 92 CHECK_ERR(status, "VTSessionSetProperty(MaxKeyFrameIntervalDuration)"); 93 SetEncoderBitrateAndLimit(lastBitrate); 94 status=VTSessionSetProperty(session, kVTCompressionPropertyKey_RealTime, kCFBooleanTrue); 95 CHECK_ERR(status, "VTSessionSetProperty(RealTime)"); 96 LOGD("VTCompressionSession initialized"); 97 98 // TODO change camera frame rate dynamically based on resolution + codec 99} 100 101void VideoToolboxEncoderSource::RequestKeyFrame(){ 102 keyframeRequested=true; 103} 104 105void VideoToolboxEncoderSource::EncodeFrame(CMSampleBufferRef frame){ 106 if(!session) 107 return; 108 CMFormatDescriptionRef format=CMSampleBufferGetFormatDescription(frame); 109 CMMediaType type=CMFormatDescriptionGetMediaType(format); 110 if(type!=kCMMediaType_Video){ 111 //LOGW("Received non-video CMSampleBuffer"); 112 return; 113 } 114 if(bitrateChangeRequested){ 115 LOGI("VideoToolboxEocnder: setting bitrate to %u", bitrateChangeRequested); 116 SetEncoderBitrateAndLimit(bitrateChangeRequested); 117 lastBitrate=bitrateChangeRequested; 118 bitrateChangeRequested=0; 119 } 120 CFDictionaryRef frameProps=NULL; 121 if(keyframeRequested){ 122 LOGI("VideoToolboxEncoder: requesting keyframe"); 123 const void* keys[]={kVTEncodeFrameOptionKey_ForceKeyFrame}; 124 const void* values[]={kCFBooleanTrue}; 125 frameProps=CFDictionaryCreate(NULL, keys, values, 1, NULL, NULL); 126 keyframeRequested=false; 127 } 128 129 //CMVideoDimensions size=CMVideoFormatDescriptionGetDimensions(format); 130 //LOGD("EncodeFrame %d x %d", size.width, size.height); 131 CVImageBufferRef imgBuffer=CMSampleBufferGetImageBuffer(frame); 132 //OSType pixFmt=CVPixelBufferGetPixelFormatType(imgBuffer); 133 //LOGV("pixel format: %c%c%c%c", PRINT_FOURCC(pixFmt)); 134 OSStatus status=VTCompressionSessionEncodeFrame(session, imgBuffer, CMSampleBufferGetPresentationTimeStamp(frame), CMSampleBufferGetDuration(frame), frameProps, NULL, NULL); 135 CHECK_ERR(status, "VTCompressionSessionEncodeFrame"); 136 if(frameProps) 137 CFRelease(frameProps); 138} 139 140void VideoToolboxEncoderSource::SetBitrate(uint32_t bitrate){ 141 bitrateChangeRequested=bitrate; 142} 143 144void VideoToolboxEncoderSource::EncoderCallback(OSStatus status, CMSampleBufferRef buffer, VTEncodeInfoFlags flags){ 145 if(status!=noErr){ 146 LOGE("EncoderCallback error: %d", status); 147 return; 148 } 149 if(flags & kVTEncodeInfo_FrameDropped){ 150 LOGW("VideoToolboxEncoder: Frame dropped"); 151 } 152 if(!CMSampleBufferGetNumSamples(buffer)){ 153 LOGW("Empty CMSampleBuffer"); 154 return; 155 } 156 const uint8_t startCode[]={0, 0, 0, 1}; 157 if(needUpdateStreamParams){ 158 LOGI("VideoToolboxEncoder: Updating stream params"); 159 CMFormatDescriptionRef format=CMSampleBufferGetFormatDescription(buffer); 160 CMVideoDimensions size=CMVideoFormatDescriptionGetDimensions(format); 161 width=size.width; 162 height=size.height; 163 csd.clear(); 164 if(codec==CODEC_AVC){ 165 for(size_t i=0;i<2;i++){ 166 const uint8_t* ps=NULL; 167 size_t pl=0; 168 status=CMVideoFormatDescriptionGetH264ParameterSetAtIndex(format, i, &ps, &pl, NULL, NULL); 169 CHECK_ERR(status, "CMVideoFormatDescriptionGetH264ParameterSetAtIndex"); 170 Buffer b(pl+4); 171 b.CopyFrom(ps, 4, pl); 172 b.CopyFrom(startCode, 0, 4); 173 csd.push_back(std::move(b)); 174 } 175 }else if(codec==CODEC_HEVC){ 176 LOGD("here1"); 177 BufferOutputStream csdBuf(1024); 178 for(size_t i=0;i<3;i++){ 179 const uint8_t* ps=NULL; 180 size_t pl=0; 181 status=CMVideoFormatDescriptionGetHEVCParameterSetAtIndex(format, i, &ps, &pl, NULL, NULL); 182 CHECK_ERR(status, "CMVideoFormatDescriptionGetHEVCParameterSetAtIndex"); 183 csdBuf.WriteBytes(startCode, 4); 184 csdBuf.WriteBytes(ps, pl); 185 } 186 csd.push_back(std::move(csdBuf)); 187 } 188 needUpdateStreamParams=false; 189 } 190 CMBlockBufferRef blockBuffer=CMSampleBufferGetDataBuffer(buffer); 191 size_t len=CMBlockBufferGetDataLength(blockBuffer); 192 193 int frameFlags=0; 194 CFArrayRef attachmentsArray=CMSampleBufferGetSampleAttachmentsArray(buffer, 0); 195 if(attachmentsArray && CFArrayGetCount(attachmentsArray)){ 196 CFBooleanRef notSync; 197 CFDictionaryRef dict=(CFDictionaryRef)CFArrayGetValueAtIndex(attachmentsArray, 0); 198 BOOL keyExists=CFDictionaryGetValueIfPresent(dict, kCMSampleAttachmentKey_NotSync, (const void **)¬Sync); 199 if(!keyExists || !CFBooleanGetValue(notSync)){ 200 frameFlags |= VIDEO_FRAME_FLAG_KEYFRAME; 201 } 202 }else{ 203 frameFlags |= VIDEO_FRAME_FLAG_KEYFRAME; 204 } 205 206 Buffer frame(len); 207 CMBlockBufferCopyDataBytes(blockBuffer, 0, len, *frame); 208 uint32_t offset=0; 209 while(offset<len){ 210 uint32_t nalLen=CFSwapInt32BigToHost(*reinterpret_cast<uint32_t*>(*frame+offset)); 211 //LOGV("NAL length %u", nalLen); 212 frame.CopyFrom(startCode, offset, 4); 213 offset+=nalLen+4; 214 } 215 callback(std::move(frame), frameFlags); 216 217 //LOGV("EncoderCallback: %u bytes total", (unsigned int)len); 218} 219 220void VideoToolboxEncoderSource::SetEncoderBitrateAndLimit(uint32_t bitrate){ 221 OSStatus status=VTSessionSetProperty(session, kVTCompressionPropertyKey_AverageBitRate, (__bridge CFTypeRef)@(bitrate)); 222 CHECK_ERR(status, "VTSessionSetProperty(AverageBitRate)"); 223 224 int64_t dataLimitValue=(int64_t)(bitrate/8); 225 CFNumberRef bytesPerSecond=CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &dataLimitValue); 226 int64_t oneValue=1; 227 CFNumberRef one=CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt64Type, &oneValue); 228 const void* numbers[]={bytesPerSecond, one}; 229 CFArrayRef limits=CFArrayCreate(NULL, numbers, 2, &kCFTypeArrayCallBacks); 230 status=VTSessionSetProperty(session, kVTCompressionPropertyKey_DataRateLimits, limits); 231 CFRelease(bytesPerSecond); 232 CFRelease(one); 233 CFRelease(limits); 234 CHECK_ERR(status, "VTSessionSetProperty(DataRateLimits"); 235} 236 237std::vector<uint32_t> VideoToolboxEncoderSource::GetAvailableEncoders(){ 238 std::vector<uint32_t> res; 239 res.push_back(CODEC_AVC); 240 CFArrayRef encoders; 241 OSStatus status=VTCopyVideoEncoderList(NULL, &encoders); 242 for(CFIndex i=0;i<CFArrayGetCount(encoders);i++){ 243 CFDictionaryRef v=(CFDictionaryRef)CFArrayGetValueAtIndex(encoders, i); 244 NSDictionary* encoder=(__bridge NSDictionary*)v; 245 //NSString* name=(NSString*)CFDictionaryGetValue(v, kVTVideoEncoderList_EncoderName); 246 uint32_t codecType=[(NSNumber*)encoder[(NSString*)kVTVideoEncoderList_CodecType] intValue]; 247 //LOGV("Encoders[%u]: %s, %c%c%c%c", i, [(NSString*)encoder[(NSString*)kVTVideoEncoderList_EncoderName] cStringUsingEncoding:NSUTF8StringEncoding], PRINT_FOURCC(codecType)); 248 if(codecType==kCMVideoCodecType_HEVC){ 249 res.push_back(CODEC_HEVC); 250 break; 251 } 252 } 253 CFRelease(encoders); 254 return res; 255} 256