vendor: OpenCV 5.0.0 snapshot at 40738fb16ceddb5fb3fea747585f7ce6abb0605b

2026-08-22 00:10:33 +08:00
commit f7f077da11
6933 changed files with 2335208 additions and 0 deletions
@@ -0,0 +1,575 @@
+//
+//  CvVideoCamera2.mm
+//
+//  Created by Giles Payne on 2020/03/11.
+//
+
+#import "Mat.h"
+#import "CvCamera2.h"
+#import <UIKit/UIKit.h>
+
+static CGFloat DegreesToRadians(CGFloat degrees) {return degrees * M_PI / 180;}
+
+#pragma mark - Private Interface
+
+@interface CvVideoCamera2 () {
+    int recordingCountDown;
+}
+
+- (void)createVideoDataOutput;
+- (void)createVideoFileOutput;
+
+@property (nonatomic, strong) CALayer *customPreviewLayer;
+@property (nonatomic, strong) AVCaptureVideoDataOutput *videoDataOutput;
+
+@end
+
+
+#pragma mark - Implementation
+
+@implementation CvVideoCamera2
+{
+    id<CvVideoCameraDelegate2> _delegate;
+    dispatch_queue_t videoDataOutputQueue;
+    CMTime lastSampleTime;
+}
+
+- (void)setDelegate:(id<CvVideoCameraDelegate2>)newDelegate {
+    _delegate = newDelegate;
+}
+
+- (id<CvVideoCameraDelegate2>)delegate {
+    return _delegate;
+}
+
+#pragma mark - Constructors
+
+- (id)initWithParentView:(UIView*)parent {
+    self = [super initWithParentView:parent];
+    if (self) {
+        parent.contentMode = UIViewContentModeScaleAspectFill;
+        self.useAVCaptureVideoPreviewLayer = NO;
+        self.recordVideo = NO;
+        self.rotateVideo = NO;
+        self.defaultAVCaptureDevicePosition = AVCaptureDevicePositionBack;
+        self.defaultAVCaptureSessionPreset = AVCaptureSessionPresetHigh;
+        self.defaultAVCaptureVideoOrientation = AVCaptureVideoOrientationPortrait;
+        self.defaultFPS = 30;
+        self.grayscaleMode = NO;
+    }
+    return self;
+}
+
+#pragma mark - Public interface
+
+
+- (void)start {
+    if (self.running == YES) {
+        return;
+    }
+
+    recordingCountDown = 10;
+    [super start];
+
+    if (self.recordVideo == YES) {
+        NSError* error = nil;
+        if ([[NSFileManager defaultManager] fileExistsAtPath:[self videoFileString]]) {
+            [[NSFileManager defaultManager] removeItemAtPath:[self videoFileString] error:&error];
+        }
+        if (error == nil) {
+            NSLog(@"[Camera] Delete file %@", [self videoFileString]);
+        }
+    }
+}
+
+- (void)stop {
+    if (self.running == YES) {
+        [super stop];
+
+        if (self.recordVideo == YES) {
+            if (self.recordAssetWriter) {
+                if (self.recordAssetWriter.status == AVAssetWriterStatusWriting) {
+                    [self.recordAssetWriter finishWritingWithCompletionHandler:^void() {
+                        NSLog(@"[Camera] recording stopped");
+                    }];
+                } else {
+                    NSLog(@"[Camera] Recording Error: asset writer status is not writing");
+                }
+            }
+        }
+
+        if (self.customPreviewLayer) {
+            [self.customPreviewLayer removeFromSuperlayer];
+            self.customPreviewLayer = nil;
+        }
+    }
+}
+
+// TODO fix
+- (void)adjustLayoutToInterfaceOrientation:(UIInterfaceOrientation)interfaceOrientation {
+    NSLog(@"layout preview layer");
+    if (self.parentView != nil) {
+
+        CALayer* layer = self.customPreviewLayer;
+        CGRect bounds = self.customPreviewLayer.bounds;
+        int rotation_angle = 0;
+        bool flip_bounds = false;
+
+        switch (interfaceOrientation) {
+            case UIInterfaceOrientationPortrait:
+                NSLog(@"to Portrait");
+                rotation_angle = 270;
+                break;
+            case UIInterfaceOrientationPortraitUpsideDown:
+                rotation_angle = 90;
+                NSLog(@"to UpsideDown");
+                break;
+            case UIInterfaceOrientationLandscapeLeft:
+                rotation_angle = 0;
+                NSLog(@"to LandscapeLeft");
+                break;
+            case UIInterfaceOrientationLandscapeRight:
+                rotation_angle = 180;
+                NSLog(@"to LandscapeRight");
+                break;
+            default:
+                break; // leave the layer in its last known orientation
+        }
+
+        switch (self.defaultAVCaptureVideoOrientation) {
+            case AVCaptureVideoOrientationLandscapeRight:
+                rotation_angle += 180;
+                break;
+            case AVCaptureVideoOrientationPortraitUpsideDown:
+                rotation_angle += 270;
+                break;
+            case AVCaptureVideoOrientationPortrait:
+                rotation_angle += 90;
+            case AVCaptureVideoOrientationLandscapeLeft:
+                break;
+            default:
+                break;
+        }
+        rotation_angle = rotation_angle % 360;
+
+        if (rotation_angle == 90 || rotation_angle == 270) {
+            flip_bounds = true;
+        }
+
+        if (flip_bounds) {
+            NSLog(@"flip bounds");
+            bounds = CGRectMake(0, 0, bounds.size.height, bounds.size.width);
+        }
+
+        layer.position = CGPointMake(self.parentView.frame.size.width/2., self.parentView.frame.size.height/2.);
+        self.customPreviewLayer.bounds = CGRectMake(0, 0, self.parentView.frame.size.width, self.parentView.frame.size.height);
+
+        layer.affineTransform = CGAffineTransformMakeRotation( DegreesToRadians(rotation_angle) );
+        layer.bounds = bounds;
+    }
+
+}
+
+// TODO fix
+- (void)layoutPreviewLayer {
+    NSLog(@"layout preview layer");
+    if (self.parentView != nil) {
+
+        CALayer* layer = self.customPreviewLayer;
+        CGRect bounds = self.customPreviewLayer.bounds;
+        int rotation_angle = 0;
+        bool flip_bounds = false;
+
+        switch (self.currentDeviceOrientation) {
+            case UIDeviceOrientationPortrait:
+                rotation_angle = 270;
+                break;
+            case UIDeviceOrientationPortraitUpsideDown:
+                rotation_angle = 90;
+                break;
+            case UIDeviceOrientationLandscapeLeft:
+                NSLog(@"left");
+                rotation_angle = 180;
+                break;
+            case UIDeviceOrientationLandscapeRight:
+                NSLog(@"right");
+                rotation_angle = 0;
+                break;
+            case UIDeviceOrientationFaceUp:
+            case UIDeviceOrientationFaceDown:
+            default:
+                break; // leave the layer in its last known orientation
+        }
+
+        switch (self.defaultAVCaptureVideoOrientation) {
+            case AVCaptureVideoOrientationLandscapeRight:
+                rotation_angle += 180;
+                break;
+            case AVCaptureVideoOrientationPortraitUpsideDown:
+                rotation_angle += 270;
+                break;
+            case AVCaptureVideoOrientationPortrait:
+                rotation_angle += 90;
+            case AVCaptureVideoOrientationLandscapeLeft:
+                break;
+            default:
+                break;
+        }
+        rotation_angle = rotation_angle % 360;
+
+        if (rotation_angle == 90 || rotation_angle == 270) {
+            flip_bounds = true;
+        }
+
+        if (flip_bounds) {
+            NSLog(@"flip bounds");
+            bounds = CGRectMake(0, 0, bounds.size.height, bounds.size.width);
+        }
+
+        layer.position = CGPointMake(self.parentView.frame.size.width/2., self.parentView.frame.size.height/2.);
+        layer.affineTransform = CGAffineTransformMakeRotation( DegreesToRadians(rotation_angle) );
+        layer.bounds = bounds;
+    }
+
+}
+
+#pragma mark - Private Interface
+
+- (void)createVideoDataOutput {
+    // Make a video data output
+    self.videoDataOutput = [AVCaptureVideoDataOutput new];
+
+    // In grayscale mode we want YUV (YpCbCr 4:2:0) so we can directly access the graylevel intensity values (Y component)
+    // In color mode we, BGRA format is used
+    OSType format = self.grayscaleMode ? kCVPixelFormatType_420YpCbCr8BiPlanarFullRange : kCVPixelFormatType_32BGRA;
+
+    self.videoDataOutput.videoSettings  = [NSDictionary dictionaryWithObject:[NSNumber numberWithUnsignedInt:format]
+                                                                      forKey:(id)kCVPixelBufferPixelFormatTypeKey];
+
+    // discard if the data output queue is blocked (as we process the still image)
+    [self.videoDataOutput setAlwaysDiscardsLateVideoFrames:YES];
+
+    if ( [self.captureSession canAddOutput:self.videoDataOutput] ) {
+        [self.captureSession addOutput:self.videoDataOutput];
+    }
+    [[self.videoDataOutput connectionWithMediaType:AVMediaTypeVideo] setEnabled:YES];
+
+
+    // set default FPS
+    AVCaptureDeviceInput *currentInput = [self.captureSession.inputs objectAtIndex:0];
+    AVCaptureDevice *device = currentInput.device;
+
+    NSError *error = nil;
+    [device lockForConfiguration:&error];
+
+    float maxRate = ((AVFrameRateRange*) [device.activeFormat.videoSupportedFrameRateRanges objectAtIndex:0]).maxFrameRate;
+    if (maxRate > self.defaultFPS - 1 && error == nil) {
+        [device setActiveVideoMinFrameDuration:CMTimeMake(1, self.defaultFPS)];
+        [device setActiveVideoMaxFrameDuration:CMTimeMake(1, self.defaultFPS)];
+        NSLog(@"[Camera] FPS set to %d", self.defaultFPS);
+    } else {
+        NSLog(@"[Camera] unable to set defaultFPS at %d FPS, max is %f FPS", self.defaultFPS, maxRate);
+    }
+
+    if (error != nil) {
+        NSLog(@"[Camera] unable to set defaultFPS: %@", error);
+    }
+
+    [device unlockForConfiguration];
+
+    // set video mirroring for front camera (more intuitive)
+    if ([self.videoDataOutput connectionWithMediaType:AVMediaTypeVideo].supportsVideoMirroring) {
+        if (self.defaultAVCaptureDevicePosition == AVCaptureDevicePositionFront) {
+            [self.videoDataOutput connectionWithMediaType:AVMediaTypeVideo].videoMirrored = YES;
+        } else {
+            [self.videoDataOutput connectionWithMediaType:AVMediaTypeVideo].videoMirrored = NO;
+        }
+    }
+
+    // set default video orientation
+    if ([self.videoDataOutput connectionWithMediaType:AVMediaTypeVideo].supportsVideoOrientation) {
+        [self.videoDataOutput connectionWithMediaType:AVMediaTypeVideo].videoOrientation = self.defaultAVCaptureVideoOrientation;
+    }
+
+
+    // create a custom preview layer
+    self.customPreviewLayer = [CALayer layer];
+    self.customPreviewLayer.bounds = CGRectMake(0, 0, self.parentView.frame.size.width, self.parentView.frame.size.height);
+    self.customPreviewLayer.position = CGPointMake(self.parentView.frame.size.width/2., self.parentView.frame.size.height/2.);
+    [self updateOrientation];
+
+    // create a serial dispatch queue used for the sample buffer delegate as well as when a still image is captured
+    // a serial dispatch queue must be used to guarantee that video frames will be delivered in order
+    // see the header doc for setSampleBufferDelegate:queue: for more information
+    videoDataOutputQueue = dispatch_queue_create("VideoDataOutputQueue", DISPATCH_QUEUE_SERIAL);
+    [self.videoDataOutput setSampleBufferDelegate:self queue:videoDataOutputQueue];
+
+
+    NSLog(@"[Camera] created AVCaptureVideoDataOutput");
+}
+
+- (void)createVideoFileOutput {
+    /* Video File Output in H.264, via AVAsserWriter */
+    NSLog(@"Create Video with dimensions %dx%d", self.imageWidth, self.imageHeight);
+
+    NSDictionary *outputSettings
+     = [NSDictionary dictionaryWithObjectsAndKeys:[NSNumber numberWithInt:self.imageWidth], AVVideoWidthKey,
+                                                  [NSNumber numberWithInt:self.imageHeight], AVVideoHeightKey,
+                                                  AVVideoCodecTypeH264, AVVideoCodecKey,
+                                                  nil
+     ];
+
+
+    self.recordAssetWriterInput = [AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeVideo outputSettings:outputSettings];
+
+
+    int pixelBufferFormat = (self.grayscaleMode == YES) ? kCVPixelFormatType_420YpCbCr8BiPlanarFullRange : kCVPixelFormatType_32BGRA;
+
+    self.recordPixelBufferAdaptor =
+               [[AVAssetWriterInputPixelBufferAdaptor alloc]
+                    initWithAssetWriterInput:self.recordAssetWriterInput
+                    sourcePixelBufferAttributes:[NSDictionary dictionaryWithObjectsAndKeys:[NSNumber numberWithInt:pixelBufferFormat], kCVPixelBufferPixelFormatTypeKey, nil]];
+
+    NSError* error = nil;
+    NSLog(@"Create AVAssetWriter with url: %@", [self videoFileURL]);
+    self.recordAssetWriter = [AVAssetWriter assetWriterWithURL:[self videoFileURL]
+                                                      fileType:AVFileTypeMPEG4
+                                                         error:&error];
+    if (error != nil) {
+        NSLog(@"[Camera] Unable to create AVAssetWriter: %@", error);
+    }
+
+    [self.recordAssetWriter addInput:self.recordAssetWriterInput];
+    self.recordAssetWriterInput.expectsMediaDataInRealTime = YES;
+
+    NSLog(@"[Camera] created AVAssetWriter");
+}
+
+- (void)createCaptureOutput {
+    [self createVideoDataOutput];
+    if (self.recordVideo == YES) {
+        [self createVideoFileOutput];
+    }
+}
+
+- (void)createCustomVideoPreview {
+    [self.parentView.layer addSublayer:self.customPreviewLayer];
+}
+
+- (CVPixelBufferRef) pixelBufferFromCGImage: (CGImageRef) image {
+
+    CGSize frameSize = CGSizeMake(CGImageGetWidth(image), CGImageGetHeight(image));
+    NSDictionary *options = [NSDictionary dictionaryWithObjectsAndKeys:
+                             [NSNumber numberWithBool:NO], kCVPixelBufferCGImageCompatibilityKey,
+                             [NSNumber numberWithBool:NO], kCVPixelBufferCGBitmapContextCompatibilityKey,
+                             nil];
+    CVPixelBufferRef pxbuffer = NULL;
+    CVReturn status = CVPixelBufferCreate(kCFAllocatorDefault, frameSize.width,
+                                          frameSize.height,  kCVPixelFormatType_32ARGB, (CFDictionaryRef) CFBridgingRetain(options),
+                                          &pxbuffer);
+    NSParameterAssert(status == kCVReturnSuccess && pxbuffer != NULL);
+
+    CVPixelBufferLockBaseAddress(pxbuffer, 0);
+    void *pxdata = CVPixelBufferGetBaseAddress(pxbuffer);
+
+
+    CGColorSpaceRef rgbColorSpace = CGColorSpaceCreateDeviceRGB();
+    CGContextRef context = CGBitmapContextCreate(pxdata, frameSize.width,
+                                                 frameSize.height, 8, 4*frameSize.width, rgbColorSpace,
+                                                 kCGImageAlphaPremultipliedFirst);
+
+    CGContextDrawImage(context, CGRectMake(0, 0, CGImageGetWidth(image),
+                                           CGImageGetHeight(image)), image);
+    CGColorSpaceRelease(rgbColorSpace);
+    CGContextRelease(context);
+
+    CVPixelBufferUnlockBaseAddress(pxbuffer, 0);
+
+    return pxbuffer;
+}
+
+#pragma mark - Protocol AVCaptureVideoDataOutputSampleBufferDelegate
+
+- (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection
+{
+    (void)captureOutput;
+    (void)connection;
+    auto strongDelegate = self.delegate;
+    if (strongDelegate) {
+
+        // convert from Core Media to Core Video
+        CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
+        CVPixelBufferLockBaseAddress(imageBuffer, 0);
+
+        void* bufferAddress;
+        size_t width;
+        size_t height;
+        size_t bytesPerRow;
+
+        CGColorSpaceRef colorSpace;
+        CGContextRef context;
+
+        int format_opencv;
+
+        OSType format = CVPixelBufferGetPixelFormatType(imageBuffer);
+        if (format == kCVPixelFormatType_420YpCbCr8BiPlanarFullRange) {
+
+            format_opencv = CV_8UC1;
+
+            bufferAddress = CVPixelBufferGetBaseAddressOfPlane(imageBuffer, 0);
+            width = CVPixelBufferGetWidthOfPlane(imageBuffer, 0);
+            height = CVPixelBufferGetHeightOfPlane(imageBuffer, 0);
+            bytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(imageBuffer, 0);
+
+        } else { // expect kCVPixelFormatType_32BGRA
+
+            format_opencv = CV_8UC4;
+
+            bufferAddress = CVPixelBufferGetBaseAddress(imageBuffer);
+            width = CVPixelBufferGetWidth(imageBuffer);
+            height = CVPixelBufferGetHeight(imageBuffer);
+            bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer);
+
+        }
+
+        // delegate image processing to the delegate
+        cv::Mat image((int)height, (int)width, format_opencv, bufferAddress, bytesPerRow);
+
+        CGImage* dstImage;
+
+        if ([strongDelegate respondsToSelector:@selector(processImage:)]) {
+            [strongDelegate processImage:[Mat fromNative:image]];
+        }
+
+        // check if matrix data pointer or dimensions were changed by the delegate
+        bool iOSimage = false;
+        if (height == (size_t)image.rows && width == (size_t)image.cols && format_opencv == image.type() && bufferAddress == image.data && bytesPerRow == image.step) {
+            iOSimage = true;
+        }
+
+
+        // (create color space, create graphics context, render buffer)
+        CGBitmapInfo bitmapInfo;
+
+        // basically we decide if it's a grayscale, rgb or rgba image
+        if (image.channels() == 1) {
+            colorSpace = CGColorSpaceCreateDeviceGray();
+            bitmapInfo = kCGImageAlphaNone;
+        } else if (image.channels() == 3) {
+            colorSpace = CGColorSpaceCreateDeviceRGB();
+            bitmapInfo = kCGImageAlphaNone;
+            if (iOSimage) {
+                bitmapInfo |= kCGBitmapByteOrder32Little;
+            } else {
+                bitmapInfo |= kCGBitmapByteOrder32Big;
+            }
+        } else {
+            colorSpace = CGColorSpaceCreateDeviceRGB();
+            bitmapInfo = kCGImageAlphaPremultipliedFirst;
+            if (iOSimage) {
+                bitmapInfo |= kCGBitmapByteOrder32Little;
+            } else {
+                bitmapInfo |= kCGBitmapByteOrder32Big;
+            }
+        }
+
+        if (iOSimage) {
+            context = CGBitmapContextCreate(bufferAddress, width, height, 8, bytesPerRow, colorSpace, bitmapInfo);
+            dstImage = CGBitmapContextCreateImage(context);
+            CGContextRelease(context);
+        } else {
+
+            NSData *data = [NSData dataWithBytes:image.data length:image.elemSize()*image.total()];
+            CGDataProviderRef provider = CGDataProviderCreateWithCFData((__bridge CFDataRef)data);
+
+            // Creating CGImage from cv::Mat
+            dstImage = CGImageCreate(image.cols,                                 // width
+                                     image.rows,                                 // height
+                                     8,                                          // bits per component
+                                     8 * image.elemSize(),                       // bits per pixel
+                                     image.step,                                 // bytesPerRow
+                                     colorSpace,                                 // colorspace
+                                     bitmapInfo,                                 // bitmap info
+                                     provider,                                   // CGDataProviderRef
+                                     NULL,                                       // decode
+                                     false,                                      // should interpolate
+                                     kCGRenderingIntentDefault                   // intent
+                                     );
+
+            CGDataProviderRelease(provider);
+        }
+
+
+        // render buffer
+        dispatch_sync(dispatch_get_main_queue(), ^{
+            self.customPreviewLayer.contents = (__bridge id)dstImage;
+        });
+
+
+        recordingCountDown--;
+        if (self.recordVideo == YES && recordingCountDown < 0) {
+            lastSampleTime = CMSampleBufferGetPresentationTimeStamp(sampleBuffer);
+//            CMTimeShow(lastSampleTime);
+            if (self.recordAssetWriter.status != AVAssetWriterStatusWriting) {
+                [self.recordAssetWriter startWriting];
+                [self.recordAssetWriter startSessionAtSourceTime:lastSampleTime];
+                if (self.recordAssetWriter.status != AVAssetWriterStatusWriting) {
+                    NSLog(@"[Camera] Recording Error: asset writer status is not writing: %@", self.recordAssetWriter.error);
+                    return;
+                } else {
+                    NSLog(@"[Camera] Video recording started");
+                }
+            }
+
+            if (self.recordAssetWriterInput.readyForMoreMediaData) {
+                CVImageBufferRef pixelBuffer = [self pixelBufferFromCGImage:dstImage];
+                if (! [self.recordPixelBufferAdaptor appendPixelBuffer:pixelBuffer
+                                                  withPresentationTime:lastSampleTime] ) {
+                    NSLog(@"Video Writing Error");
+                }
+                if (pixelBuffer != nullptr)
+                    CVPixelBufferRelease(pixelBuffer);
+            }
+
+        }
+
+        // cleanup
+        CGImageRelease(dstImage);
+        CGColorSpaceRelease(colorSpace);
+        CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
+    }
+}
+
+- (void)updateOrientation {
+    if (self.rotateVideo == YES)
+    {
+        NSLog(@"rotate..");
+        self.customPreviewLayer.bounds = CGRectMake(0, 0, self.parentView.frame.size.width, self.parentView.frame.size.height);
+        [self layoutPreviewLayer];
+    }
+}
+
+- (void)saveVideo {
+    if (self.recordVideo == NO) {
+        return;
+    }
+
+    UISaveVideoAtPathToSavedPhotosAlbum([self videoFileString], nil, nil, NULL);
+}
+
+- (NSURL *)videoFileURL {
+    NSString *outputPath = [[NSString alloc] initWithFormat:@"%@%@", NSTemporaryDirectory(), @"output.mov"];
+    NSURL *outputURL = [NSURL fileURLWithPath:outputPath];
+    NSFileManager *fileManager = [NSFileManager defaultManager];
+    if ([fileManager fileExistsAtPath:outputPath]) {
+        NSLog(@"file exists");
+    }
+    return outputURL;
+}
+
+- (NSString *)videoFileString {
+    NSString *outputPath = [[NSString alloc] initWithFormat:@"%@%@", NSTemporaryDirectory(), @"output.mov"];
+    return outputPath;
+}
+
+@end