iPhone - 使用 AVAssetWriterInputPixelBufferAdaptor 录制视频时从麦克风捕获音频

Question

我正在尝试制作一个应用程序来记录 UIImageView 的内容以及麦克风音频，并将它们实时记录到视频中。 (有点像会说话的汤姆猫应用程序)

我正在使用 AVAssetWriterInputPixelBufferAdaptor 毫无困难地记录 UIImageView 的内容，但我根本不知道如何将音频与此合并。这也不是因为缺乏尝试，我已经在这个特定问题上花了一个多星期的时间，梳理了这个网站、谷歌、iPhone 开发者论坛等。这不是我喜欢的。

与此最接近的引用是: How do I export UIImage array as a movie?

我可以分别捕获视频和音频，然后在事后将它们编码在一起(我真的很努力地解决这个问题)，但是让这个应用程序实时录制非常理想。

这里是一些框架代码，演示了我迄今为止的记录:

这是 .h 文件

//
//  RecordingTestProjectViewController.h
//  RecordingTestProject
//
//  Created by Sean Luck on 7/26/11.
//  Copyright 2011 __MyCompanyName__. All rights reserved.
//

#import <UIKit/UIKit.h>
#import <AVFoundation/AVFoundation.h>
#import <CoreGraphics/CoreGraphics.h>
#import <CoreMedia/CoreMedia.h>
#import <CoreVideo/CoreVideo.h>
#import <QuartzCore/QuartzCore.h>

@interface RecordingTestProjectViewController : UIViewController 
    {
        UIImageView *testView;
        NSTimer *theTimer;

        NSTimer *assetWriterTimer;
        AVMutableComposition *mutableComposition;
        AVAssetWriter *assetWriter;
        AVAssetWriterInput *assetWriterInput;

        AVAssetWriterInput *_audioWriterInput;
        AVCaptureDeviceInput *audioInput;
        AVCaptureSession *_capSession;

        AVAssetWriterInputPixelBufferAdaptor *assetWriterPixelBufferAdaptor;
        CFAbsoluteTime firstFrameWallClockTime;
        int count;
    }

-(void) writeSample: (NSTimer*) _timer; 
-(void) startRecording;
-(void) pauseRecording;
-(void) stopRecording;
-(NSString*) pathToDocumentsDirectory; 


@end

和 .m 文件

//
//  RecordingTestProjectViewController.m
//  RecordingTestProject
//
//  Created by Sean Luck on 7/26/11.
//  Copyright 2011 __MyCompanyName__. All rights reserved.
//

#import "RecordingTestProjectViewController.h"
#import <AVFoundation/AVFoundation.h>
#import <CoreGraphics/CoreGraphics.h>
#import <CoreMedia/CoreMedia.h>
#import <CoreVideo/CoreVideo.h>
#import <QuartzCore/QuartzCore.h>

#define OUTPUT_FILE_NAME @"screen.mov"

#define TIME_SCALE 600

@implementation RecordingTestProjectViewController

- (void)dealloc
    {
        [super dealloc];
    }

- (void)didReceiveMemoryWarning
    {
        // Releases the view if it doesn't have a superview.
        [super didReceiveMemoryWarning];

        // Release any cached data, images, etc that aren't in use.
    }

#pragma mark - View lifecycle


// Implement viewDidLoad to do additional setup after loading the view, typically from a nib.
- (void)viewDidLoad
    {

        testView = [[UIImageView alloc] initWithImage:nil];
        testView.frame = CGRectMake(0, 0, self.view.frame.size.width, self.view.frame.size.height);
        testView.userInteractionEnabled=NO;
        testView.backgroundColor = [UIColor lightGrayColor];
        [self.view addSubview:testView];
        [testView release];



        [super viewDidLoad];
        [self startRecording];
    }







-(void) writeSample: (NSTimer*) _timer 
    {

        if ([assetWriterInput isReadyForMoreMediaData]) 
            {


                NSLog(@"count=%.i",count);
                count=count+10;

                UIGraphicsBeginImageContext(testView.frame.size);
                [testView.image drawInRect:CGRectMake(0, 0, testView.frame.size.width, testView.frame.size.height)];
                CGContextSetBlendMode(UIGraphicsGetCurrentContext(), kCGBlendModeNormal);
                CGContextSetRGBStrokeColor(UIGraphicsGetCurrentContext(), 0,1,1,1);
                CGContextSetLineWidth(UIGraphicsGetCurrentContext(), 5);
                CGContextBeginPath(UIGraphicsGetCurrentContext());
                CGContextMoveToPoint(UIGraphicsGetCurrentContext(), 10+count/2, 10+count);
                CGContextAddLineToPoint(UIGraphicsGetCurrentContext(), 20+count*2, 20+count);
                CGContextStrokePath(UIGraphicsGetCurrentContext());
                testView.image = UIGraphicsGetImageFromCurrentImageContext();
                UIGraphicsEndImageContext();


                CVReturn cvErr = kCVReturnSuccess;
                CGImageRef image = (CGImageRef) [testView.image CGImage];

                // prepare the pixel buffer
                CVPixelBufferRef pixelBuffer = NULL;
                CFDataRef imageData= CGDataProviderCopyData(CGImageGetDataProvider(image));
                cvErr = CVPixelBufferCreateWithBytes(kCFAllocatorDefault,testView.frame.size.width,testView.frame.size.height,kCVPixelFormatType_32BGRA,(void*)CFDataGetBytePtr(imageData),CGImageGetBytesPerRow(image),NULL,NULL,NULL,&pixelBuffer);


                // calculate the time
                CFAbsoluteTime thisFrameWallClockTime = CFAbsoluteTimeGetCurrent();
                CFTimeInterval elapsedTime = thisFrameWallClockTime - firstFrameWallClockTime;
                CMTime presentationTime =  CMTimeMake (elapsedTime * TIME_SCALE, TIME_SCALE);

                if (!cvErr) 
                    {
                        // write the sample
                        BOOL appended = [assetWriterPixelBufferAdaptor appendPixelBuffer:pixelBuffer withPresentationTime:presentationTime];

                        if (appended) 
                            {

                            } 
                        else 
                            {
                                NSLog (@"failed to append");
                                [self stopRecording];
                            }
                    }


                CVPixelBufferRelease(pixelBuffer);
                CFRelease(imageData);

            }

        if (count>1000) 
            {
                [self stopRecording];
            }
    }

-(void) startRecording 
    {
        // Doesn't record audio at all. Needs to be implemented.

        // create the AVAssetWriter
        NSString *moviePath = [[self pathToDocumentsDirectory] stringByAppendingPathComponent:OUTPUT_FILE_NAME];
        if ([[NSFileManager defaultManager] fileExistsAtPath:moviePath]) 
            {
                [[NSFileManager defaultManager] removeItemAtPath:moviePath error:nil];
            }

        NSURL *movieURL = [NSURL fileURLWithPath:moviePath];
        NSError *movieError = nil;

        [assetWriter release];
        assetWriter = [[AVAssetWriter alloc] initWithURL:movieURL fileType: AVFileTypeQuickTimeMovie error: &movieError];
        NSDictionary *assetWriterInputSettings = [NSDictionary dictionaryWithObjectsAndKeys: AVVideoCodecH264, AVVideoCodecKey,[NSNumber numberWithInt:testView.frame.size.width], AVVideoWidthKey,[NSNumber numberWithInt:testView.frame.size.height], AVVideoHeightKey,nil];

        [assetWriterInput release];
        assetWriterInput =[AVAssetWriterInput assetWriterInputWithMediaType:AVMediaTypeVideo outputSettings:assetWriterInputSettings];
        assetWriterInput.expectsMediaDataInRealTime = YES;
        [assetWriter addInput:assetWriterInput];


        [assetWriterPixelBufferAdaptor release];
        assetWriterPixelBufferAdaptor = [[AVAssetWriterInputPixelBufferAdaptor  alloc] initWithAssetWriterInput:assetWriterInput sourcePixelBufferAttributes:nil];
        [assetWriter startWriting];

        firstFrameWallClockTime = CFAbsoluteTimeGetCurrent();
        [assetWriter startSessionAtSourceTime: CMTimeMake(0, TIME_SCALE)];

        // start writing samples to it
        [assetWriterTimer release];
        assetWriterTimer = [NSTimer scheduledTimerWithTimeInterval:0.2 target:self selector:@selector (writeSample:) userInfo:nil repeats:YES];

        [movieURL release];
        [assetWriterInputSettings release];

    }


-(void) stopRecording 
    {
        if (assetWriterTimer!=nil) 
            {
                [assetWriterTimer invalidate];
                assetWriterTimer = nil;
                [assetWriter finishWriting];


                NSString *moviePath = [[self pathToDocumentsDirectory] stringByAppendingPathComponent:OUTPUT_FILE_NAME];
                UISaveVideoAtPathToSavedPhotosAlbum (moviePath, nil, nil, nil);

            }

    }





-(void) pauseRecording
    {
        // needs to be implemented
    }




-(NSString*) pathToDocumentsDirectory 
    {
        NSArray *paths = NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES);
        NSString *documentsDirectory = [paths objectAtIndex:0];
        return documentsDirectory;

    }


- (void)viewDidUnload
    {
        [super viewDidUnload];
        // Release any retained subviews of the main view.
        // e.g. self.myOutlet = nil;
    }

- (BOOL)shouldAutorotateToInterfaceOrientation:(UIInterfaceOrientation)interfaceOrientation
    {
        // Return YES for supported orientations
        return YES;
    }

@end

就是这样。如果任何人都可以修改此代码来录制麦克风音频，我相信除了我之外还有很多人会感激它。添加音频后，这将成为一个直接的框架，可以在应用程序屏幕转换中进行，以便开发人员演示他们的应用程序。

仅供引用:我的代码是开源的，很大程度上是根据 http://www.subfurther.com/blog/2011/04/12/voices-that-matter-iphone-spring-2011/ 的 VTMScreenRecorderTest 项目建模的。 。我进行了小幅度的修改并清理了内存泄漏。

Answer 1

我刚刚发现，如果您想将麦克风与录制的视频一起使用，则 CMTimestamp 必须同步。我尝试通过使用 CMSampleBufferSetOutputPresentationTimeStamp 更改音频上的采样时间来同步它们...但我遗漏了一些东西或者它不起作用。如果您捕获麦克风开始的时间并将其作为偏移量添加到视频中，则 StackOverflow 上的 AVAssetWriter 的每个示例似乎都能正常工作 -

以下是我为保持视频与麦克风同步所做的事情:

Img = Image
Snd = Sound
Avmedia = AVAssetWriter
etc.
#define Cmtm__Pref_Timescale_uk 10000
          CMTime Duration_cmtm = CMTimeMakeWithSeconds( Avmedia_v->Img_Duration__Sec_df, Cmtm__Pref_Timescale_uk );
          Avmedia_v->Img_cmtm = CMTimeAdd( Duration_cmtm, Avmedia_v->Exprt__Begin_cmtm );

          if( [ Avmedia_v->Clrbuf__Adaptor_v appendPixelBuffer: Clrbuf_v withPresentationTime:
               Avmedia_v->Img_cmtm ] is no )
          {
            //If the operation was unsuccessful,
            //invoke the AVAssetWriter object’s finishWriting method in order to save a partially completed asset.
            [ Avmedia_v->Avwriter_v finishWriting ];

            CLog( Lv_Minor, "Movie Img exprt failed" );
          }