From dfe6c736218dd7f18eccb32a231fe9628a749447 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9D=D0=B8=D0=BA=D0=BE=D0=BB=D0=B0=D0=B8=CC=86=20=D0=90?= =?UTF-8?q?=D1=88=D0=B0=D0=BD=D0=B8=D0=BD?= Date: Mon, 8 Feb 2016 00:21:29 +0300 Subject: [PATCH] add vizualizer for player buffers --- .../StreamingKit.xcodeproj/project.pbxproj | 8 + StreamingKit/StreamingKit/AEFloatConverter.h | 124 ++++++++++ StreamingKit/StreamingKit/AEFloatConverter.m | 211 ++++++++++++++++++ StreamingKit/StreamingKit/STKAudioPlayer.h | 2 + StreamingKit/StreamingKit/STKAudioPlayer.m | 24 ++ 5 files changed, 369 insertions(+) create mode 100644 StreamingKit/StreamingKit/AEFloatConverter.h create mode 100644 StreamingKit/StreamingKit/AEFloatConverter.m diff --git a/StreamingKit/StreamingKit.xcodeproj/project.pbxproj b/StreamingKit/StreamingKit.xcodeproj/project.pbxproj index 5f8109d..0acafa6 100644 --- a/StreamingKit/StreamingKit.xcodeproj/project.pbxproj +++ b/StreamingKit/StreamingKit.xcodeproj/project.pbxproj @@ -15,6 +15,8 @@ 5B949CD71A1140E4005675A0 /* STKHTTPDataSource.h in Headers */ = {isa = PBXBuildFile; fileRef = A1E7C4FB188D5E550010896F /* STKHTTPDataSource.h */; settings = {ATTRIBUTES = (Public, ); }; }; 5B949CD81A1140E4005675A0 /* STKLocalFileDataSource.h in Headers */ = {isa = PBXBuildFile; fileRef = A1E7C4FD188D5E550010896F /* STKLocalFileDataSource.h */; settings = {ATTRIBUTES = (Public, ); }; }; 5B949CD91A1140E4005675A0 /* STKQueueEntry.h in Headers */ = {isa = PBXBuildFile; fileRef = A1BF65D0189A6582004DD08C /* STKQueueEntry.h */; settings = {ATTRIBUTES = (Public, ); }; }; + 958A8E291C67E9B700AA861D /* AEFloatConverter.h in Headers */ = {isa = PBXBuildFile; fileRef = 958A8E271C67E9B700AA861D /* AEFloatConverter.h */; }; + 958A8E2A1C67E9B700AA861D /* AEFloatConverter.m in Sources */ = {isa = PBXBuildFile; fileRef = 958A8E281C67E9B700AA861D /* AEFloatConverter.m */; }; A1A4996B189E744400E2A2E2 /* Cocoa.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = A1A4996A189E744400E2A2E2 /* Cocoa.framework */; }; A1A49975189E744500E2A2E2 /* StreamingKitMac.m in Sources */ = {isa = PBXBuildFile; fileRef = A1A49974189E744500E2A2E2 /* StreamingKitMac.m */; }; A1A4997B189E744500E2A2E2 /* XCTest.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = A1E7C4D9188D57F60010896F /* XCTest.framework */; }; @@ -91,6 +93,8 @@ /* End PBXCopyFilesBuildPhase section */ /* Begin PBXFileReference section */ + 958A8E271C67E9B700AA861D /* AEFloatConverter.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AEFloatConverter.h; sourceTree = ""; }; + 958A8E281C67E9B700AA861D /* AEFloatConverter.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = AEFloatConverter.m; sourceTree = ""; }; A1A49969189E744400E2A2E2 /* libStreamingKitMac.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libStreamingKitMac.a; sourceTree = BUILT_PRODUCTS_DIR; }; A1A4996A189E744400E2A2E2 /* Cocoa.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Cocoa.framework; path = Library/Frameworks/Cocoa.framework; sourceTree = DEVELOPER_DIR; }; A1A4996D189E744500E2A2E2 /* Foundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Foundation.framework; path = Library/Frameworks/Foundation.framework; sourceTree = SDKROOT; }; @@ -270,6 +274,8 @@ A1E7C4CD188D57F50010896F /* StreamingKit */ = { isa = PBXGroup; children = ( + 958A8E271C67E9B700AA861D /* AEFloatConverter.h */, + 958A8E281C67E9B700AA861D /* AEFloatConverter.m */, A1E7C4F1188D5E550010896F /* STKAudioPlayer.h */, A1E7C4F2188D5E550010896F /* STKAudioPlayer.m */, A1E7C4F3188D5E550010896F /* STKAutoRecoveringHTTPDataSource.h */, @@ -331,6 +337,7 @@ 5B949CD41A1140E4005675A0 /* STKCoreFoundationDataSource.h in Headers */, 5B949CD51A1140E4005675A0 /* STKDataSource.h in Headers */, 5B949CD61A1140E4005675A0 /* STKDataSourceWrapper.h in Headers */, + 958A8E291C67E9B700AA861D /* AEFloatConverter.h in Headers */, 5B949CD71A1140E4005675A0 /* STKHTTPDataSource.h in Headers */, 5B949CD81A1140E4005675A0 /* STKLocalFileDataSource.h in Headers */, 5B949CD91A1140E4005675A0 /* STKQueueEntry.h in Headers */, @@ -525,6 +532,7 @@ A1E7C505188D5E550010896F /* STKLocalFileDataSource.m in Sources */, A1BF65D2189A6582004DD08C /* STKQueueEntry.m in Sources */, A1E7C504188D5E550010896F /* STKHTTPDataSource.m in Sources */, + 958A8E2A1C67E9B700AA861D /* AEFloatConverter.m in Sources */, A1E7C503188D5E550010896F /* STKDataSourceWrapper.m in Sources */, A1E7C502188D5E550010896F /* STKDataSource.m in Sources */, A1BF65D5189A65C6004DD08C /* NSMutableArray+STKAudioPlayer.m in Sources */, diff --git a/StreamingKit/StreamingKit/AEFloatConverter.h b/StreamingKit/StreamingKit/AEFloatConverter.h new file mode 100644 index 0000000..8b2e55c --- /dev/null +++ b/StreamingKit/StreamingKit/AEFloatConverter.h @@ -0,0 +1,124 @@ +// +// AEFloatConverter.h +// The Amazing Audio Engine +// +// Created by Michael Tyson on 25/10/2012. +// +// This software is provided 'as-is', without any express or implied +// warranty. In no event will the authors be held liable for any damages +// arising from the use of this software. +// +// Permission is granted to anyone to use this software for any purpose, +// including commercial applications, and to alter it and redistribute it +// freely, subject to the following restrictions: +// +// 1. The origin of this software must not be misrepresented; you must not +// claim that you wrote the original software. If you use this software +// in a product, an acknowledgment in the product documentation would be +// appreciated but is not required. +// +// 2. Altered source versions must be plainly marked as such, and must not be +// misrepresented as being the original software. +// +// 3. This notice may not be removed or altered from any source distribution. +// + +#ifdef __cplusplus +extern "C" { +#endif + +#import +#import + + /*! + * Universal converter to float format + * + * Use this class to easily convert arbitrary audio formats to floating point + * for use with utilities like the Accelerate framework. + */ + @interface AEFloatConverter : NSObject + +/*! + * Initialize + * + * @param sourceFormat The audio format to use + */ +- (id)initWithSourceFormat:(AudioStreamBasicDescription)sourceFormat; + +/*! + * Convert audio to floating-point + * + * This C function, safe to use in a Core Audio realtime thread context, will take + * an audio buffer list of audio in the format you provided at initialisation, and + * convert it into a noninterleaved float array. + * + * @param converter Pointer to the converter object. + * @param sourceBuffer An audio buffer list containing the source audio. + * @param targetBuffers An array of floating-point arrays to store the converted float audio into. + * Note that you must provide the correct number of arrays, to match the number of channels. + * @param frames The number of frames to convert. + * @return YES on success; NO on failure + */ +BOOL AEFloatConverterToFloat(AEFloatConverter* converter, AudioBufferList *sourceBuffer, float * const * targetBuffers, UInt32 frames); + +/*! + * Convert audio to floating-point, in a buffer list + * + * This C function, safe to use in a Core Audio realtime thread context, will take + * an audio buffer list of audio in the format you provided at initialisation, and + * convert it into a noninterleaved float format. + * + * @param converter Pointer to the converter object. + * @param sourceBuffer An audio buffer list containing the source audio. + * @param targetBuffer An audio buffer list to store the converted floating-point audio. + * @param frames The number of frames to convert. + * @return YES on success; NO on failure + */ +BOOL AEFloatConverterToFloatBufferList(AEFloatConverter* converter, AudioBufferList *sourceBuffer, AudioBufferList *targetBuffer, UInt32 frames); + +/*! + * Convert audio from floating-point + * + * This C function, safe to use in a Core Audio realtime thread context, will take + * an audio buffer list of audio in the format you provided at initialisation, and + * convert it into a float array. + * + * @param converter Pointer to the converter object. + * @param sourceBuffers An array of floating-point arrays containing the floating-point audio to convert. + * Note that you must provide the correct number of arrays, to match the number of channels. + * @param targetBuffer An audio buffer list to store the converted audio into. + * @param frames The number of frames to convert. + * @return YES on success; NO on failure + */ +BOOL AEFloatConverterFromFloat(AEFloatConverter* converter, float * const * sourceBuffers, AudioBufferList *targetBuffer, UInt32 frames); + +/*! + * Convert audio from floating-point, in a buffer list + * + * This C function, safe to use in a Core Audio realtime thread context, will take + * an audio buffer list of audio in the format you provided at initialisation, and + * convert it into a float array. + * + * @param converter Pointer to the converter object. + * @param sourceBuffer An audio buffer list containing the source audio. + * @param targetBuffer An audio buffer list to store the converted audio into. + * @param frames The number of frames to convert. + * @return YES on success; NO on failure + */ +BOOL AEFloatConverterFromFloatBufferList(AEFloatConverter* converter, AudioBufferList *sourceBuffer, AudioBufferList *targetBuffer, UInt32 frames); + +/*! + * The AudioStreamBasicDescription representing the converted floating-point format + */ +@property (nonatomic, readonly) AudioStreamBasicDescription floatingPointAudioDescription; + +/*! + * The source audio format set at initialization + */ +@property (nonatomic, readonly) AudioStreamBasicDescription sourceFormat; + +@end + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/StreamingKit/StreamingKit/AEFloatConverter.m b/StreamingKit/StreamingKit/AEFloatConverter.m new file mode 100644 index 0000000..e1634c8 --- /dev/null +++ b/StreamingKit/StreamingKit/AEFloatConverter.m @@ -0,0 +1,211 @@ +// +// AEFloatConverter.m +// The Amazing Audio Engine +// +// Created by Michael Tyson on 25/10/2012. +// +// This software is provided 'as-is', without any express or implied +// warranty. In no event will the authors be held liable for any damages +// arising from the use of this software. +// +// Permission is granted to anyone to use this software for any purpose, +// including commercial applications, and to alter it and redistribute it +// freely, subject to the following restrictions: +// +// 1. The origin of this software must not be misrepresented; you must not +// claim that you wrote the original software. If you use this software +// in a product, an acknowledgment in the product documentation would be +// appreciated but is not required. +// +// 2. Altered source versions must be plainly marked as such, and must not be +// misrepresented as being the original software. +// +// 3. This notice may not be removed or altered from any source distribution. +// + +#import "AEFloatConverter.h" + +#define checkResult(result,operation) (_checkResult((result),(operation),strrchr(__FILE__, '/')+1,__LINE__)) +static inline BOOL _checkResult(OSStatus result, const char *operation, const char* file, int line) { + if ( result != noErr ) { + NSLog(@"%s:%d: %s result %d %08X %4.4s", file, line, operation, (int)result, (int)result, (char*)&result); + return NO; + } + return YES; +} + +#define kNoMoreDataErr -2222 + +struct complexInputDataProc_t { + AudioBufferList *sourceBuffer; +}; + +@interface AEFloatConverter () { + AudioStreamBasicDescription _sourceAudioDescription; + AudioStreamBasicDescription _floatAudioDescription; + AudioConverterRef _toFloatConverter; + AudioConverterRef _fromFloatConverter; + AudioBufferList *_scratchFloatBufferList; +} + +static OSStatus complexInputDataProc(AudioConverterRef inAudioConverter, + UInt32 *ioNumberDataPackets, + AudioBufferList *ioData, + AudioStreamPacketDescription **outDataPacketDescription, + void *inUserData); +@end + +@implementation AEFloatConverter +@synthesize sourceFormat = _sourceAudioDescription; + +-(id)initWithSourceFormat:(AudioStreamBasicDescription)sourceFormat { + if ( !(self = [super init]) ) return nil; + + _floatAudioDescription.mFormatID = kAudioFormatLinearPCM; + _floatAudioDescription.mFormatFlags = kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked | kAudioFormatFlagIsNonInterleaved; + _floatAudioDescription.mChannelsPerFrame = sourceFormat.mChannelsPerFrame; + _floatAudioDescription.mBytesPerPacket = sizeof(float); + _floatAudioDescription.mFramesPerPacket = 1; + _floatAudioDescription.mBytesPerFrame = sizeof(float); + _floatAudioDescription.mBitsPerChannel = 8 * sizeof(float); + _floatAudioDescription.mSampleRate = sourceFormat.mSampleRate; + + _sourceAudioDescription = sourceFormat; + + if ( memcmp(&sourceFormat, &_floatAudioDescription, sizeof(AudioStreamBasicDescription)) != 0 ) { + checkResult(AudioConverterNew(&sourceFormat, &_floatAudioDescription, &_toFloatConverter), "AudioConverterNew"); + checkResult(AudioConverterNew(&_floatAudioDescription, &sourceFormat, &_fromFloatConverter), "AudioConverterNew"); + _scratchFloatBufferList = (AudioBufferList*)malloc(sizeof(AudioBufferList) + (_floatAudioDescription.mChannelsPerFrame-1)*sizeof(AudioBuffer)); + _scratchFloatBufferList->mNumberBuffers = _floatAudioDescription.mChannelsPerFrame; + for ( int i=0; i<_scratchFloatBufferList->mNumberBuffers; i++ ) { + _scratchFloatBufferList->mBuffers[i].mNumberChannels = 1; + } + } + + return self; +} + +-(void)dealloc { + if ( _toFloatConverter ) AudioConverterDispose(_toFloatConverter); + if ( _fromFloatConverter ) AudioConverterDispose(_fromFloatConverter); + if ( _scratchFloatBufferList ) free(_scratchFloatBufferList); + // [super dealloc]; +} + + +BOOL AEFloatConverterToFloat(AEFloatConverter* THIS, AudioBufferList *sourceBuffer, float * const * targetBuffers, UInt32 frames) { + if ( frames == 0 ) return YES; + + if ( THIS->_toFloatConverter ) { + UInt32 priorDataByteSize = sourceBuffer->mBuffers[0].mDataByteSize; + for ( int i=0; imNumberBuffers; i++ ) { + sourceBuffer->mBuffers[i].mDataByteSize = frames * THIS->_sourceAudioDescription.mBytesPerFrame; + } + + for ( int i=0; i_scratchFloatBufferList->mNumberBuffers; i++ ) { + THIS->_scratchFloatBufferList->mBuffers[i].mData = targetBuffers[i]; + THIS->_scratchFloatBufferList->mBuffers[i].mDataByteSize = frames * sizeof(float); + } + + OSStatus result = AudioConverterFillComplexBuffer(THIS->_toFloatConverter, + complexInputDataProc, + &(struct complexInputDataProc_t) { .sourceBuffer = sourceBuffer }, + &frames, + THIS->_scratchFloatBufferList, + NULL); + + for ( int i=0; imNumberBuffers; i++ ) { + sourceBuffer->mBuffers[i].mDataByteSize = priorDataByteSize; + } + + if ( !checkResult(result, "AudioConverterConvertComplexBuffer") ) { + return NO; + } + + } else { + for ( int i=0; imNumberBuffers; i++ ) { + memcpy(targetBuffers[i], sourceBuffer->mBuffers[i].mData, frames * sizeof(float)); + } + } + + return YES; +} + +BOOL AEFloatConverterToFloatBufferList(AEFloatConverter* converter, AudioBufferList *sourceBuffer, AudioBufferList *targetBuffer, UInt32 frames) { + assert(targetBuffer->mNumberBuffers == converter->_floatAudioDescription.mChannelsPerFrame); + + float *targetBuffers[targetBuffer->mNumberBuffers]; + for ( int i=0; imNumberBuffers; i++ ) { + targetBuffers[i] = (float*)targetBuffer->mBuffers[i].mData; + } + return AEFloatConverterToFloat(converter, sourceBuffer, targetBuffers, frames); +} + +BOOL AEFloatConverterFromFloat(AEFloatConverter* THIS, float * const * sourceBuffers, AudioBufferList *targetBuffer, UInt32 frames) { + if ( frames == 0 ) return YES; + + if ( THIS->_fromFloatConverter ) { + for ( int i=0; i_scratchFloatBufferList->mNumberBuffers; i++ ) { + THIS->_scratchFloatBufferList->mBuffers[i].mData = sourceBuffers[i]; + THIS->_scratchFloatBufferList->mBuffers[i].mDataByteSize = frames * sizeof(float); + } + + UInt32 priorDataByteSize = targetBuffer->mBuffers[0].mDataByteSize; + for ( int i=0; imNumberBuffers; i++ ) { + targetBuffer->mBuffers[i].mDataByteSize = frames * THIS->_sourceAudioDescription.mBytesPerFrame; + } + + OSStatus result = AudioConverterFillComplexBuffer(THIS->_fromFloatConverter, + complexInputDataProc, + &(struct complexInputDataProc_t) { .sourceBuffer = THIS->_scratchFloatBufferList }, + &frames, + targetBuffer, + NULL); + + for ( int i=0; imNumberBuffers; i++ ) { + targetBuffer->mBuffers[i].mDataByteSize = priorDataByteSize; + } + + if ( !checkResult(result, "AudioConverterConvertComplexBuffer") ) { + return NO; + } + } else { + for ( int i=0; imNumberBuffers; i++ ) { + memcpy(targetBuffer->mBuffers[i].mData, sourceBuffers[i], frames * sizeof(float)); + } + } + + return YES; +} + +BOOL AEFloatConverterFromFloatBufferList(AEFloatConverter* converter, AudioBufferList *sourceBuffer, AudioBufferList *targetBuffer, UInt32 frames) { + assert(sourceBuffer->mNumberBuffers == converter->_floatAudioDescription.mChannelsPerFrame); + + float *sourceBuffers[sourceBuffer->mNumberBuffers]; + for ( int i=0; imNumberBuffers; i++ ) { + sourceBuffers[i] = (float*)sourceBuffer->mBuffers[i].mData; + } + return AEFloatConverterFromFloat(converter, sourceBuffers, targetBuffer, frames); +} + +static OSStatus complexInputDataProc(AudioConverterRef inAudioConverter, + UInt32 *ioNumberDataPackets, + AudioBufferList *ioData, + AudioStreamPacketDescription **outDataPacketDescription, + void *inUserData) { + struct complexInputDataProc_t *arg = (struct complexInputDataProc_t*)inUserData; + if ( !arg->sourceBuffer ) { + return kNoMoreDataErr; + } + + memcpy(ioData, arg->sourceBuffer, sizeof(AudioBufferList) + (arg->sourceBuffer->mNumberBuffers-1)*sizeof(AudioBuffer)); + arg->sourceBuffer = NULL; + + return noErr; +} + +-(AudioStreamBasicDescription)floatingPointAudioDescription { + return _floatAudioDescription; +} + +@end \ No newline at end of file diff --git a/StreamingKit/StreamingKit/STKAudioPlayer.h b/StreamingKit/StreamingKit/STKAudioPlayer.h index 0ce6d53..f76ce9d 100644 --- a/StreamingKit/StreamingKit/STKAudioPlayer.h +++ b/StreamingKit/StreamingKit/STKAudioPlayer.h @@ -129,6 +129,8 @@ typedef void(^STKFrameFilter)(UInt32 channelsPerFrame, UInt32 bytesPerFrame, UIn /// Raised when items queued items are cleared (usually because of a call to play, setDataSource or stop) -(void) audioPlayer:(STKAudioPlayer*)audioPlayer didCancelQueuedItems:(NSArray*)queuedItems; +-(void) plotGraphWithBuffer:(float*)buffer andLength:(UInt32)count; + @end @interface STKAudioPlayer : NSObject diff --git a/StreamingKit/StreamingKit/STKAudioPlayer.m b/StreamingKit/StreamingKit/STKAudioPlayer.m index c97ec36..2a1f4c2 100755 --- a/StreamingKit/StreamingKit/STKAudioPlayer.m +++ b/StreamingKit/StreamingKit/STKAudioPlayer.m @@ -41,6 +41,7 @@ #import "NSMutableArray+STKAudioPlayer.h" #import "libkern/OSAtomic.h" #import +#import "AEFloatConverter.h" #ifndef DBL_MAX #define DBL_MAX 1.7976931348623157e+308 @@ -271,6 +272,9 @@ static AudioStreamBasicDescription recordAudioStreamBasicDescription; volatile BOOL disposeWasRequested; volatile BOOL seekToTimeWasRequested; volatile STKAudioPlayerStopReason stopReason; + + float **_floatBuffers; + AEFloatConverter *_floatConverter; } @property (readwrite) STKAudioPlayerInternalState internalState; @@ -526,7 +530,21 @@ static void AudioFileStreamPacketsProc(void* clientData, UInt32 numberBytes, UIn upcomingQueue = [[NSMutableArray alloc] init]; bufferingQueue = [[NSMutableArray alloc] init]; + + + //initialie the float converter + // Allocate the float buffers + _floatConverter = [[AEFloatConverter alloc] initWithSourceFormat:canonicalAudioStreamBasicDescription]; + size_t sizeToAllocate = sizeof(float*) * canonicalAudioStreamBasicDescription.mChannelsPerFrame; + sizeToAllocate = MAX(8, sizeToAllocate); + _floatBuffers = (float**)malloc( sizeToAllocate ); + UInt32 outputBufferSize = 32 * 1024; // 32 KB + for ( int i=0; i< canonicalAudioStreamBasicDescription.mChannelsPerFrame; i++ ) { + _floatBuffers[i] = (float*)malloc(outputBufferSize); + } + + [self resetPcmBuffers]; [self createAudioGraph]; [self createPlaybackThread]; @@ -3209,6 +3227,12 @@ static OSStatus OutputRenderCallback(void* inRefCon, AudioUnitRenderActionFlags* { [self appendFrameFilterWithName:@"STKMeteringFilter" block:^(UInt32 channelsPerFrame, UInt32 bytesPerFrame, UInt32 frameCount, void* frames) { + AEFloatConverterToFloat(_floatConverter,&(pcmAudioBufferList),_floatBuffers,frameCount); + + if ([self.delegate respondsToSelector:@selector(plotGraphWithBuffer:andLength:)]) { + [self.delegate plotGraphWithBuffer:*(_floatBuffers) andLength:frameCount]; + } + SInt16* samples16 = (SInt16*)frames; SInt32* samples32 = (SInt32*)frames; UInt32 countLeft = 0;