From 52b24b51f22fb19aa897ac982455ad77587c7cdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yi=C4=9Fithan=20Y=C3=BCceda=C4=9F?= Date: Sun, 28 Jul 2024 16:55:12 +0300 Subject: [PATCH] refactor: add audio listener, update api --- android/cpp-adapter.cpp | 25 ++-- android/src/main/AndroidManifest.xml | 4 +- android/src/main/AndroidManifestNew.xml | 1 + .../src/main/java/com/pitchy/PitchyModule.kt | 111 ++++++++++++++++-- cpp/react-native-pitchy.cpp | 23 ++-- cpp/react-native-pitchy.h | 3 +- ios/Pitchy.h | 6 +- ios/Pitchy.mm | 95 +++++++++++---- src/index.tsx | 73 ++++++++---- 9 files changed, 245 insertions(+), 96 deletions(-) diff --git a/android/cpp-adapter.cpp b/android/cpp-adapter.cpp index c78c906..9d487bd 100644 --- a/android/cpp-adapter.cpp +++ b/android/cpp-adapter.cpp @@ -7,21 +7,16 @@ #include extern "C" JNIEXPORT jdouble JNICALL -Java_com_pitchy_PitchyModule_nativeAutoCorrelate(JNIEnv *env, jobject /* this */, jdoubleArray buf, jdouble sampleRate) +Java_com_pitchy_PitchyModule_nativeAutoCorrelate(JNIEnv *env, jobject thiz, jshortArray buffer, jint sampleRate, jdouble minVolume) { - jsize len = env->GetArrayLength(buf); - jdouble *bufArray = env->GetDoubleArrayElements(buf, 0); - std::vector vec(bufArray, bufArray + len); - env->ReleaseDoubleArrayElements(buf, bufArray, 0); - return pitchy::autoCorrelate(vec, sampleRate); -} + // pitchy::autoCorrelate(const std::vector &buf, double sampleRate, double minVolume) + // Convert jshortArray to std::vector + jshort *buf = env->GetShortArrayElements(buffer, 0); + jsize size = env->GetArrayLength(buffer); + std::vector vec(buf, buf + size); + env->ReleaseShortArrayElements(buffer, buf, 0); -extern "C" JNIEXPORT jdouble JNICALL -Java_com_pitchy_PitchyModule_nativeCalculateVolume(JNIEnv *env, jobject /* this */, jdoubleArray buf) -{ - jsize len = env->GetArrayLength(buf); - jdouble *bufArray = env->GetDoubleArrayElements(buf, 0); - std::vector vec(bufArray, bufArray + len); - env->ReleaseDoubleArrayElements(buf, bufArray, 0); - return pitchy::calculateVolume(vec); + // Call pitchy::autoCorrelate + double result = pitchy::autoCorrelate(vec, sampleRate, minVolume); + return result; } \ No newline at end of file diff --git a/android/src/main/AndroidManifest.xml b/android/src/main/AndroidManifest.xml index f9d7fd4..d5bfed9 100644 --- a/android/src/main/AndroidManifest.xml +++ b/android/src/main/AndroidManifest.xml @@ -1,3 +1,3 @@ - + + diff --git a/android/src/main/AndroidManifestNew.xml b/android/src/main/AndroidManifestNew.xml index a2f47b6..ed7aac4 100644 --- a/android/src/main/AndroidManifestNew.xml +++ b/android/src/main/AndroidManifestNew.xml @@ -1,2 +1,3 @@ + diff --git a/android/src/main/java/com/pitchy/PitchyModule.kt b/android/src/main/java/com/pitchy/PitchyModule.kt index c117356..89f3223 100644 --- a/android/src/main/java/com/pitchy/PitchyModule.kt +++ b/android/src/main/java/com/pitchy/PitchyModule.kt @@ -4,31 +4,120 @@ import com.facebook.react.bridge.ReactApplicationContext import com.facebook.react.bridge.ReactContextBaseJavaModule import com.facebook.react.bridge.ReactMethod import com.facebook.react.bridge.Promise -import com.facebook.react.bridge.ReadableArray +import com.facebook.react.bridge.ReadableMap +import com.facebook.react.bridge.Arguments +import com.facebook.react.bridge.WritableMap +import com.facebook.react.modules.core.DeviceEventManagerModule + +import android.media.AudioRecord +import android.media.AudioFormat +import android.media.MediaRecorder + +import kotlin.concurrent.thread class PitchyModule(reactContext: ReactApplicationContext) : ReactContextBaseJavaModule(reactContext) { + private var isRecording = false + private var isInitialized = false + + private var audioRecord: AudioRecord? = null + private var recordingThread: Thread? = null + + private var sampleRate: Int = 44100 + + private var minVolume: Double = 0.0 + private var bufferSize: Int = 0 + override fun getName(): String { return NAME } @ReactMethod - fun autoCorrelate(buf: ReadableArray, sampleRate: Double, promise: Promise) { - val bufArray = buf.toArrayList().map { it as Double }.toDoubleArray() - val pitch = nativeAutoCorrelate(bufArray, sampleRate) - promise.resolve(pitch) + fun init(config: ReadableMap) { + minVolume = config.getDouble("minVolume") + bufferSize = config.getInt("bufferSize") + + audioRecord = AudioRecord(MediaRecorder.AudioSource.MIC, sampleRate, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, bufferSize) + + isInitialized = true + } + + @ReactMethod + fun isRecording(promise: Promise) { + promise.resolve(isRecording) } @ReactMethod - fun calculateVolume(buf: ReadableArray, promise: Promise) { - val bufArray = buf.toArrayList().map { it as Double }.toDoubleArray() - val volume = nativeCalculateVolume(bufArray) - promise.resolve(volume) + fun addListener(eventName: String) { + // Keep: Required for RN built in Event Emitter Calls. } - private external fun nativeAutoCorrelate(buf: DoubleArray, sampleRate: Double): Double - private external fun nativeCalculateVolume(buf: DoubleArray): Double + @ReactMethod + fun removeListeners(count: Int) { + // Keep: Required for RN built in Event Emitter Calls. + } + + @ReactMethod + fun start(promise: Promise) { + + if(!isInitialized) { + promise.reject("E_NOT_INITIALIZED", "Not initialized") + return + } + + if (isRecording) { + promise.reject("E_ALREADY_RECORDING", "Already recording") + return + } + + startRecording() + promise.resolve(true) + } + + @ReactMethod + fun stop(promise: Promise) { + if (!isRecording) { + promise.reject("E_NOT_RECORDING", "Not recording") + return + } + + stopRecording() + promise.resolve(true) + } + + private fun startRecording(){ + audioRecord?.startRecording() + isRecording = true + + recordingThread = thread(start = true) { + val buffer = ShortArray(bufferSize) + while (isRecording) { + val read = audioRecord?.read(buffer, 0, bufferSize) + if (read != null && read > 0) { + detectPitch(buffer) + } + } + } + } + + private fun stopRecording() { + isRecording = false + audioRecord?.stop() + audioRecord?.release() + audioRecord = null + recordingThread?.interrupt() + recordingThread = null + } + + private external fun nativeAutoCorrelate(buffer: ShortArray, sampleRate: Int, minVolume: Double): Double + + private fun detectPitch(buffer: ShortArray){ + val pitch = nativeAutoCorrelate(buffer, sampleRate, minVolume) + val params: WritableMap = Arguments.createMap() + params.putDouble("pitch", pitch) + reactApplicationContext.getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter::class.java).emit("onPitchDetected", params) + } companion object { const val NAME = "Pitchy" diff --git a/cpp/react-native-pitchy.cpp b/cpp/react-native-pitchy.cpp index a5a7025..6500faf 100644 --- a/cpp/react-native-pitchy.cpp +++ b/cpp/react-native-pitchy.cpp @@ -8,9 +8,13 @@ namespace pitchy { - double autoCorrelate(const std::vector &buf, double sampleRate) + double getVolumeDecibel(double rms) + { + return 20 * std::log10(rms); + } + + double autoCorrelate(const std::vector &buf, double sampleRate, double minVolume) { - // Implements the ACF2+ algorithm int SIZE = buf.size(); double rms = 0; @@ -20,7 +24,8 @@ namespace pitchy rms += val * val; } rms = std::sqrt(rms / SIZE); - if (rms < 0.01) // not enough signal + double decibel = getVolumeDecibel(rms); + if (decibel < minVolume) return -1; int r1 = 0, r2 = SIZE - 1; @@ -69,16 +74,4 @@ namespace pitchy return sampleRate / T0; } - double calculateVolume(const std::vector &buf) - { - // Calculate RMS (Root Mean Square) - double sumSquares = std::accumulate(buf.begin(), buf.end(), 0.0, [](double a, double b) - { return a + b * b; }); - double rms = std::sqrt(sumSquares / buf.size()); - - // Convert RMS to decibels - double decibels = 20 * std::log10(rms); - - return decibels; - } } diff --git a/cpp/react-native-pitchy.h b/cpp/react-native-pitchy.h index 8ece25c..fd45398 100644 --- a/cpp/react-native-pitchy.h +++ b/cpp/react-native-pitchy.h @@ -5,8 +5,7 @@ namespace pitchy { - double autoCorrelate(const std::vector &buf, double sampleRate); - double calculateVolume(const std::vector &buf); + double autoCorrelate(const std::vector &buf, double sampleRate, double minVolume); } #endif /* PITCHY_H */ diff --git a/ios/Pitchy.h b/ios/Pitchy.h index f6a52ce..85cabb2 100644 --- a/ios/Pitchy.h +++ b/ios/Pitchy.h @@ -4,12 +4,14 @@ #ifdef RCT_NEW_ARCH_ENABLED #import "RNPitchySpec.h" +#import -@interface Pitchy : NSObject +@interface Pitchy : RCTEventEmitter #else #import +#import -@interface Pitchy : NSObject +@interface Pitchy : RCTEventEmitter #endif @end diff --git a/ios/Pitchy.mm b/ios/Pitchy.mm index 5811826..4a5f14f 100644 --- a/ios/Pitchy.mm +++ b/ios/Pitchy.mm @@ -1,41 +1,84 @@ #import "Pitchy.h" +#import + +@implementation Pitchy { + AVAudioEngine *audioEngine; + double sampleRate; + double minVolume; + BOOL isRecording; + BOOL isInitialized; +} -@implementation Pitchy RCT_EXPORT_MODULE() -// Method to expose autoCorrelate -RCT_EXPORT_METHOD(autoCorrelate:(NSArray *)buf - sampleRate:(double)sampleRate - resolve:(RCTPromiseResolveBlock)resolve - reject:(RCTPromiseRejectBlock)reject) -{ - // Convert NSArray to std::vector - std::vector cBuf; - for (NSNumber *num in buf) { - cBuf.push_back([num doubleValue]); +- (NSArray *)supportedEvents { + return @[@"onPitchDetected"]; +} + +RCT_EXPORT_METHOD(init:(NSDictionary *)config) { + if (!isInitialized) { + audioEngine = [[AVAudioEngine alloc] init]; + AVAudioInputNode *inputNode = [audioEngine inputNode]; + + AVAudioFormat *format = [inputNode inputFormatForBus:0]; + sampleRate = format.sampleRate; + minVolume = [config[@"minVolume"] doubleValue]; + + [inputNode installTapOnBus:0 bufferSize:[config[@"bufferSize"] unsignedIntValue] format:format block:^(AVAudioPCMBuffer * _Nonnull buffer, AVAudioTime * _Nonnull when) { + [self detectPitch:buffer]; + }]; + + isInitialized = YES; } +} - // Call the autoCorrelate function - double result = pitchy::autoCorrelate(cBuf, sampleRate); +RCT_EXPORT_METHOD(isRecording:(RCTPromiseResolveBlock)resolve + reject:(RCTPromiseRejectBlock)reject) { + resolve([NSNumber numberWithBool:isRecording]); +} - resolve([NSNumber numberWithDouble:result]); +RCT_EXPORT_METHOD(start:(RCTPromiseResolveBlock)resolve + reject:(RCTPromiseRejectBlock)reject) { + if (!isInitialized) { + reject(@"not_initialized", @"Pitchy module is not initialized", nil); + return; + } + + if(isRecording){ + reject(@"already_recording", @"Already recording", nil); + return; + } + + NSError *error = nil; + [audioEngine startAndReturnError:&error]; + if (error) { + reject(@"start_error", @"Failed to start audio engine", error); + } else { + isRecording = YES; + resolve(@(YES)); + } } -// Method to expose calculateVolume -RCT_EXPORT_METHOD(calculateVolume:(NSArray *)buf - resolve:(RCTPromiseResolveBlock)resolve - reject:(RCTPromiseRejectBlock)reject) -{ - // Convert NSArray to std::vector - std::vector cBuf; - for (NSNumber *num in buf) { - cBuf.push_back([num doubleValue]); +RCT_EXPORT_METHOD(stop:(RCTPromiseResolveBlock)resolve + reject:(RCTPromiseRejectBlock)reject) { + + if (!isRecording) { + reject(@"not_recording", @"Not recording", nil); + return; } - // Call the calculateVolume function - double result = pitchy::calculateVolume(cBuf); + [audioEngine stop]; + isRecording = NO; + resolve(@(YES)); +} + +- (void)detectPitch:(AVAudioPCMBuffer *)buffer { + float *channelData = buffer.floatChannelData[0]; + std::vector buf(channelData, channelData + buffer.frameLength); - resolve([NSNumber numberWithDouble:result]); + double detectedPitch = pitchy::autoCorrelate(buf, sampleRate, minVolume); + + [self sendEventWithName:@"onPitchDetected" body:@{@"pitch": @(detectedPitch)}]; } @end diff --git a/src/index.tsx b/src/index.tsx index eb2f974..9a88324 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -1,4 +1,4 @@ -import { NativeModules, Platform } from 'react-native'; +import { NativeModules, NativeEventEmitter, Platform } from 'react-native'; const LINKING_ERROR = `The package 'react-native-pitchy' doesn't seem to be linked. Make sure: \n\n` + @@ -6,7 +6,7 @@ const LINKING_ERROR = '- You rebuilt the app after installing the package\n' + '- You are not using Expo Go\n'; -const Pitchy = NativeModules.Pitchy +const PitchyNativeModule = NativeModules.Pitchy ? NativeModules.Pitchy : new Proxy( {}, @@ -17,24 +17,51 @@ const Pitchy = NativeModules.Pitchy } ); -/** - * Detects the pitch of the audio data in the buffer. - * @param buf The audio data buffer. - * @param sampleRate The sample rate of the audio data. - * @returns A promise that resolves to the detected pitch in Hz. - */ -export function autoCorrelate( - buf: ArrayLike, - sampleRate: number -): Promise { - return Pitchy.autoCorrelate(buf, sampleRate); -} - -/** - * Calculates the volume of the audio data in the buffer. - * @param buf The audio data buffer. - * @returns A promise that resolves to the calculated volume in dB. - */ -export function calculateVolume(buf: ArrayLike): Promise { - return Pitchy.calculateVolume(buf); -} +const eventEmitter = new NativeEventEmitter(PitchyNativeModule); + +export type PitchyAlgorithm = 'ACF2+'; + +export type PitchyConfig = { + /** + * The size of the buffer used to record audio. + * @default 4096 + */ + bufferSize?: number; + /** + * The minimum volume required to start detecting pitch. + * @default 45 + */ + minVolume?: number; + /** + * The algorithm used to detect pitch. + * @default 'ACF2+' + */ + algorithm?: PitchyAlgorithm; +}; + +export type PitchyEventCallback = ({ pitch }: { pitch: number }) => void; + +const Pitchy = { + init(config?: PitchyConfig) { + return PitchyNativeModule.init({ + bufferSize: 4096, + minVolume: -60, + algorithm: 'ACF2+', + ...config, + }); + }, + start(): Promise { + return PitchyNativeModule.start(); + }, + stop(): Promise { + return PitchyNativeModule.stop(); + }, + isRecording(): Promise { + return PitchyNativeModule.isRecording(); + }, + addListener(callback: PitchyEventCallback) { + return eventEmitter.addListener('onPitchDetected', callback); + }, +}; + +export default Pitchy;