Updates

deeeed · Jul 20, 2024 · dd346ef · dd346ef
1 parent 91711be
commit dd346ef
Show file tree

Hide file tree

Showing 56 changed files with 2,796 additions and 0 deletions.
diff --git a/playground/.nojekyll b/playground/.nojekyll
diff --git a/playground/404.html b/playground/404.html
@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <meta http-equiv="refresh" content="0; URL='/expo-audio-stream'" />
+  <script>
+    // This script will redirect any 404 errors back to the main index page.
+    window.location.href = '/expo-audio-stream';
+  </script>
+</head>
+<body>
+  <p>If you are not redirected, <a href="/expo-audio-stream">click here</a>.</p>
+</body>
+</html>
diff --git a/playground/_expo/static/js/web/index-28b0f482e240f5e1ca96546bb039b476.js b/playground/_expo/static/js/web/index-28b0f482e240f5e1ca96546bb039b476.js
diff --git a/...ild/vendor/react-native-vector-icons/Fonts/AntDesign.3a2ba31570920eeb9b1d217cabe58315.ttf b/...ild/vendor/react-native-vector-icons/Fonts/AntDesign.3a2ba31570920eeb9b1d217cabe58315.ttf
diff --git a/.../build/vendor/react-native-vector-icons/Fonts/Entypo.31b5ffea3daddc69dd01a1f3d6cf63c5.ttf b/.../build/vendor/react-native-vector-icons/Fonts/Entypo.31b5ffea3daddc69dd01a1f3d6cf63c5.ttf
diff --git a/...ild/vendor/react-native-vector-icons/Fonts/EvilIcons.140c53a7643ea949007aa9a282153849.ttf b/...ild/vendor/react-native-vector-icons/Fonts/EvilIcons.140c53a7643ea949007aa9a282153849.ttf
diff --git a/...build/vendor/react-native-vector-icons/Fonts/Feather.a76d309774d33d9856f650bed4292a23.ttf b/...build/vendor/react-native-vector-icons/Fonts/Feather.a76d309774d33d9856f650bed4292a23.ttf
diff --git a/...d/vendor/react-native-vector-icons/Fonts/FontAwesome.b06871f281fee6b241d60582ae9369b9.ttf b/...d/vendor/react-native-vector-icons/Fonts/FontAwesome.b06871f281fee6b241d60582ae9369b9.ttf
diff --git a/.../react-native-vector-icons/Fonts/FontAwesome5_Brands.3b89dd103490708d19a95adcae52210e.ttf b/.../react-native-vector-icons/Fonts/FontAwesome5_Brands.3b89dd103490708d19a95adcae52210e.ttf
diff --git a/...react-native-vector-icons/Fonts/FontAwesome5_Regular.1f77739ca9ff2188b539c36f30ffa2be.ttf b/...react-native-vector-icons/Fonts/FontAwesome5_Regular.1f77739ca9ff2188b539c36f30ffa2be.ttf
diff --git a/...r/react-native-vector-icons/Fonts/FontAwesome5_Solid.605ed7926cf39a2ad5ec2d1f9d391d3d.ttf b/...r/react-native-vector-icons/Fonts/FontAwesome5_Solid.605ed7926cf39a2ad5ec2d1f9d391d3d.ttf
diff --git a/.../react-native-vector-icons/Fonts/FontAwesome6_Brands.bb8cd014d7a55672934233c354e1c4a3.ttf b/.../react-native-vector-icons/Fonts/FontAwesome6_Brands.bb8cd014d7a55672934233c354e1c4a3.ttf
diff --git a/...react-native-vector-icons/Fonts/FontAwesome6_Regular.675809e48e35c47d51c7d6fcc687ee28.ttf b/...react-native-vector-icons/Fonts/FontAwesome6_Regular.675809e48e35c47d51c7d6fcc687ee28.ttf
diff --git a/...r/react-native-vector-icons/Fonts/FontAwesome6_Solid.0248ab19e74fc3731de14d647db4687a.ttf b/...r/react-native-vector-icons/Fonts/FontAwesome6_Solid.0248ab19e74fc3731de14d647db4687a.ttf
diff --git a/...uild/vendor/react-native-vector-icons/Fonts/Fontisto.b49ae8ab2dbccb02c4d11caaacf09eab.ttf b/...uild/vendor/react-native-vector-icons/Fonts/Fontisto.b49ae8ab2dbccb02c4d11caaacf09eab.ttf
diff --git a/...ld/vendor/react-native-vector-icons/Fonts/Foundation.e20945d7c929279ef7a6f1db184a4470.ttf b/...ld/vendor/react-native-vector-icons/Fonts/Foundation.e20945d7c929279ef7a6f1db184a4470.ttf
diff --git a/...uild/vendor/react-native-vector-icons/Fonts/Ionicons.6148e7019854f3bde85b633cb88f3c25.ttf b/...uild/vendor/react-native-vector-icons/Fonts/Ionicons.6148e7019854f3bde85b633cb88f3c25.ttf
diff --git a/...act-native-vector-icons/Fonts/MaterialCommunityIcons.b62641afc9ab487008e996a5c5865e56.ttf b/...act-native-vector-icons/Fonts/MaterialCommunityIcons.b62641afc9ab487008e996a5c5865e56.ttf
diff --git a/...vendor/react-native-vector-icons/Fonts/MaterialIcons.4e85bc9ebe07e0340c9c4fc2f6c38908.ttf b/...vendor/react-native-vector-icons/Fonts/MaterialIcons.4e85bc9ebe07e0340c9c4fc2f6c38908.ttf
diff --git a/...uild/vendor/react-native-vector-icons/Fonts/Octicons.f7c53c47a66934504fcbc7cc164895a7.ttf b/...uild/vendor/react-native-vector-icons/Fonts/Octicons.f7c53c47a66934504fcbc7cc164895a7.ttf
diff --git a/...ndor/react-native-vector-icons/Fonts/SimpleLineIcons.d2285965fe34b05465047401b8595dd0.ttf b/...ndor/react-native-vector-icons/Fonts/SimpleLineIcons.d2285965fe34b05465047401b8595dd0.ttf
diff --git a/.../build/vendor/react-native-vector-icons/Fonts/Zocial.1681f34aaca71b8dfb70756bca331eb2.ttf b/.../build/vendor/react-native-vector-icons/Fonts/Zocial.1681f34aaca71b8dfb70756bca331eb2.ttf
diff --git a/.../elements/lib/module/assets/back-icon-mask.5223c8d9b0d08b82a5670fb5f71faf78.png b/.../elements/lib/module/assets/back-icon-mask.5223c8d9b0d08b82a5670fb5f71faf78.png
diff --git a/...ation/elements/lib/module/assets/back-icon.35ba0eaec5a4f5ed12ca16fabeae451d.png b/...ation/elements/lib/module/assets/back-icon.35ba0eaec5a4f5ed12ca16fabeae451d.png
diff --git a/...ts/__node_modules/expo-router/assets/error.563d5e3294b67811d0a1aede6f601e30.png b/...ts/__node_modules/expo-router/assets/error.563d5e3294b67811d0a1aede6f601e30.png
diff --git a/...ets/__node_modules/expo-router/assets/file.b6c297a501e289394b0bc5dc69c265e6.png b/...ets/__node_modules/expo-router/assets/file.b6c297a501e289394b0bc5dc69c265e6.png
diff --git a/.../__node_modules/expo-router/assets/forward.9d9c5644f55c2f6e4b7f247c378b2fe9.png b/.../__node_modules/expo-router/assets/forward.9d9c5644f55c2f6e4b7f247c378b2fe9.png
diff --git a/...sets/__node_modules/expo-router/assets/pkg.5974eb3e1c5314e8d5a822702d7d0740.png b/...sets/__node_modules/expo-router/assets/pkg.5974eb3e1c5314e8d5a822702d7d0740.png
diff --git a/...ges/expo-audio-ui/src/AudioVisualizer/Roboto-Regular.8a36205bd9b83e03af0591a004bc97f4.ttf b/...ges/expo-audio-ui/src/AudioVisualizer/Roboto-Regular.8a36205bd9b83e03af0591a004bc97f4.ttf
diff --git a/playground/assets/assets/Roboto/Roboto-Regular.8a36205bd9b83e03af0591a004bc97f4.ttf b/playground/assets/assets/Roboto/Roboto-Regular.8a36205bd9b83e03af0591a004bc97f4.ttf
diff --git a/playground/assets/assets/icon.ca3508abf03493907293aec8d3c86b66.png b/playground/assets/assets/icon.ca3508abf03493907293aec8d3c86b66.png
diff --git a/playground/audio-features-extractor.js b/playground/audio-features-extractor.js
@@ -0,0 +1,324 @@
+// playground/public/audio-features-extractor.js
+
+// Unique ID counter
+let uniqueIdCounter = 0
+
+self.onmessage = function (event) {
+    const {
+        channelData, // this is only the newly recorded data when live recording.
+        sampleRate,
+        pointsPerSecond,
+        algorithm,
+        bitDepth,
+        fullAudioDurationMs,
+        numberOfChannels,
+        features: _features,
+    } = event.data
+
+    console.log('[AudioFeaturesExtractor] Worker received message', event.data)
+    const features = _features || {}
+
+    const SILENCE_THRESHOLD = 0.01
+    const MIN_SILENCE_DURATION = 1.5 * sampleRate // 1.5 seconds of silence
+    const SPEECH_INERTIA_DURATION = 0.1 * sampleRate // Speech inertia duration in samples
+    const RMS_THRESHOLD = 0.01
+    const ZCR_THRESHOLD = 0.1
+
+    // Placeholder functions for feature extraction
+    const extractMFCC = (segmentData, sampleRate) => {
+        // Implement MFCC extraction logic here
+        return []
+    }
+
+    const extractSpectralCentroid = (segmentData, sampleRate) => {
+        const magnitudeSpectrum = segmentData.map((v) => v * v)
+        const sum = magnitudeSpectrum.reduce((a, b) => a + b, 0)
+        if (sum === 0) return 0
+
+        const weightedSum = magnitudeSpectrum.reduce(
+            (acc, value, index) => acc + index * value,
+            0
+        )
+        return (
+            ((weightedSum / sum) * (sampleRate / 2)) / magnitudeSpectrum.length
+        )
+    }
+
+    const extractSpectralFlatness = (segmentData) => {
+        const magnitudeSpectrum = segmentData.map((v) => Math.abs(v))
+        const geometricMean = Math.exp(
+            magnitudeSpectrum
+                .map((v) => Math.log(v + Number.MIN_VALUE))
+                .reduce((a, b) => a + b) / magnitudeSpectrum.length
+        )
+        const arithmeticMean =
+            magnitudeSpectrum.reduce((a, b) => a + b) / magnitudeSpectrum.length
+        return arithmeticMean === 0 ? 0 : geometricMean / arithmeticMean
+    }
+
+    const extractSpectralRollOff = (segmentData, sampleRate) => {
+        const magnitudeSpectrum = segmentData.map((v) => Math.abs(v))
+        const totalEnergy = magnitudeSpectrum.reduce((a, b) => a + b, 0)
+        const rollOffThreshold = totalEnergy * 0.85
+        let cumulativeEnergy = 0
+
+        for (let i = 0; i < magnitudeSpectrum.length; i++) {
+            cumulativeEnergy += magnitudeSpectrum[i]
+            if (cumulativeEnergy >= rollOffThreshold) {
+                return (i / magnitudeSpectrum.length) * (sampleRate / 2)
+            }
+        }
+
+        return 0
+    }
+
+    const extractSpectralBandwidth = (segmentData, sampleRate) => {
+        const centroid = extractSpectralCentroid(segmentData, sampleRate)
+        const magnitudeSpectrum = segmentData.map((v) => Math.abs(v))
+        const sum = magnitudeSpectrum.reduce((a, b) => a + b, 0)
+        if (sum === 0) return 0
+
+        const weightedSum = magnitudeSpectrum.reduce(
+            (acc, value, index) => acc + value * Math.pow(index - centroid, 2),
+            0
+        )
+        return Math.sqrt(weightedSum / sum)
+    }
+
+    const extractChromagram = (segmentData, sampleRate) => {
+        return [] // TODO implement
+    }
+
+    const extractHNR = (segmentData) => {
+        const frameSize = segmentData.length
+        const autocorrelation = new Float32Array(frameSize)
+
+        // Compute the autocorrelation of the segment data
+        for (let i = 0; i < frameSize; i++) {
+            let sum = 0
+            for (let j = 0; j < frameSize - i; j++) {
+                sum += segmentData[j] * segmentData[j + i]
+            }
+            autocorrelation[i] = sum
+        }
+
+        // Find the maximum autocorrelation value (excluding the zero lag)
+        const maxAutocorrelation = Math.max(...autocorrelation.subarray(1))
+
+        // Compute the HNR
+        return autocorrelation[0] !== 0
+            ? 10 *
+                  Math.log10(
+                      maxAutocorrelation /
+                          (autocorrelation[0] - maxAutocorrelation)
+                  )
+            : 0
+    }
+
+    const extractWaveform = (
+        channelData, // Float32Array
+        sampleRate, // number
+        pointsPerSecond, // number
+        algorithm // string
+    ) => {
+        const totalSamples = channelData.length
+        const segmentDuration = totalSamples / sampleRate
+        const totalPoints = Math.max(
+            Math.ceil(segmentDuration * pointsPerSecond),
+            1
+        )
+        const pointInterval = Math.ceil(totalSamples / totalPoints)
+        const dataPoints = []
+        let minAmplitude = Infinity
+        let maxAmplitude = -Infinity
+        let silenceStart = null
+        let lastSpeechEnd = -Infinity
+        let isSpeech = false
+
+        console.log(
+            `[AudioFeaturesExtractor] bitDepth=${bitDepth} samples=${totalSamples} sampleRate=${sampleRate} pointsPerSecond=${pointsPerSecond} algorithm=${algorithm}`
+        )
+        console.log(
+            `[AudioFeaturesExtractor] Extracting waveform ${pointInterval} samples per point`
+        )
+        console.log(
+            `[AudioFeaturesExtractor] segmentDuration: ${segmentDuration} seconds VS fullAudioDurationMs=${fullAudioDurationMs} ms`
+        )
+        const expectedPoints = segmentDuration * pointsPerSecond
+        const samplesPerPoint = Math.ceil(channelData.length / expectedPoints)
+        console.log(
+            `[AudioFeaturesExtractor] Extracting waveform with expectedPoints=${expectedPoints} , samplesPerPoints=${samplesPerPoint}`
+        )
+
+        for (let i = 0; i < expectedPoints; i++) {
+            const start = i * samplesPerPoint
+            const end = Math.min(start + samplesPerPoint, totalSamples)
+
+            let sumSquares = 0
+            let zeroCrossings = 0
+            let prevValue = channelData[start]
+            let localMinAmplitude = Infinity
+            let localMaxAmplitude = -Infinity
+            let hasNonZeroValue = false
+
+            // compute values for the segment
+            for (let j = start; j < end; j++) {
+                const value = channelData[j]
+                sumSquares += value * value
+                if (j > start && value * prevValue < 0) {
+                    zeroCrossings++
+                }
+                prevValue = value
+
+                const absValue = Math.abs(value)
+                localMinAmplitude = Math.min(localMinAmplitude, absValue)
+                localMaxAmplitude = Math.max(localMaxAmplitude, absValue)
+
+                if (absValue !== 0) {
+                    hasNonZeroValue = true
+                }
+            }
+
+            // Post-processing checks
+            if (!hasNonZeroValue) {
+                // All values are zero
+                localMinAmplitude = 0
+                localMaxAmplitude = 0
+            }
+
+            const rms = Math.sqrt(sumSquares / (end - start))
+            minAmplitude = Math.min(minAmplitude, rms)
+            maxAmplitude = Math.max(maxAmplitude, rms)
+
+            const energy = sumSquares
+            const zcr = zeroCrossings / (end - start)
+
+            const silent = rms < SILENCE_THRESHOLD
+            const dB = 20 * Math.log10(rms)
+
+            if (silent) {
+                if (silenceStart === null) {
+                    silenceStart = start
+                } else if (start - silenceStart > MIN_SILENCE_DURATION) {
+                    // Silence detected for longer than the threshold, set amplitude to 0
+                    localMaxAmplitude = 0
+                    localMinAmplitude = 0
+                    isSpeech = false
+                }
+            } else {
+                silenceStart = null
+                if (
+                    !isSpeech &&
+                    start - lastSpeechEnd < SPEECH_INERTIA_DURATION
+                ) {
+                    isSpeech = true
+                }
+                lastSpeechEnd = end
+            }
+
+            const activeSpeech =
+                (rms > RMS_THRESHOLD && zcr > ZCR_THRESHOLD) ||
+                (isSpeech && start - lastSpeechEnd < SPEECH_INERTIA_DURATION)
+
+            if (activeSpeech) {
+                isSpeech = true
+                lastSpeechEnd = end
+            } else {
+                isSpeech = false
+            }
+
+            const bytesPerSample = bitDepth / 8
+            const startPosition = start * bytesPerSample * numberOfChannels // Calculate start position in bytes
+            const endPosition = end * bytesPerSample * numberOfChannels // Calculate end position in bytes
+
+            // Compute features
+            const segmentData = channelData.slice(start, end)
+            const mfcc = features.mfcc
+                ? extractMFCC(segmentData, sampleRate)
+                : []
+            const spectralCentroid = features.spectralCentroid
+                ? extractSpectralCentroid(segmentData, sampleRate)
+                : 0
+            const spectralFlatness = features.spectralFlatness
+                ? extractSpectralFlatness(segmentData)
+                : 0
+            const spectralRollOff = features.spectralRollOff
+                ? extractSpectralRollOff(segmentData, sampleRate)
+                : 0
+            const spectralBandwidth = features.spectralBandwidth
+                ? extractSpectralBandwidth(segmentData, sampleRate)
+                : 0
+            const chromagram = features.chromagram
+                ? extractChromagram(segmentData, sampleRate)
+                : []
+            const hnr = features.hnr ? extractHNR(segmentData) : 0
+
+            const newData = {
+                id: uniqueIdCounter++, // Assign unique ID and increment the counter
+                amplitude: algorithm === 'peak' ? localMaxAmplitude : rms,
+                activeSpeech,
+                dB,
+                silent,
+                features: {
+                    energy,
+                    rms,
+                    minAmplitude: localMinAmplitude,
+                    maxAmplitude: localMaxAmplitude,
+                    zcr,
+                    mfcc: [], // Placeholder for MFCC features
+                    spectralCentroid, // Computed spectral centroid
+                    spectralFlatness, // Computed spectral flatness
+                    spectralRollOff, // Computed spectral roll-off
+                    spectralBandwidth, // Computed spectral bandwidth
+                    chromagram, // Computed chromagram
+                    hnr, // Computed HNR
+                },
+                startTime: start / sampleRate,
+                endTime: end / sampleRate,
+                startPosition,
+                endPosition,
+                samples: end - start,
+                speaker: 0, // Assuming speaker detection is to be handled later
+            }
+            if (newData.id < 2) {
+                console.log(`[AudioFeaturesExtractor] i=${i}`, newData)
+            }
+
+            dataPoints.push(newData)
+        }
+
+        return {
+            pointsPerSecond,
+            durationMs: fullAudioDurationMs,
+            bitDepth,
+            samples: totalSamples,
+            numberOfChannels,
+            sampleRate,
+            dataPoints,
+            amplitudeRange: {
+                min: minAmplitude,
+                max: maxAmplitude,
+            },
+            speakerChanges: [], // Placeholder for future speaker detection logic
+        }
+    }
+
+    try {
+        const result = extractWaveform(
+            channelData,
+            sampleRate,
+            pointsPerSecond,
+            algorithm
+        )
+        self.postMessage({
+            command: 'features',
+            result,
+        })
+    } catch (error) {
+        console.error('[AudioFeaturesExtractor] Error in processing', error)
+        self.postMessage({ error: error.message })
+    } finally {
+        // Do not close the worker so it can be re-used for subsequent messages
+        // self.close();
+    }
+}
diff --git a/playground/audio_samples/advertisement_speech_over_music.wav b/playground/audio_samples/advertisement_speech_over_music.wav
diff --git a/playground/audio_samples/chorus.wav b/playground/audio_samples/chorus.wav
diff --git a/playground/audio_samples/classical.wav b/playground/audio_samples/classical.wav
diff --git a/playground/audio_samples/conference.wav b/playground/audio_samples/conference.wav
diff --git a/playground/audio_samples/jazz.wav b/playground/audio_samples/jazz.wav
diff --git a/playground/audio_samples/osr_us_000_0010_8k.wav b/playground/audio_samples/osr_us_000_0010_8k.wav
diff --git a/playground/audio_samples/recorder_hello_world.wav b/playground/audio_samples/recorder_hello_world.wav
diff --git a/playground/audio_samples/recorder_jre_lex_watch.wav b/playground/audio_samples/recorder_jre_lex_watch.wav
diff --git a/playground/audio_samples/sports.wav b/playground/audio_samples/sports.wav
diff --git a/playground/audio_samples/television_speech_between_music.wav b/playground/audio_samples/television_speech_between_music.wav