Feat/analyser node (#242)

* feat: added ts interface for AnalyserNode

* refactor: refactored AudioNode

* refactor: moved Constants.h to core

* feat: added AnalyserNode core class

* feat: added HostObject for AnalyserNode

* feat: added host functions caching

* refactor: removed unused components and moved common files to common example folders and format

* fix: fixed metronome, removed unused files and moved common files

* fix: back to previous version of hihat

* fix: fixed start/stop mechanism bug

* feat: added analyser constants to Constants.h

* fix: fixed The global process.env.EXPO_OS is not defined warning in fabric-example

* feat: added createAnalyser to is interfaces

* refactor: refactored AnalyserNode creation and itself

* feat: implemented getFloatTimeDomainData and getByteTimeDomainData

* fix: fixed Analyser HostObject exported functions

* fix: prettier fixes and fixed AnalyserNode props modifiability

* feat: added AudioVisualizer example for testing and fixed eslint config

* fix: fixed AnalyserNode writing to inputBuffer

* feat: added ts error handling

* feat: added utils for float to uint8 conversion

* feat: added linear to decibels conversion

* feat: implemented AnalyserNode

* feat: implemented doFFT in FFTFrame

* ci: yarn format and small fixes

* refactor: refactored FFTFrame, added setups caching

* fix: fixed variables names in FFTFrame

* docs: updated Web Audio API coverage

* feat: added very first version of AudioVisualizer example

* fix: small prettier and yarn format fixes

* refactor: refactored example

* fix: small fixes

* feat: added comments to AnalyserNode impl

* fix: fixed frequencyBinCount prop

* fix: removed log

* feat: boost AudioVisualizer

* fix: fixed TimeChartLine size

* refactor: added removing file:// in decodeAudioSource method (#268)

Co-authored-by: Maciej Makowski <maciej.makowski2608@gmail.com>

* feat: some changes to the audio visualizer example (#270)

* feat: some changes to the audio visualizer example

* Update apps/common-app/src/examples/AudioVisualizer/Charts.tsx

* refactor: refactored AudioVisualizer example

* fix: fixed error handling

* refactor: back to useState

---------

Co-authored-by: Maciej Makowski <maciej.makowski@swmansion.com>
Co-authored-by: Maciej Makowski <maciej.makowski2608@gmail.com>
Co-authored-by: Michał Sęk <michal.sek@swmansion.com>

Author: 4 people

.eslintrc.js

@@ -17,6 +17,8 @@ module.exports = {
       rules: {
         '@typescript-eslint/no-unsafe-call': 'off',
         '@typescript-eslint/no-unsafe-member-access': 'off',
+        '@typescript-eslint/no-floating-promises': 'off',
+        '@typescript-eslint/no-misused-promises': 'off',
         '@typescript-eslint/no-unsafe-return': 'off',
         '@typescript-eslint/no-unsafe-assignment': 'off',
         '@typescript-eslint/no-unsafe-argument': 'off',

apps/common-app/src/components/Container.tsx

@@ -8,18 +8,18 @@ import { colors } from '../styles';
 type ContainerProps = PropsWithChildren<{
   style?: StyleProp<ViewStyle>;
   centered?: boolean;
+  disablePadding?: boolean;
 }>;
 
 const headerPadding = 120; // eyeballed
 
 const Container: React.FC<ContainerProps> = (props) => {
-  const { children, style, centered } = props;
+  const { children, style, centered, disablePadding } = props;
 
   return (
     <SafeAreaView
       edges={['bottom', 'left', 'right']}
-      style={[styles.basic, centered && styles.centered, style]}
-    >
+      style={[styles.basic, centered && styles.centered, !disablePadding && styles.padding, style]}>
       <BGGradient />
       {children}
     </SafeAreaView>
@@ -31,10 +31,12 @@ export default Container;
 const styles = StyleSheet.create({
   basic: {
     flex: 1,
-    padding: 24,
     paddingTop: headerPadding,
     backgroundColor: colors.background,
   },
+  padding: {
+    padding: 24,
+  },
   centered: {
     alignItems: 'center',
     justifyContent: 'center',

apps/common-app/src/components/Slider.tsx

@@ -134,8 +134,7 @@ const Slider: React.FC<SliderProps> = (props) => {
             style={[
               styles.label,
               !!minLabelWidth && { minWidth: minLabelWidth },
-            ]}
-          >
+            ]}>
             {label}
           </Text>
           <Spacer.Horizontal size={12} />

apps/common-app/src/examples/AudioFile/AudioFile.tsx

@@ -12,10 +12,10 @@ import { ActivityIndicator } from 'react-native';
 const AudioFile: FC = () => {
   const [isPlaying, setIsPlaying] = useState(false);
   const [isLoading, setIsLoading] = useState(false);
+  const [audioBuffer, setAudioBuffer] = useState<AudioBuffer | null>(null);
 
   const audioContextRef = useRef<AudioContext | null>(null);
   const audioBufferSourceNodeRef = useRef<AudioBufferSourceNode | null>(null);
-  const [audioBuffer, setAudioBuffer] = useState<AudioBuffer | null>(null);
 
   const setup = () => {
     if (!audioContextRef.current) {
@@ -42,7 +42,7 @@ const AudioFile: FC = () => {
         setIsPlaying(false);
 
         setIsLoading(true);
-        await fetchAudioBuffer(result.assets[0].uri.replace('file://', ''));
+        await fetchAudioBuffer(result.assets[0].uri);
         setIsLoading(false);
       }
     } catch (error) {
@@ -99,7 +99,7 @@ const AudioFile: FC = () => {
       <Button
         title={isPlaying ? 'Stop' : 'Play'}
         onPress={handlePress}
-        disabled={!audioBuffer ? true : false}
+        disabled={!audioBuffer}
       />
     </Container>
   );

apps/common-app/src/examples/AudioVisualizer/AudioVisualizer.tsx

@@ -0,0 +1,142 @@
+import React, { useState, useEffect, useRef } from 'react';
+import * as FileSystem from 'expo-file-system';
+import {
+  AudioContext,
+  AnalyserNode,
+  AudioBuffer,
+  AudioBufferSourceNode,
+} from 'react-native-audio-api';
+import { ActivityIndicator, View, StyleSheet } from 'react-native';
+
+import FreqTimeChart from './FreqTimeChart';
+import { Container, Button } from '../../components';
+import { layout } from '../../styles';
+
+const FFT_SIZE = 512;
+
+const URL =
+  'https://software-mansion-labs.github.io/react-native-audio-api/audio/music/example-music-02.mp3';
+
+const AudioVisualizer: React.FC = () => {
+  const [isPlaying, setIsPlaying] = useState(false);
+  const [isLoading, setIsLoading] = useState(false);
+
+  const [times, setTimes] = useState<number[]>(
+    new Array(FFT_SIZE / 2).fill(127)
+  );
+  const [freqs, setFreqs] = useState<number[]>(new Array(FFT_SIZE / 2).fill(0));
+
+  const audioContextRef = useRef<AudioContext | null>(null);
+  const analyserRef = useRef<AnalyserNode | null>(null);
+  const bufferSourceRef = useRef<AudioBufferSourceNode | null>(null);
+  const audioBufferRef = useRef<AudioBuffer | null>(null);
+
+  const handlePlayPause = () => {
+    if (isPlaying) {
+      bufferSourceRef.current?.stop();
+    } else {
+      if (!audioContextRef.current || !analyserRef.current) {
+        return;
+      }
+
+      bufferSourceRef.current = audioContextRef.current.createBufferSource();
+      bufferSourceRef.current.buffer = audioBufferRef.current;
+      bufferSourceRef.current.connect(analyserRef.current);
+
+      bufferSourceRef.current.start();
+
+      requestAnimationFrame(draw);
+    }
+
+    setIsPlaying((prev) => !prev);
+  };
+
+  const draw = () => {
+    if (!analyserRef.current) {
+      return;
+    }
+
+    const bufferLength = analyserRef.current.frequencyBinCount;
+
+    const timesArray = new Array(bufferLength);
+    analyserRef.current.getByteTimeDomainData(timesArray);
+    setTimes(timesArray);
+
+    const freqsArray = new Array(bufferLength);
+    analyserRef.current.getByteFrequencyData(freqsArray);
+    setFreqs(freqsArray);
+
+    requestAnimationFrame(draw);
+  };
+
+  useEffect(() => {
+    if (!audioContextRef.current) {
+      audioContextRef.current = new AudioContext();
+    }
+
+    if (!analyserRef.current) {
+      analyserRef.current = audioContextRef.current.createAnalyser();
+      analyserRef.current.fftSize = FFT_SIZE;
+      analyserRef.current.smoothingTimeConstant = 0.8;
+
+      analyserRef.current.connect(audioContextRef.current.destination);
+    }
+
+    const fetchBuffer = async () => {
+      setIsLoading(true);
+      audioBufferRef.current = await FileSystem.downloadAsync(
+        URL,
+        FileSystem.documentDirectory + 'audio.mp3'
+      ).then(({ uri }) => {
+        return audioContextRef.current!.decodeAudioDataSource(uri);
+      });
+
+      setIsLoading(false);
+    };
+
+    fetchBuffer();
+
+    return () => {
+      audioContextRef.current?.close();
+    };
+  }, []);
+
+  return (
+    <Container disablePadding>
+      <View style={{ flex: 0.2 }} />
+      <FreqTimeChart
+        timeData={times}
+        frequencyData={freqs}
+        frequencyBinCount={
+          analyserRef.current?.frequencyBinCount || FFT_SIZE / 2
+        }
+      />
+      <View
+        style={{ flex: 0.5, justifyContent: 'center', alignItems: 'center' }}>
+        {isLoading && <ActivityIndicator color="#FFFFFF" />}
+        <View style={styles.button}>
+          <Button
+            onPress={handlePlayPause}
+            title={isPlaying ? 'Pause' : 'Play'}
+            disabled={!audioBufferRef.current}
+          />
+        </View>
+      </View>
+    </Container>
+  );
+};
+
+const styles = StyleSheet.create({
+  container: {
+    flex: 1,
+    justifyContent: 'center',
+    alignItems: 'center',
+  },
+  button: {
+    justifyContent: 'center',
+    flexDirection: 'row',
+    marginTop: layout.spacing * 2,
+  },
+});
+
+export default AudioVisualizer;

apps/common-app/src/examples/AudioVisualizer/Canvas.tsx

@@ -0,0 +1,80 @@
+import React, {
+  useState,
+  useContext,
+  createContext,
+  PropsWithChildren,
+  useMemo,
+} from 'react';
+import { LayoutChangeEvent, StyleSheet } from 'react-native';
+import { Canvas as SKCanvas } from '@shopify/react-native-skia';
+
+interface Size {
+  width: number;
+  height: number;
+}
+
+interface CanvasContext {
+  initialized: boolean;
+  size: Size;
+}
+
+const CanvasContext = createContext<CanvasContext>({
+  initialized: false,
+  size: { width: 0, height: 0 },
+});
+
+const Canvas: React.FC<PropsWithChildren> = ({ children }) => {
+  const [size, setSize] = useState<Size>({ width: 0, height: 0 });
+
+  const onCanvasLayout = (event: LayoutChangeEvent) => {
+    const { width, height } = event.nativeEvent.layout;
+
+    setSize({ width, height });
+  };
+
+  const context = useMemo(
+    () => ({
+      initialized: true,
+      size: { width: size.width, height: size.height },
+    }),
+    [size.width, size.height]
+  );
+
+  return (
+    <SKCanvas style={styles.canvas} onLayout={onCanvasLayout}>
+      <CanvasContext.Provider value={context}>
+        {children}
+      </CanvasContext.Provider>
+    </SKCanvas>
+  );
+};
+
+export function useCanvas() {
+  const canvasContext = useContext(CanvasContext);
+
+  if (!canvasContext.initialized) {
+    throw new Error('Canvas context not initialized');
+  }
+
+  return canvasContext;
+}
+
+export function withCanvas<P extends object>(
+  Component: React.ComponentType<P>
+) {
+  return (props: P) => {
+    return (
+      <Canvas>
+        <Component {...props} />
+      </Canvas>
+    );
+  };
+}
+
+export default Canvas;
+
+const styles = StyleSheet.create({
+  canvas: {
+    flex: 1,
+  },
+});