Skip to content
This repository has been archived by the owner on Sep 11, 2024. It is now read-only.

Commit

Permalink
Generalise VoiceRecording (#9304)
Browse files Browse the repository at this point in the history
  • Loading branch information
weeman1337 authored Sep 21, 2022
1 parent 71cf9bf commit c182c1c
Show file tree
Hide file tree
Showing 11 changed files with 422 additions and 103 deletions.
166 changes: 166 additions & 0 deletions src/audio/VoiceMessageRecording.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,166 @@
/*
Copyright 2022 The Matrix.org Foundation C.I.C.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

import { IEncryptedFile, MatrixClient } from "matrix-js-sdk/src/matrix";
import { SimpleObservable } from "matrix-widget-api";

import { uploadFile } from "../ContentMessages";
import { IDestroyable } from "../utils/IDestroyable";
import { Singleflight } from "../utils/Singleflight";
import { Playback } from "./Playback";
import { IRecordingUpdate, RecordingState, VoiceRecording } from "./VoiceRecording";

export interface IUpload {
mxc?: string; // for unencrypted uploads
encrypted?: IEncryptedFile;
}

/**
* This class can be used to record a single voice message.
*/
export class VoiceMessageRecording implements IDestroyable {
private lastUpload: IUpload;
private buffer = new Uint8Array(0); // use this.audioBuffer to access
private playback: Playback;

public constructor(
private matrixClient: MatrixClient,
private voiceRecording: VoiceRecording,
) {
this.voiceRecording.onDataAvailable = this.onDataAvailable;
}

public async start(): Promise<void> {
if (this.lastUpload || this.hasRecording) {
throw new Error("Recording already prepared");
}

return this.voiceRecording.start();
}

public async stop(): Promise<Uint8Array> {
await this.voiceRecording.stop();
return this.audioBuffer;
}

public on(event: string | symbol, listener: (...args: any[]) => void): this {
this.voiceRecording.on(event, listener);
return this;
}

public off(event: string | symbol, listener: (...args: any[]) => void): this {
this.voiceRecording.off(event, listener);
return this;
}

public emit(event: string, ...args: any[]): boolean {
return this.voiceRecording.emit(event, ...args);
}

public get hasRecording(): boolean {
return this.buffer.length > 0;
}

public get isRecording(): boolean {
return this.voiceRecording.isRecording;
}

/**
* Gets a playback instance for this voice recording. Note that the playback will not
* have been prepared fully, meaning the `prepare()` function needs to be called on it.
*
* The same playback instance is returned each time.
*
* @returns {Playback} The playback instance.
*/
public getPlayback(): Playback {
this.playback = Singleflight.for(this, "playback").do(() => {
return new Playback(this.audioBuffer.buffer, this.voiceRecording.amplitudes); // cast to ArrayBuffer proper;
});
return this.playback;
}

public async upload(inRoomId: string): Promise<IUpload> {
if (!this.hasRecording) {
throw new Error("No recording available to upload");
}

if (this.lastUpload) return this.lastUpload;

try {
this.emit(RecordingState.Uploading);
const { url: mxc, file: encrypted } = await uploadFile(
this.matrixClient,
inRoomId,
new Blob(
[this.audioBuffer],
{
type: this.contentType,
},
),
);
this.lastUpload = { mxc, encrypted };
this.emit(RecordingState.Uploaded);
} catch (e) {
this.emit(RecordingState.Ended);
throw e;
}
return this.lastUpload;
}

public get durationSeconds(): number {
return this.voiceRecording.durationSeconds;
}

public get contentType(): string {
return this.voiceRecording.contentType;
}

public get contentLength(): number {
return this.buffer.length;
}

public get liveData(): SimpleObservable<IRecordingUpdate> {
return this.voiceRecording.liveData;
}

public get isSupported(): boolean {
return this.voiceRecording.isSupported;
}

destroy(): void {
this.playback?.destroy();
this.voiceRecording.destroy();
}

private onDataAvailable = (data: ArrayBuffer) => {
const buf = new Uint8Array(data);
const newBuf = new Uint8Array(this.buffer.length + buf.length);
newBuf.set(this.buffer, 0);
newBuf.set(buf, this.buffer.length);
this.buffer = newBuf;
};

private get audioBuffer(): Uint8Array {
// We need a clone of the buffer to avoid accidentally changing the position
// on the real thing.
return this.buffer.slice(0);
}
}

export const createVoiceMessageRecording = (matrixClient: MatrixClient) => {
return new VoiceMessageRecording(matrixClient, new VoiceRecording());
};
88 changes: 7 additions & 81 deletions src/audio/VoiceRecording.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,20 +16,16 @@ limitations under the License.

import * as Recorder from 'opus-recorder';
import encoderPath from 'opus-recorder/dist/encoderWorker.min.js';
import { MatrixClient } from "matrix-js-sdk/src/client";
import { SimpleObservable } from "matrix-widget-api";
import EventEmitter from "events";
import { IEncryptedFile } from "matrix-js-sdk/src/@types/event";
import { logger } from "matrix-js-sdk/src/logger";

import MediaDeviceHandler from "../MediaDeviceHandler";
import { IDestroyable } from "../utils/IDestroyable";
import { Singleflight } from "../utils/Singleflight";
import { PayloadEvent, WORKLET_NAME } from "./consts";
import { UPDATE_EVENT } from "../stores/AsyncStore";
import { Playback } from "./Playback";
import { createAudioContext } from "./compat";
import { uploadFile } from "../ContentMessages";
import { FixedRollingArray } from "../utils/FixedRollingArray";
import { clamp } from "../utils/numbers";
import mxRecorderWorkletPath from "./RecorderWorklet";
Expand All @@ -55,38 +51,23 @@ export enum RecordingState {
Uploaded = "uploaded",
}

export interface IUpload {
mxc?: string; // for unencrypted uploads
encrypted?: IEncryptedFile;
}

export class VoiceRecording extends EventEmitter implements IDestroyable {
private recorder: Recorder;
private recorderContext: AudioContext;
private recorderSource: MediaStreamAudioSourceNode;
private recorderStream: MediaStream;
private recorderWorklet: AudioWorkletNode;
private recorderProcessor: ScriptProcessorNode;
private buffer = new Uint8Array(0); // use this.audioBuffer to access
private lastUpload: IUpload;
private recording = false;
private observable: SimpleObservable<IRecordingUpdate>;
private amplitudes: number[] = []; // at each second mark, generated
private playback: Playback;
public amplitudes: number[] = []; // at each second mark, generated
private liveWaveform = new FixedRollingArray(RECORDING_PLAYBACK_SAMPLES, 0);

public constructor(private client: MatrixClient) {
super();
}
public onDataAvailable: (data: ArrayBuffer) => void;

public get contentType(): string {
return "audio/ogg";
}

public get contentLength(): number {
return this.buffer.length;
}

public get durationSeconds(): number {
if (!this.recorder) throw new Error("Duration not available without a recording");
return this.recorderContext.currentTime;
Expand Down Expand Up @@ -165,13 +146,9 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
encoderComplexity: 3, // 0-10, 10 is slow and high quality.
resampleQuality: 3, // 0-10, 10 is slow and high quality
});
this.recorder.ondataavailable = (a: ArrayBuffer) => {
const buf = new Uint8Array(a);
const newBuf = new Uint8Array(this.buffer.length + buf.length);
newBuf.set(this.buffer, 0);
newBuf.set(buf, this.buffer.length);
this.buffer = newBuf;
};

// not using EventEmitter here because it leads to detached bufferes
this.recorder.ondataavailable = (data: ArrayBuffer) => this?.onDataAvailable(data);
} catch (e) {
logger.error("Error starting recording: ", e);
if (e instanceof DOMException) { // Unhelpful DOMExceptions are common - parse them sanely
Expand All @@ -191,12 +168,6 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
}
}

private get audioBuffer(): Uint8Array {
// We need a clone of the buffer to avoid accidentally changing the position
// on the real thing.
return this.buffer.slice(0);
}

public get liveData(): SimpleObservable<IRecordingUpdate> {
if (!this.recording) throw new Error("No observable when not recording");
return this.observable;
Expand All @@ -206,10 +177,6 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
return !!Recorder.isRecordingSupported();
}

public get hasRecording(): boolean {
return this.buffer.length > 0;
}

private onAudioProcess = (ev: AudioProcessingEvent) => {
this.processAudioUpdate(ev.playbackTime);

Expand Down Expand Up @@ -251,9 +218,6 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
};

public async start(): Promise<void> {
if (this.lastUpload || this.hasRecording) {
throw new Error("Recording already prepared");
}
if (this.recording) {
throw new Error("Recording already in progress");
}
Expand All @@ -267,7 +231,7 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
this.emit(RecordingState.Started);
}

public async stop(): Promise<Uint8Array> {
public async stop(): Promise<void> {
return Singleflight.for(this, "stop").do(async () => {
if (!this.recording) {
throw new Error("No recording to stop");
Expand All @@ -293,54 +257,16 @@ export class VoiceRecording extends EventEmitter implements IDestroyable {
this.recording = false;
await this.recorder.close();
this.emit(RecordingState.Ended);

return this.audioBuffer;
});
}

/**
* Gets a playback instance for this voice recording. Note that the playback will not
* have been prepared fully, meaning the `prepare()` function needs to be called on it.
*
* The same playback instance is returned each time.
*
* @returns {Playback} The playback instance.
*/
public getPlayback(): Playback {
this.playback = Singleflight.for(this, "playback").do(() => {
return new Playback(this.audioBuffer.buffer, this.amplitudes); // cast to ArrayBuffer proper;
});
return this.playback;
}

public destroy() {
// noinspection JSIgnoredPromiseFromCall - not concerned about stop() being called async here
this.stop();
this.removeAllListeners();
this.onDataAvailable = undefined;
Singleflight.forgetAllFor(this);
// noinspection JSIgnoredPromiseFromCall - not concerned about being called async here
this.playback?.destroy();
this.observable.close();
}

public async upload(inRoomId: string): Promise<IUpload> {
if (!this.hasRecording) {
throw new Error("No recording available to upload");
}

if (this.lastUpload) return this.lastUpload;

try {
this.emit(RecordingState.Uploading);
const { url: mxc, file: encrypted } = await uploadFile(this.client, inRoomId, new Blob([this.audioBuffer], {
type: this.contentType,
}));
this.lastUpload = { mxc, encrypted };
this.emit(RecordingState.Uploaded);
} catch (e) {
this.emit(RecordingState.Ended);
throw e;
}
return this.lastUpload;
}
}
5 changes: 3 additions & 2 deletions src/components/views/audio_messages/LiveRecordingClock.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -16,12 +16,13 @@ limitations under the License.

import React from "react";

import { IRecordingUpdate, VoiceRecording } from "../../../audio/VoiceRecording";
import { IRecordingUpdate } from "../../../audio/VoiceRecording";
import Clock from "./Clock";
import { MarkedExecution } from "../../../utils/MarkedExecution";
import { VoiceMessageRecording } from "../../../audio/VoiceMessageRecording";

interface IProps {
recorder: VoiceRecording;
recorder: VoiceMessageRecording;
}

interface IState {
Expand Down
5 changes: 3 additions & 2 deletions src/components/views/audio_messages/LiveRecordingWaveform.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,14 @@ limitations under the License.

import React from "react";

import { IRecordingUpdate, RECORDING_PLAYBACK_SAMPLES, VoiceRecording } from "../../../audio/VoiceRecording";
import { IRecordingUpdate, RECORDING_PLAYBACK_SAMPLES } from "../../../audio/VoiceRecording";
import { arrayFastResample, arraySeed } from "../../../utils/arrays";
import Waveform from "./Waveform";
import { MarkedExecution } from "../../../utils/MarkedExecution";
import { VoiceMessageRecording } from "../../../audio/VoiceMessageRecording";

interface IProps {
recorder: VoiceRecording;
recorder: VoiceMessageRecording;
}

interface IState {
Expand Down
Loading

0 comments on commit c182c1c

Please sign in to comment.