feat(frontend): Audio player で波形を表示するように (MisskeyIO#827)

Co-authored-by: あわわわとーにゅ <17376330+u1-liquid@users.noreply.github.com>
Co-authored-by: tar_bin <tar.bin.master@gmail.com>

packages/frontend/src/components/MkAudioVisualizer.vue

@@ -0,0 +1,219 @@
+<template>
+<div ref="container" :class="$style.root">
+</div>
+</template>
+
+<script lang="ts" setup>
+import { onMounted, onUnmounted, shallowRef, nextTick, ref } from 'vue';
+import * as Misskey from 'misskey-js';
+import * as THREE from 'three';
+import vertexShader from '../../node_modules/webgl-audiovisualizer/audiovisial-vertex.shader?raw';
+import fragmentShader from '../../node_modules/webgl-audiovisualizer/audiovisial-fragment.shader?raw';
+import circleMask from '../../node_modules/webgl-audiovisualizer/circlemask.png';
+
+const props = defineProps<{
+	user: Misskey.entities.UserLite;
+	audioEl: HTMLAudioElement | undefined;
+	analyser: AnalyserNode | undefined;
+}>();
+
+const container = shallowRef<HTMLDivElement>();
+
+const isPlaying = ref(false);
+const fftSize = 2048;
+
+let prevTime = 0;
+let angle1 = 0;
+let angle2 = 0;
+
+let scene, camera, renderer, width, height, uniforms, texture, maskTexture, dataArray1, dataArray2, dataArrayOrigin, bufferLength: number;
+
+const init = () => {
+	const parent = container.value ?? { offsetWidth: 0 };
+	width = parent.offsetWidth;
+	height = Math.floor(width * 9 / 16);
+
+	scene = new THREE.Scene();
+	camera = new THREE.OrthographicCamera();
+	camera.left = width / -2;
+	camera.right = width / 2;
+	camera.top = height / 2;
+	camera.bottom = height / -2;
+	camera.updateProjectionMatrix();
+
+	renderer = new THREE.WebGLRenderer({ antialias: true });
+	renderer.setSize(width, height);
+
+	if (container.value) {
+		container.value.appendChild(renderer.domElement);
+	}
+
+	const loader = new THREE.TextureLoader();
+	texture = loader.load(props.user.avatarUrl ?? '');
+	maskTexture = loader.load(circleMask);
+	uniforms = {
+		enableAudio: {
+			value: 0,
+		},
+		uTex: { value: texture },
+		uMask: { value: maskTexture },
+		time: {
+			value: 0,
+		},
+		resolution: {
+			value: new THREE.Vector2(width, height),
+		},
+	};
+
+	const material = new THREE.ShaderMaterial({
+		uniforms: uniforms,
+		vertexShader: vertexShader,
+		fragmentShader: fragmentShader,
+	});
+
+	const geometry = new THREE.PlaneGeometry(2, 2);
+
+	const mesh = new THREE.Mesh(geometry, material);
+	scene.add(mesh);
+
+	renderer.setAnimationLoop(animate);
+};
+
+const play = () => {
+	if (props.analyser) {
+		bufferLength = props.analyser.frequencyBinCount;
+		dataArrayOrigin = new Uint8Array(bufferLength);
+		dataArray1 = new Uint8Array(bufferLength);
+		dataArray2 = new Uint8Array(bufferLength);
+		uniforms = {
+			enableAudio: {
+				value: 1,
+			},
+			tAudioData1: { value: new THREE.DataTexture(dataArray1, fftSize / 2, 1, THREE.RedFormat) },
+			tAudioData2: { value: new THREE.DataTexture(dataArray2, fftSize / 2, 1, THREE.RedFormat) },
+			uTex: { value: texture },
+			uMask: { value: maskTexture },
+			time: {
+				value: 0,
+			},
+			resolution: {
+				value: new THREE.Vector2(width, height),
+			},
+		};
+		const material = new THREE.ShaderMaterial({
+			uniforms: uniforms,
+			vertexShader: vertexShader,
+			fragmentShader: fragmentShader,
+		});
+
+		const geometry = new THREE.PlaneGeometry(2, 2);
+
+		const mesh = new THREE.Mesh(geometry, material);
+		scene.add(mesh);
+
+		renderer.setAnimationLoop(animate);
+	}
+};
+
+const pauseAnimation = () => {
+	isPlaying.value = false;
+};
+
+const resumeAnimation = () => {
+	isPlaying.value = true;
+	renderer.setAnimationLoop(play);
+};
+
+const animate = (time) => {
+	if (angle1 >= bufferLength) {
+		angle1 = 0;
+	}
+	if (angle2 >= bufferLength) {
+		angle2 = 0;
+	}
+
+	if (props.analyser) {
+		if ((time - prevTime) > 10) {
+			for (let i = 0; i < bufferLength; i++) {
+				let n1 = (i + angle1) % bufferLength;
+				let n2 = (i + angle2) % bufferLength;
+				if (dataArrayOrigin[n1] > dataArray1[i]) {
+					dataArray1[i] = dataArray1[i] < 255 ? (dataArrayOrigin[i] + dataArray1[i]) / 2 : 255;
+				}
+				if (dataArrayOrigin[n2] > dataArray2[i]) {
+					dataArray2[i] = dataArray2[i] < 255 ? (dataArrayOrigin[i] + dataArray2[i]) / 2 : 255;
+				}
+			}
+		}
+		if ((time - prevTime) > 20) {
+			for (let i = 0; i < bufferLength; i++) {
+				let n1 = (i + angle1) % bufferLength;
+				let n2 = (i + angle2) % bufferLength;
+				if (dataArrayOrigin[n1] < dataArray1[i]) {
+					dataArray1[i] = dataArray1[i] > 0 ? dataArray1[i] - 1 : 0;
+				}
+				if (dataArrayOrigin[n2] < dataArray2[i]) {
+					dataArray2[i] = dataArray2[i] > 0 ? dataArray2[i] - 1 : 0;
+				}
+			}
+			uniforms.tAudioData1.value.needsUpdate = true;
+			uniforms.tAudioData2.value.needsUpdate = true;
+			prevTime = time;
+		}
+
+		if (isPlaying.value) {
+			props.analyser.getByteTimeDomainData(dataArrayOrigin);
+		} else {
+			for (let i = 0; i < bufferLength; i++) {
+				dataArrayOrigin[i] = 0;
+			}
+		}
+
+		angle1 += parseInt(String(bufferLength / 360 * 2)); //こうしないと型エラー出てうざい
+		angle2 += parseInt(String(bufferLength / 360 * 3));
+	}
+	uniforms.time.value = time;
+	renderer.render(scene, camera);
+};
+
+const onResize = () => {
+	const parent = container.value ?? { offsetWidth: 0 };
+	width = parent.offsetWidth;
+	height = Math.floor(width * 9 / 16);
+	camera.left = width / -2;
+	camera.right = width / 2;
+	camera.top = height / 2;
+	camera.bottom = height / -2;
+	camera.updateProjectionMatrix();
+	renderer.setSize(width, height);
+	uniforms.resolution.value.set(width, height);
+};
+
+onMounted(async () => {
+	nextTick().then(() => {
+		init();
+		window.addEventListener('resize', onResize);
+	});
+});
+
+onUnmounted(() => {
+	if (renderer) {
+		renderer.dispose();
+	}
+});
+
+defineExpose({
+	pauseAnimation,
+	resumeAnimation,
+});
+
+</script>
+
+<style lang="scss" module>
+.root {
+	position: relative;
+	width: 100%;
+	height: 100%;
+	overflow: hidden;
+}
+</style>

packages/frontend/src/components/MkMediaAudio.vue

@@ -35,40 +35,43 @@ SPDX-License-Identifier: AGPL-3.0-only
 		</audio>
 	</div>
 
-	<div v-else :class="$style.audioControls">
-		<audio
-			ref="audioEl"
-			preload="metadata"
-		>
-			<source :src="audio.url">
-		</audio>
-		<div :class="[$style.controlsChild, $style.controlsLeft]">
-			<button class="_button" :class="$style.controlButton" @click.prevent.stop="togglePlayPause">
-				<i v-if="isPlaying" class="ti ti-player-pause-filled"></i>
-				<i v-else class="ti ti-player-play-filled"></i>
-			</button>
-		</div>
-		<div :class="[$style.controlsChild, $style.controlsRight]">
-			<button class="_button" :class="$style.controlButton" @click.prevent.stop="showMenu">
-				<i class="ti ti-settings"></i>
-			</button>
-		</div>
-		<div :class="[$style.controlsChild, $style.controlsTime]">{{ hms(elapsedTimeMs) }}</div>
-		<div :class="[$style.controlsChild, $style.controlsVolume]">
-			<button class="_button" :class="$style.controlButton" @click.prevent.stop="toggleMute">
-				<i v-if="volume === 0" class="ti ti-volume-3"></i>
-				<i v-else class="ti ti-volume"></i>
-			</button>
+	<div v-else>
+		<MkAudioVisualizer v-if="user" ref="audioVisualizer" :audioEl="audioEl" :analyser="analyserNode" :user="user" :profileImage="user.avatarUrl"/>
+		<div :class="$style.audioControls">
+			<audio
+				ref="audioEl"
+				preload="metadata"
+			>
+				<source :src="audio.url">
+			</audio>
+			<div :class="[$style.controlsChild, $style.controlsLeft]">
+				<button class="_button" :class="$style.controlButton" @click.prevent.stop="togglePlayPause">
+					<i v-if="isPlaying" class="ti ti-player-pause-filled"></i>
+					<i v-else class="ti ti-player-play-filled"></i>
+				</button>
+			</div>
+			<div :class="[$style.controlsChild, $style.controlsRight]">
+				<button class="_button" :class="$style.controlButton" @click.prevent.stop="showMenu">
+					<i class="ti ti-settings"></i>
+				</button>
+			</div>
+			<div :class="[$style.controlsChild, $style.controlsTime]">{{ hms(elapsedTimeMs) }}</div>
+			<div :class="[$style.controlsChild, $style.controlsVolume]">
+				<button class="_button" :class="$style.controlButton" @click.prevent.stop="toggleMute">
+					<i v-if="volume === 0" class="ti ti-volume-3"></i>
+					<i v-else class="ti ti-volume"></i>
+				</button>
+				<MkMediaRange
+					v-model="volume"
+					:class="$style.volumeSeekbar"
+				/>
+			</div>
 			<MkMediaRange
-				v-model="volume"
-				:class="$style.volumeSeekbar"
+				v-model="rangePercent"
+				:class="$style.seekbarRoot"
+				:buffer="bufferedDataRatio"
 			/>
 		</div>
-		<MkMediaRange
-			v-model="rangePercent"
-			:class="$style.seekbarRoot"
-			:buffer="bufferedDataRatio"
-		/>
 	</div>
 </div>
 </template>
@@ -82,12 +85,14 @@ import { i18n } from '@/i18n.js';
 import * as os from '@/os.js';
 import bytes from '@/filters/bytes.js';
 import { hms } from '@/filters/hms.js';
+import MkAudioVisualizer from '@/components/MkAudioVisualizer.vue';
 import MkMediaRange from '@/components/MkMediaRange.vue';
 import { pleaseLogin } from '@/scripts/please-login.js';
 import { $i, iAmModerator } from '@/account.js';
 
 const props = defineProps<{
 	audio: Misskey.entities.DriveFile;
+	user?: Misskey.entities.UserLite;
 }>();
 
 const keymap = {
@@ -126,6 +131,7 @@ function hasFocus() {
 
 const playerEl = shallowRef<HTMLDivElement>();
 const audioEl = shallowRef<HTMLAudioElement>();
+const audioVisualizer = ref<InstanceType<typeof MkAudioVisualizer>>();
 
 // eslint-disable-next-line vue/no-setup-props-destructure
 const hide = ref((defaultStore.state.nsfw === 'force' || defaultStore.state.dataSaver.media) ? true : (props.audio.isSensitive && defaultStore.state.nsfw !== 'ignore'));
@@ -240,6 +246,11 @@ const isPlaying = ref(false);
 const isActuallyPlaying = ref(false);
 const elapsedTimeMs = ref(0);
 const durationMs = ref(0);
+const audioContext = ref<AudioContext | null>(null);
+const sourceNode = ref<MediaElementAudioSourceNode | null>(null);
+const gainNode = ref<GainNode | null>(null);
+const analyserGainNode = ref<GainNode | null>(null);
+const analyserNode = ref<AnalyserNode | null>(null);
 const rangePercent = computed({
 	get: () => {
 		return (elapsedTimeMs.value / durationMs.value) || 0;
@@ -262,11 +273,33 @@ const bufferedDataRatio = computed(() => {
 function togglePlayPause() {
 	if (!isReady.value || !audioEl.value) return;
 
+	if (!sourceNode.value) {
+		audioContext.value = new (window.AudioContext || window.webkitAudioContext)();
+		sourceNode.value = audioContext.value.createMediaElementSource(audioEl.value);
+
+		analyserGainNode.value = audioContext.value.createGain();
+		gainNode.value = audioContext.value.createGain();
+		analyserNode.value = audioContext.value.createAnalyser();
+
+		sourceNode.value.connect(analyserGainNode.value);
+		analyserGainNode.value.connect(analyserNode.value);
+		analyserNode.value.connect(gainNode.value);
+		gainNode.value.connect(audioContext.value.destination);
+
+		analyserNode.value.fftSize = 2048;
+
+		analyserGainNode.value.gain.setValueAtTime(0.8, audioContext.value.currentTime);
+
+		gainNode.value.gain.setValueAtTime(volume.value, audioContext.value.currentTime);
+	}
+
 	if (isPlaying.value) {
 		audioEl.value.pause();
+		audioVisualizer.value?.pauseAnimation();
 		isPlaying.value = false;
 	} else {
 		audioEl.value.play();
+		audioVisualizer.value?.resumeAnimation();
 		isPlaying.value = true;
 		oncePlayed.value = true;
 	}
@@ -324,6 +357,7 @@ function init() {
 				oncePlayed.value = false;
 				isActuallyPlaying.value = false;
 				isPlaying.value = false;
+				audioVisualizer.value?.pauseAnimation();
 			});
 
 			durationMs.value = audioEl.value.duration * 1000;
@@ -332,8 +366,7 @@ function init() {
 					durationMs.value = audioEl.value.duration * 1000;
 				}
 			});
-
-			audioEl.value.volume = volume.value;
+			gainNode.value?.gain.setValueAtTime(volume.value, audioContext.value?.currentTime);
 		}
 	}, {
 		immediate: true,
@@ -341,7 +374,7 @@ function init() {
 }
 
 watch(volume, (to) => {
-	if (audioEl.value) audioEl.value.volume = to;
+	if (audioEl.value) gainNode.value?.gain.setValueAtTime(to, audioContext.value?.currentTime);
 });
 
 watch(speed, (to) => {

packages/frontend/src/components/MkMediaBanner.vue

@@ -5,12 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only
 
 <template>
 <div :class="$style.root">
-	<MkMediaAudio v-if="media.type.startsWith('audio') && media.type !== 'audio/midi'" :audio="media"/>
-	<div v-else-if="media.isSensitive && hide" :class="$style.sensitive" @click="showHiddenContent">
+	<div v-if="media.isSensitive && hide" :class="$style.sensitive" @click="showHiddenContent">
 		<span style="font-size: 1.6em;"><i class="ti ti-alert-triangle"></i></span>
 		<b>{{ i18n.ts.sensitive }}</b>
 		<span>{{ i18n.ts.clickToShow }}</span>
 	</div>
+	<MkMediaAudio v-else-if="media.type.startsWith('audio') && media.type !== 'audio/midi'" :audio="media" :user="user"/>
 	<a
 		v-else :class="$style.download"
 		:href="media.url"
@@ -33,6 +33,7 @@ import { $i } from '@/account.js';
 
 const props = withDefaults(defineProps<{
 	media: Misskey.entities.DriveFile;
+	user?: Misskey.entities.UserLite;
 }>(), {
 });
 

packages/frontend/src/components/MkMediaList.vue

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only
 
 <template>
 <div>
-	<XBanner v-for="media in mediaList.filter(media => !previewable(media))" :key="media.id" :media="media"/>
+	<XBanner v-for="media in mediaList.filter(media => !previewable(media))" :key="media.id" :media="media" :user="user"/>
 	<div v-if="mediaList.filter(media => previewable(media)).length > 0" :class="$style.container">
 		<div
 			ref="gallery"
@@ -42,6 +42,7 @@ import { defaultStore } from '@/store.js';
 
 const props = defineProps<{
 	mediaList: Misskey.entities.DriveFile[];
+	user?: Misskey.entities.UserLite;
 	raw?: boolean;
 }>();
 

packages/frontend/src/components/MkNote.vue

@@ -79,7 +79,7 @@ SPDX-License-Identifier: AGPL-3.0-only
 						</div>
 					</div>
 					<div v-if="appearNote.files && appearNote.files.length > 0">
-						<MkMediaList :mediaList="appearNote.files"/>
+						<MkMediaList :mediaList="appearNote.files" :user="appearNote.user"/>
 					</div>
 					<MkPoll v-if="appearNote.poll" :noteId="appearNote.id" :poll="appearNote.poll" :class="$style.poll"/>
 					<div v-if="isEnabledUrlPreview">

packages/frontend/src/components/MkNoteDetailed.vue

@@ -92,7 +92,7 @@ SPDX-License-Identifier: AGPL-3.0-only
 					</div>
 				</div>
 				<div v-if="appearNote.files && appearNote.files.length > 0">
-					<MkMediaList :mediaList="appearNote.files"/>
+					<MkMediaList :mediaList="appearNote.files" :user="appearNote.user"/>
 				</div>
 				<MkPoll v-if="appearNote.poll" ref="pollViewer" :noteId="appearNote.id" :poll="appearNote.poll" :class="$style.poll"/>
 				<div v-if="isEnabledUrlPreview">

packages/frontend/src/components/MkSubNoteContent.vue

@@ -14,7 +14,7 @@ SPDX-License-Identifier: AGPL-3.0-only
 	</div>
 	<details v-if="note.files && note.files.length > 0">
 		<summary>({{ i18n.tsx.withNFiles({ n: note.files.length }) }})</summary>
-		<MkMediaList :mediaList="note.files"/>
+		<MkMediaList :mediaList="note.files" :user="note.user"/>
 	</details>
 	<details v-if="note.poll">
 		<summary>{{ i18n.ts.poll }}</summary>