Skip to content

Stabilize Media3 audio visualizer updates #512

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
wizdom13 merged 1 commit into from
Sep 9, 2026
Merged

Stabilize Media3 audio visualizer updates #512

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 111 additions & 24 deletions app/src/main/java/org/schabi/newpipe/player/visualizer/VisualizerAudioProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,28 +6,49 @@

import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.Arrays;
import java.util.ArrayDeque;
import java.util.function.LongSupplier;

/**
* Pass-through PCM processor that exposes a small, normalized waveform for the player visualizer.
* It operates on decoded audio and therefore does not require microphone access.
*/
public final class VisualizerAudioProcessor extends BaseAudioProcessor {
public static final int SAMPLE_COUNT = 128;
static final long FRAME_DURATION_NANOS = 1_000_000_000L / 60L;
private static final int TARGET_FRAME_RATE = 60;
private static final int MAX_QUEUED_FRAMES = TARGET_FRAME_RATE * 2;

private final float[] workingSamples = new float[SAMPLE_COUNT];
private volatile float[] latestSamples = new float[SAMPLE_COUNT];
private final Object frameLock = new Object();
private final ArrayDeque<float[]> queuedFrames = new ArrayDeque<>();
private final LongSupplier nanoTimeSupplier;
private float[] accumulatingSamples = new float[SAMPLE_COUNT];
private float[] latestSamples = new float[SAMPLE_COUNT];
private int accumulatedSampleCount;
private long nextFrameTimeNanos;
private volatile boolean enabled;

/** Create a processor paced by the monotonic system clock. */
public VisualizerAudioProcessor() {
this(System::nanoTime);
}

VisualizerAudioProcessor(final LongSupplier nanoTimeSupplier) {
this.nanoTimeSupplier = nanoTimeSupplier;
}

/**
* Enable or disable waveform capture. Audio remains pass-through in both states.
*
* @param enabled whether waveform capture should run
*/
public void setEnabled(final boolean enabled) {
if (this.enabled == enabled) {
return;
}
this.enabled = enabled;
if (!enabled) {
latestSamples = new float[SAMPLE_COUNT];
synchronized (frameLock) {
resetCapturedFrames();
}
}

Expand All @@ -38,17 +59,41 @@ public void setEnabled(final boolean enabled) {
* @return number of samples copied
*/
public int copyLatestSamples(final float[] target) {
final float[] snapshot = latestSamples;
final int count = Math.min(target.length, snapshot.length);
System.arraycopy(snapshot, 0, target, 0, count);
return count;
synchronized (frameLock) {
advanceVisibleFrame(nanoTimeSupplier.getAsLong());
final int count = Math.min(target.length, latestSamples.length);
System.arraycopy(latestSamples, 0, target, 0, count);
return count;
}
}

@Override
protected AudioFormat onConfigure(final AudioFormat inputAudioFormat)
throws AudioProcessor.UnhandledAudioFormatException {
return inputAudioFormat.encoding == C.ENCODING_PCM_16BIT
? inputAudioFormat : AudioFormat.NOT_SET;
if (inputAudioFormat.encoding != C.ENCODING_PCM_16BIT) {
return AudioFormat.NOT_SET;
}
synchronized (frameLock) {
accumulatingSamples = new float[Math.max(SAMPLE_COUNT,
inputAudioFormat.sampleRate / TARGET_FRAME_RATE)];
resetCapturedFrames();
}
return inputAudioFormat;
}

@Override
protected void onFlush() {
synchronized (frameLock) {
resetCapturedFrames();
}
}

@Override
protected void onReset() {
synchronized (frameLock) {
accumulatingSamples = new float[SAMPLE_COUNT];
resetCapturedFrames();
}
}

@Override
Expand All @@ -70,22 +115,64 @@ private void captureWaveform(final ByteBuffer inputBuffer) {
final ByteBuffer samples = inputBuffer.duplicate().order(ByteOrder.LITTLE_ENDIAN);
final int channelCount = Math.max(1, inputAudioFormat.channelCount);
final int frameSize = channelCount * 2;
final int frameCount = samples.remaining() / frameSize;
if (frameCount == 0) {
return;

synchronized (frameLock) {
if (!enabled) {
return;
}
while (samples.remaining() >= frameSize) {
float mixed = 0.0f;
for (int channel = 0; channel < channelCount; channel++) {
mixed += samples.getShort() / 32768.0f;
}
accumulatingSamples[accumulatedSampleCount++] = mixed / channelCount;
if (accumulatedSampleCount == accumulatingSamples.length) {
enqueueAccumulatedFrame();
}
}
}
}

Arrays.fill(workingSamples, 0.0f);
private void enqueueAccumulatedFrame() {
final float[] frame = new float[SAMPLE_COUNT];
for (int outputIndex = 0; outputIndex < SAMPLE_COUNT; outputIndex++) {
final int frame = Math.min(frameCount - 1,
outputIndex * frameCount / SAMPLE_COUNT);
final int frameOffset = samples.position() + frame * frameSize;
float mixed = 0.0f;
for (int channel = 0; channel < channelCount; channel++) {
mixed += samples.getShort(frameOffset + channel * 2) / 32768.0f;
}
workingSamples[outputIndex] = mixed / channelCount;
final int sampleIndex = Math.min(accumulatedSampleCount - 1,
outputIndex * accumulatedSampleCount / SAMPLE_COUNT);
frame[outputIndex] = accumulatingSamples[sampleIndex];
}
if (queuedFrames.size() == MAX_QUEUED_FRAMES) {
queuedFrames.removeFirst();
}
queuedFrames.addLast(frame);
accumulatedSampleCount = 0;
}

private void advanceVisibleFrame(final long nowNanos) {
if (queuedFrames.isEmpty()) {
nextFrameTimeNanos = 0L;
return;
}
if (nextFrameTimeNanos == 0L) {
latestSamples = queuedFrames.removeFirst();
nextFrameTimeNanos = nowNanos + FRAME_DURATION_NANOS;
return;
}
if (nowNanos < nextFrameTimeNanos) {
return;
}
latestSamples = workingSamples.clone();

final long dueFrameCount = 1L
+ (nowNanos - nextFrameTimeNanos) / FRAME_DURATION_NANOS;
for (long frame = 0; frame < dueFrameCount && !queuedFrames.isEmpty(); frame++) {
latestSamples = queuedFrames.removeFirst();
}
nextFrameTimeNanos += dueFrameCount * FRAME_DURATION_NANOS;
}

private void resetCapturedFrames() {
queuedFrames.clear();
accumulatedSampleCount = 0;
latestSamples = new float[SAMPLE_COUNT];
nextFrameTimeNanos = 0L;
}
}
66 changes: 63 additions & 3 deletions app/src/test/java/org/schabi/newpipe/player/visualizer/VisualizerAudioProcessorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.util.concurrent.atomic.AtomicLong;

public class VisualizerAudioProcessorTest {
@Test
Expand All @@ -32,18 +33,77 @@ public void processorPassesAudioThroughAndCapturesWaveform() throws Exception {
C.ENCODING_PCM_16BIT));
processor.flush();

final ByteBuffer input = ByteBuffer.allocateDirect(512)
final ByteBuffer input = ByteBuffer.allocateDirect(3_200)
.order(ByteOrder.LITTLE_ENDIAN);
for (int i = 0; i < 128; i++) {
for (int i = 0; i < 800; i++) {
input.putShort((short) 16_384);
input.putShort((short) 16_384);
}
input.flip();
processor.queueInput(input);

assertEquals(512, processor.getOutput().remaining());
assertEquals(3_200, processor.getOutput().remaining());
final float[] waveform = new float[VisualizerAudioProcessor.SAMPLE_COUNT];
assertEquals(waveform.length, processor.copyLatestSamples(waveform));
assertTrue(waveform[0] > 0.49f);
}

@Test
public void largeMedia3BufferIsPresentedAsPacedVisualizerFrames() throws Exception {
final AtomicLong clock = new AtomicLong(1L);
final VisualizerAudioProcessor processor = new VisualizerAudioProcessor(clock::get);
processor.setEnabled(true);
processor.configure(new AudioProcessor.AudioFormat(48_000, 2,
C.ENCODING_PCM_16BIT));
processor.flush();

final ByteBuffer input = ByteBuffer.allocateDirect(6_400)
.order(ByteOrder.LITTLE_ENDIAN);
putStereoFrames(input, 800, (short) 16_384);
putStereoFrames(input, 800, (short) -16_384);
input.flip();
processor.queueInput(input);

final float[] firstFrame = new float[VisualizerAudioProcessor.SAMPLE_COUNT];
final float[] sameDisplayFrame = new float[VisualizerAudioProcessor.SAMPLE_COUNT];
final float[] secondFrame = new float[VisualizerAudioProcessor.SAMPLE_COUNT];
processor.copyLatestSamples(firstFrame);
processor.copyLatestSamples(sameDisplayFrame);
clock.addAndGet(VisualizerAudioProcessor.FRAME_DURATION_NANOS);
processor.copyLatestSamples(secondFrame);

assertTrue(firstFrame[0] > 0.49f);
assertEquals(firstFrame[0], sameDisplayFrame[0], 0.0f);
assertTrue(secondFrame[0] < -0.49f);
}

@Test
public void disablingProcessorClearsCapturedFrames() throws Exception {
final VisualizerAudioProcessor processor = new VisualizerAudioProcessor();
processor.setEnabled(true);
processor.configure(new AudioProcessor.AudioFormat(48_000, 2,
C.ENCODING_PCM_16BIT));
processor.flush();

final ByteBuffer input = ByteBuffer.allocateDirect(3_200)
.order(ByteOrder.LITTLE_ENDIAN);
putStereoFrames(input, 800, (short) 16_384);
input.flip();
processor.queueInput(input);
processor.setEnabled(false);

final float[] waveform = new float[VisualizerAudioProcessor.SAMPLE_COUNT];
processor.copyLatestSamples(waveform);
for (final float sample : waveform) {
assertEquals(0.0f, sample, 0.0f);
}
}

private static void putStereoFrames(final ByteBuffer input, final int count,
final short sample) {
for (int frame = 0; frame < count; frame++) {
input.putShort(sample);
input.putShort(sample);
}
}
}