Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 46 additions & 3 deletions packages/gui/src/components/nfts/NFTPreview.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ import useStateAbort from '../../hooks/useStateAbort';
import getFileExtension from '../../util/getFileExtension';
import getNFTId from '../../util/getNFTId';
import hasSensitiveContent from '../../util/hasSensitiveContent';
import probeMediaPlayability from '../../util/probeMediaPlayability';

import NFTHashStatus from './NFTHashStatus';

Expand Down Expand Up @@ -153,9 +154,18 @@ export default function NFTPreview(props: NFTPreviewProps) {
`nft-preview-ignore-size-limit-${nftId}`,
);

// Verified media files Chromium turned out not to decode (an HEVC video on
// Linux, say), recorded by preparePreview after probing the cached file —
// the sandboxed player cannot report the failure itself. Handed back to the
// verifier so it passes over an unplayable preview video and settles on the
// next source (preview image, data file) instead; the data file itself is
// never skipped, so an unplayable data file ends up as the notice below.
const [unplayableUris, setUnplayableUris] = useStateAbort<string[]>([]);

const { preview, isLoading: isLoadingVerifyHash } = useNFTVerifyHash(nftId, {
preview: isPreview,
ignoreSizeLimit,
excludedPreviewUris: unplayableUris,
});

const { type: previewFileType, isLoading: isLoadingFileType } = useFileType(preview?.uri);
Expand Down Expand Up @@ -209,6 +219,7 @@ export default function NFTPreview(props: NFTPreviewProps) {
const isHashMismatch = isSettledHashMismatch(preview);

const previewUri = isHashMismatch ? undefined : preview?.uri;
const isUnplayable = !!previewUri && unplayableUris.includes(previewUri);

const preparePreview = useCallback(
async (signal: AbortSignal) => {
Expand Down Expand Up @@ -253,6 +264,26 @@ export default function NFTPreview(props: NFTPreviewProps) {
return;
}

// The player runs in a scriptless sandbox and cannot say when Chromium
// rejects the stream, so ask the media pipeline first. Only a definite
// verdict counts — a probe that fails for any other reason falls
// through to the player as before.
if (previewFileType === FileType.VIDEO || previewFileType === FileType.AUDIO) {
const playability = await probeMediaPlayability(
cachedURI,
previewFileType === FileType.VIDEO ? 'video' : 'audio',
{ signal },
);
if (signal.aborted) {
return;
}
if (playability === 'unsupported') {
setPreviewContent(undefined, signal);
setUnplayableUris((uris) => (uris.includes(previewUri) ? uris : [...uris, previewUri]), signal);
return;
}
}

// Interactivity is controlled outside the iframe (pointer-events on
// the iframe plus the IframePreventEvents overlay), never inside the
// srcDoc: the sandbox forbids scripts, so any srcDoc change forces a
Expand Down Expand Up @@ -305,6 +336,7 @@ export default function NFTPreview(props: NFTPreviewProps) {
isDarkMode,
setPreviewContent,
setPrepareError,
setUnplayableUris,
],
);

Expand Down Expand Up @@ -536,8 +568,9 @@ export default function NFTPreview(props: NFTPreviewProps) {
return isLoadingMetadata ? undefined : NFTPreviewStatus.UNAVAILABLE;
}

if (prepareError) {
// the verified file could not be served from the cache
if (prepareError || isUnplayable) {
// the verified file could not be served from the cache, or Chromium
// cannot decode it — either way there is nothing to show
return NFTPreviewStatus.UNAVAILABLE;
}

Expand All @@ -547,7 +580,7 @@ export default function NFTPreview(props: NFTPreviewProps) {
}

return NFTPreviewStatus.AVAILABLE;
}, [isLoading, isLoadingVerifyHash, isLoadingMetadata, preview, prepareError, previewContent]);
}, [isLoading, isLoadingVerifyHash, isLoadingMetadata, preview, prepareError, previewContent, isUnplayable]);

useEffect(() => {
// Only preview-mode tiles report: the detail view verifies the full data
Expand Down Expand Up @@ -576,6 +609,16 @@ export default function NFTPreview(props: NFTPreviewProps) {
<Trans>File does not match the expected hash</Trans>
</IconMessage>
</Background>
) : usesIframe && isUnplayable ? (
<Background>
<IconMessage icon={<NotInterested fontSize="large" />}>
{previewFileType === FileType.AUDIO ? (
<Trans>This audio format cannot be played here</Trans>
) : (
<Trans>This video format cannot be played here</Trans>
)}
</IconMessage>
</Background>
) : usesIframe && prepareError ? (
<Background>
<IconMessage icon={<NotInterested fontSize="large" />}>
Expand Down
61 changes: 48 additions & 13 deletions packages/gui/src/hooks/useNFTVerifyHash.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,32 @@ const log = debug('chia-gui:useNFTVerifyHash');
export type UseNFTVerifyHashOptions = {
preview?: boolean;
ignoreSizeLimit?: boolean;
// Preview URIs to pass over — files that verified but that Chromium turned
// out not to decode. Verification then moves on to the next source in
// priority order (preview image, data file) instead of settling on a file
// that cannot be shown. Only preview candidates are affected: the data
// file is what isVerified is derived from and is never skipped.
excludedPreviewUris?: string[];
};

function withoutExcluded(uris: string[] | undefined, excluded: Set<string>): string[] | undefined {
if (!uris || excluded.size === 0) {
return uris;
}

return uris.filter((uri) => !excluded.has(uri));
}

export default function useNFTVerifyHash(nftId?: string, options: UseNFTVerifyHashOptions = {}) {
const { preview = false, ignoreSizeLimit = false } = options;
const { preview = false, ignoreSizeLimit = false, excludedPreviewUris } = options;

// a stable key so a caller passing a fresh array each render does not
// restart preview verification
const excludedPreviewKey = excludedPreviewUris?.length ? excludedPreviewUris.join('\n') : '';
const excludedPreview = useMemo(
() => new Set(excludedPreviewKey ? excludedPreviewKey.split('\n') : []),
[excludedPreviewKey],
);
Comment thread
cursor[bot] marked this conversation as resolved.

const { getChecksum } = useCache();
// Not read directly: the value changes which URIs the main process will
Expand Down Expand Up @@ -51,24 +73,33 @@ export default function useNFTVerifyHash(nftId?: string, options: UseNFTVerifyHa
// hold the previous NFT's results, and surfacing them would flash the
// previous NFT's media (and hash verdict) until the effects reset them.
const dataInputs = useRef<{ nft?: NFTInfo }>({});
const previewInputs = useRef<{ nft?: NFTInfo; metadata?: Metadata }>({});
const previewInputs = useRef<{ nft?: NFTInfo; metadata?: Metadata; excludedKey?: string }>({});

const settledNft = !isLoadingNFT ? nft : undefined;
const settledMetadata = isLoadingMetadata ? undefined : metadata;

const isDataStale = dataInputs.current.nft !== settledNft;
const isPreviewStale = previewInputs.current.nft !== settledNft;

// An excluded uri is masked out of the stored states as well as the
// candidates: on the frame a caller excludes the uri it is currently
// showing, the stored state still holds that verified file, and surfacing
// it would let the caller settle on "unplayable" (and report the NFT as
// unavailable) before the pass below has had a chance to move on.
const isExcluded = (state: NFTPreviewState | undefined) => !!state && excludedPreview.has(state.uri);

const currentData = isDataStale ? undefined : data;
const currentPreviewVideo = isPreviewStale ? undefined : previewVideo;
const currentPreviewImage = isPreviewStale ? undefined : previewImage;
const currentPreviewVideo = isPreviewStale || isExcluded(previewVideo) ? undefined : previewVideo;
const currentPreviewImage = isPreviewStale || isExcluded(previewImage) ? undefined : previewImage;

const isDataPassPending = !!settledNft && isDataStale;
const isPreviewPassPending =
preview &&
!!settledNft &&
!!settledMetadata &&
(previewInputs.current.nft !== settledNft || previewInputs.current.metadata !== settledMetadata);
(previewInputs.current.nft !== settledNft ||
previewInputs.current.metadata !== settledMetadata ||
previewInputs.current.excludedKey !== excludedPreviewKey);

const isVerifying = isVerifyingData || isVerifyingPreview || isDataPassPending || isPreviewPassPending;

Expand Down Expand Up @@ -157,14 +188,14 @@ export default function useNFTVerifyHash(nftId?: string, options: UseNFTVerifyHa
try {
const { preview_video_uris: previewVideoUris, preview_video_hash: previewVideoHash } = nftMetadata;

const videoState = await findValidUri(previewVideoUris, previewVideoHash);
const videoState = await findValidUri(withoutExcluded(previewVideoUris, excludedPreview), previewVideoHash);
if (generationRef.current === generation) {
setPreviewVideo(videoState);
}

if (!videoState?.isVerified) {
const { preview_image_uris: previewImageUris, preview_image_hash: previewImageHash } = nftMetadata;
const imageState = await findValidUri(previewImageUris, previewImageHash);
const imageState = await findValidUri(withoutExcluded(previewImageUris, excludedPreview), previewImageHash);
if (generationRef.current === generation) {
setPreviewImage(imageState);
}
Expand All @@ -179,7 +210,7 @@ export default function useNFTVerifyHash(nftId?: string, options: UseNFTVerifyHa
}
}
},
[findValidUri],
[findValidUri, excludedPreview],
);

// Data and preview verification run as independent effects: the data file
Expand Down Expand Up @@ -220,7 +251,11 @@ export default function useNFTVerifyHash(nftId?: string, options: UseNFTVerifyHa
// verifies the data file right away, and this effect picks up the preview
// URIs once the metadata fetch settles, instead of blocking on it.
const nftMetadata = isLoadingMetadata ? undefined : metadata;
previewInputs.current = { nft: !isLoadingNFT ? nft : undefined, metadata: nftMetadata };
previewInputs.current = {
nft: !isLoadingNFT ? nft : undefined,
metadata: nftMetadata,
excludedKey: excludedPreviewKey,
};
if (!preview || !nft || isLoadingNFT || !nftMetadata) {
setIsVerifyingPreview(false);
} else {
Expand All @@ -233,7 +268,7 @@ export default function useNFTVerifyHash(nftId?: string, options: UseNFTVerifyHa
previewGeneration.current += 1;
}
};
}, [preview, nft, metadata, isLoadingNFT, isLoadingMetadata, validatePreview, ipfsGateway]);
}, [preview, nft, metadata, isLoadingNFT, isLoadingMetadata, validatePreview, ipfsGateway, excludedPreviewKey]);

const previewState = useMemo(
() =>
Expand All @@ -244,13 +279,13 @@ export default function useNFTVerifyHash(nftId?: string, options: UseNFTVerifyHa
data: currentData,
previewVideoCandidate: preview
? {
uris: metadata?.preview_video_uris,
uris: withoutExcluded(metadata?.preview_video_uris, excludedPreview),
hash: metadata?.preview_video_hash,
}
: undefined,
previewImageCandidate: preview
? {
uris: metadata?.preview_image_uris,
uris: withoutExcluded(metadata?.preview_image_uris, excludedPreview),
hash: metadata?.preview_image_hash,
}
: undefined,
Expand All @@ -259,7 +294,7 @@ export default function useNFTVerifyHash(nftId?: string, options: UseNFTVerifyHa
hash: nft?.dataHash,
},
}),
[currentPreviewVideo, currentPreviewImage, currentData, nft, metadata, preview, isVerifying],
[currentPreviewVideo, currentPreviewImage, currentData, nft, metadata, preview, isVerifying, excludedPreview],
);

return {
Expand Down
138 changes: 138 additions & 0 deletions packages/gui/src/util/probeMediaPlayability.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
import probeMediaPlayability, { resetMediaPlayabilityVerdicts } from './probeMediaPlayability';

type FakeMediaElement = HTMLVideoElement & {
emit: (event: string) => void;
loadCalls: number;
};

// the tests run without a DOM, so a fake element stands in for the probe's
// <video>/<audio> and its events are driven by hand
function fakeMediaElement(errorCode?: number): FakeMediaElement {
const listeners = new Map<string, Set<() => void>>();
const element = {
error: errorCode === undefined ? null : { code: errorCode },
loadCalls: 0,
preload: '',
muted: false,
src: '',
addEventListener: (event: string, listener: () => void) => {
if (!listeners.has(event)) {
listeners.set(event, new Set());
}
listeners.get(event)!.add(listener);
},
removeEventListener: (event: string, listener: () => void) => {
listeners.get(event)?.delete(listener);
},
removeAttribute: () => {},
load: () => {
element.loadCalls += 1;
},
emit: (event: string) => {
listeners.get(event)?.forEach((listener) => listener());
},
} as unknown as FakeMediaElement;

return element;
}

function probe(element: FakeMediaElement, src: string, kind: 'video' | 'audio' = 'video', options = {}) {
return probeMediaPlayability(src, kind, { createElement: () => element, ...options });
}

describe('probeMediaPlayability', () => {
beforeEach(() => {
resetMediaPlayabilityVerdicts();
jest.useRealTimers();
});

it('reports a file whose metadata loads as playable', async () => {
const element = fakeMediaElement();

const verdict = probe(element, 'cache://h264');
expect(element.preload).toBe('metadata');
expect(element.src).toBe('cache://h264');
element.emit('loadedmetadata');

await expect(verdict).resolves.toBe('playable');
});

it('reports a source the media pipeline cannot decode as unsupported', async () => {
const element = fakeMediaElement(4 /* MEDIA_ERR_SRC_NOT_SUPPORTED */);

const verdict = probe(element, 'cache://hevc');
element.emit('error');

await expect(verdict).resolves.toBe('unsupported');
});

it('reports a decode failure as unsupported', async () => {
const element = fakeMediaElement(3 /* MEDIA_ERR_DECODE */);

const verdict = probe(element, 'cache://corrupt', 'audio');
element.emit('error');

await expect(verdict).resolves.toBe('unsupported');
});

it('does not blame the file for a network error', async () => {
const element = fakeMediaElement(2 /* MEDIA_ERR_NETWORK */);

const verdict = probe(element, 'cache://gone');
element.emit('error');

await expect(verdict).resolves.toBe('unknown');
});

it('gives up as unknown when the element never reports', async () => {
jest.useFakeTimers();

const verdict = probe(fakeMediaElement(), 'cache://silent', 'video', { timeout: 1000 });
jest.advanceTimersByTime(1000);

await expect(verdict).resolves.toBe('unknown');
});

it('settles as unknown when aborted', async () => {
const controller = new AbortController();

const verdict = probe(fakeMediaElement(), 'cache://aborted', 'video', { signal: controller.signal });
controller.abort();

await expect(verdict).resolves.toBe('unknown');
});

it('releases the element once it has settled', async () => {
const element = fakeMediaElement();

const verdict = probe(element, 'cache://h264');
element.emit('loadedmetadata');
await verdict;

// one load() to start the probe, one after clearing src to release it
expect(element.loadCalls).toBe(2);
});

it('remembers a definite verdict and does not probe the same file again', async () => {
const element = fakeMediaElement(4);
const first = probe(element, 'cache://hevc');
element.emit('error');
await expect(first).resolves.toBe('unsupported');

const createElement = jest.fn();
await expect(probeMediaPlayability('cache://hevc', 'video', { createElement })).resolves.toBe('unsupported');
expect(createElement).not.toHaveBeenCalled();
});

it('probes again after an unknown verdict', async () => {
const first = fakeMediaElement(2);
const firstVerdict = probe(first, 'cache://flaky');
first.emit('error');
await expect(firstVerdict).resolves.toBe('unknown');

const second = fakeMediaElement();
const secondVerdict = probe(second, 'cache://flaky');
second.emit('loadedmetadata');
await expect(secondVerdict).resolves.toBe('playable');
});
});
Loading
Loading