Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 56 additions & 0 deletions data/check_md5_from_csv.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
#!/bin/bash

# Usage: ./check_md5_from_csv.sh checksums.csv /path/to/search
# Checksums obtained via Google Drive v3 API - see https://stackoverflow.com/a/49043352
# This script reads a CSV file with expected MD5 checksums and original filenames,
# searches for files in a specified directory, computes their MD5 checksums,
# and compares them against the expected values.
# It can handle filenames with optional '-XXX' suffixes before the extension, which
# is common when downloading directories from Google Drive.

CSV_FILE="$1"
SEARCH_DIR="$2"

if [[ ! -f "$CSV_FILE" || ! -d "$SEARCH_DIR" ]]; then
echo "Usage: $0 checksums.csv /path/to/search"
exit 1
fi

# Remove quotes from CSV (if any), skip header
tail -n +2 "$CSV_FILE" | sed 's/"//g' | while IFS=, read -r expected_md5 original_filename; do
# Build search pattern: original_filename plus optional '-XXX' before the extension
base_name="${original_filename%.*}" # e.g., imagenet_ai_0508_adm
extension="${original_filename##*.}" # e.g., z01

# Use glob pattern for find (literal * to match optional '-XXX')
search_pattern="${base_name}-*.$extension ${base_name}.$extension"

# Find files matching the pattern
matched_file=$(find "$SEARCH_DIR" -type f \( -name "${base_name}-*.$extension" -o -name "${base_name}.$extension" \) | head -n 1)

if [[ -z "$matched_file" ]]; then
echo "❌ File not found: $original_filename"
continue
fi

# Compute the md5sum
computed_md5=$(md5sum "$matched_file" | awk '{print $1}')

# Compare checksums
if [[ "$expected_md5" == "$computed_md5" ]]; then
# Check if filename differs
actual_filename=$(basename "$matched_file")
if [[ "$actual_filename" != "$original_filename" ]]; then
echo "✅ OK (but filename differs):"
echo " Expected: $original_filename"
echo " Found: $actual_filename"
else
echo "✅ OK: $original_filename"
fi
else
echo "❗ MISMATCH: $original_filename"
echo " Expected MD5: $expected_md5"
echo " Found MD5: $computed_md5"
echo " Actual file: $(basename "$matched_file")"
fi
done
209 changes: 209 additions & 0 deletions data/checksums.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,209 @@
md5Checksum,originalFilename
"c874b43e46d2b545c552d149a94c6102","imagenet_ai_0508_adm.z05"
"b715e17b61c12a0781da83697dc27a64","imagenet_ai_0508_adm.z06"
"791c1976c02dc53d52da5a86e210c422","imagenet_ai_0508_adm.z07"
"7ece834fa6b5b4dace4e01c544e18599","imagenet_ai_0508_adm.z02"
"ce367dc4f1bde4efa28dd3bfcf554ccc","imagenet_ai_0508_adm.z12"
"319afd6157d6be7b24713d3599f01866","imagenet_ai_0508_adm.z04"
"357f4a57e532da6e35b74e9da168f8d4","imagenet_ai_0508_adm.z10"
"eeb645b6c55457baad2e68e3d0c6bf06","imagenet_ai_0508_adm.z09"
"96dd364dcefb054b593850d9a3534fb4","imagenet_ai_0508_adm.z03"
"30dd7023222a479769398465e25fef8b","imagenet_ai_0508_adm.z08"
"a3bb1d9451d55eba9d45a71b9c57c247","imagenet_ai_0508_adm.z11"
"eda0803e5b8c24c42bdf12a33bb6137a","imagenet_ai_0508_adm.z01"
"2cce972c3793ee381b3c93b0efa26513","imagenet_ai_0508_adm.zip"
"03b8a8003ea2be5b9f92b4ee9f18692c","imagenet_ai_0419_biggan.zip"
"9ee76884bdd19a3ed363cc19143e57a1","imagenet_ai_0419_biggan.z07"
"615b966ada2e40ca60e2653a0ef9a353","imagenet_ai_0419_biggan.z01"
"9c745c6c5e4b30364ea1440ac6f1f241","imagenet_ai_0419_biggan.z03"
"3c0fcd7ffabdae0df1ff1e19e3d6bf0c","imagenet_ai_0419_biggan.z04"
"777f28e08396f9ed785698ff0fb3e6aa","imagenet_ai_0419_biggan.z06"
"c9cc2411048ebdd69d591a6df832f06a","imagenet_ai_0419_biggan.z05"
"8f4e7c0284cd369e3fe8a0a74a00eefb","imagenet_ai_0419_biggan.z02"
"3479bc2e81af2aacfb3b07e18ad3abad","imagenet_glide.zip"
"6f364e9730ef1cbeb914cfdfa8aa9018","imagenet_glide.z04"
"06ebcfca0b6d49ac42e28735ceeeab0a","imagenet_glide.z01"
"7c3ddf5100d1040f231c1b3e9a05d6bd","imagenet_glide.z10"
"b18e6af4289af0068b7b39927e02c1c5","imagenet_glide.z07"
"bc2772ffbd42ea355d608e65cb23294a","imagenet_glide.z06"
"15cc8b3b36c94288b92672fcd08aee16","imagenet_glide.z05"
"d58cc2ef5afec7b24f4c25f10a0a3191","imagenet_glide.z03"
"9ae8f0ccb86ed06e25b2ac32945b5aeb","imagenet_glide.z02"
"826f6586367a41b55e7267925a52d058","imagenet_glide.z09"
"856a50d87e3726a6accd1730e26dd762","imagenet_glide.z08"
"8a8dc050934216fdb27f7fe8b2455a67","imagenet_midjourney.z64"
"b4ec3bb2f17d4b2fa6a3c421a3ba08d7","imagenet_midjourney.z67"
"f4311072f2accefcff1c386acf85a7d9","imagenet_midjourney.z68"
"0cd3ee51a17de273b6e00c7bd463f52f","imagenet_midjourney.z65"
"bab2c7062ad7bc534c48ff006421b4ef","imagenet_midjourney.z37"
"d106ac51d0178bb7f309e7a838044a0f","imagenet_midjourney.z36"
"91e5c43ccf41cf293782c27a158be8a2","imagenet_midjourney.z70"
"fb57f9f1d52be7eaa23292b663b9e605","imagenet_midjourney.z69"
"3a3bb548bbaa404adbd00a3831c75812","imagenet_midjourney.z66"
"4f85ad6c1dcbb0a6a3e1acb91b4dd0b2","imagenet_midjourney.z63"
"a27d8ce2972796ecb950ad38e5f03d14","imagenet_midjourney.z62"
"0d87f42f12f9b47e384d94f4d8d25684","imagenet_midjourney.z59"
"6563afb8433c3b0c8b9be431f96ee293","imagenet_midjourney.z60"
"533543aaa9df6cb1834ea45bc5910b2c","imagenet_midjourney.z57"
"3b51017093336c5006241944c41d1947","imagenet_midjourney.z58"
"7a370798ba1e0ce555f9f087dad424f9","imagenet_midjourney.z40"
"0a4a79e800d7eebfeab8783f8f710278","imagenet_midjourney.z43"
"ee68165329c53ca0f303fc879d5804cf","imagenet_midjourney.z18"
"15c88b9e8d44615825fdcb91d14e80fb","imagenet_midjourney.z41"
"bdc525904babd5ac4422cec4e8fa4cd8","imagenet_midjourney.z09"
"e02e99992e08dd6396ea1114ef89044d","imagenet_midjourney.zip"
"60c7d1ef3bf9830225eeeac7afba5267","imagenet_midjourney.z61"
"67345f322ff8ebfd664cedcc0c28095d","imagenet_midjourney.z55"
"7bc6008dcb6f72a678d9bd52246ee38c","imagenet_midjourney.z56"
"560ead8d86cc16484f18ead8d1782133","imagenet_midjourney.z54"
"1cb2ee0ac0d627b185e525507c5a57bb","imagenet_midjourney.z50"
"937b14d940200038490238c113986a3e","imagenet_midjourney.z52"
"f821a92dd386a6a26414b0aa1b60642d","imagenet_midjourney.z32"
"91dc555519cdd493c2653d5cb5e27fe2","imagenet_midjourney.z53"
"cd2bc47f4c36708df26b027d701634d7","imagenet_midjourney.z46"
"28c14a54626d1029731966f9ea040b28","imagenet_midjourney.z51"
"a6e499847dc1ef31a5399d76f276327e","imagenet_midjourney.z45"
"f6be9577f412a3de6ec38cb49c38d2a2","imagenet_midjourney.z49"
"5324662078aacae2fd93ff0f678ca406","imagenet_midjourney.z48"
"7c1cccc1c1147ce9745e33eaa9df5499","imagenet_midjourney.z47"
"65113ae1df716c685954b61f549654f4","imagenet_midjourney.z44"
"02dc97ae5e7e4337679d4f05571134be","imagenet_midjourney.z42"
"637de8aa63676e5e186d5d2c7f05b75d","imagenet_midjourney.z39"
"ca6a314cd42bf28bae0e21e1771a9ec3","imagenet_midjourney.z35"
"4dc854166e51c8baee492e627803689d","imagenet_midjourney.z38"
"525b58ae6ffc83c216982b6d85024029","imagenet_midjourney.z34"
"e6ec273e2b2aee14807e7a847d54b851","imagenet_midjourney.z33"
"1c44ebd23fbaf386bb5992d70f416554","imagenet_midjourney.z31"
"3d5f1cfde5a50d46eb1866ac0c1a5eed","imagenet_midjourney.z30"
"73e39677bdd7b6a4dbbd4da067bd210e","imagenet_midjourney.z29"
"418644c422d60543a13dad1f77e0cef2","imagenet_midjourney.z28"
"2f52273c61d0c674361c2993b7c92a12","imagenet_midjourney.z27"
"c31aa612e295a121a56e742dce646794","imagenet_midjourney.z26"
"50a98304e94dda3cd373c45ae30ff2da","imagenet_midjourney.z25"
"2655aa7ad049fc85c27fb1b18d6edd17","imagenet_midjourney.z24"
"e56de848f91a4b0be0e54bd955c41dbd","imagenet_midjourney.z23"
"3b6e8de51432b2b02af5468ed3d4d7e8","imagenet_midjourney.z22"
"9d0bdefeabcb606e7d9f8af72882e8de","imagenet_midjourney.z21"
"f12aef6cc2836f6cc077a67b7e9affdc","imagenet_midjourney.z16"
"d5ed575ce9d5752bb42e35927778e6c8","imagenet_midjourney.z20"
"d8a10b7c3e8ef82caffb7914bcd76cd3","imagenet_midjourney.z17"
"ffa137b45bf6e77ee41a7ace03b58bb4","imagenet_midjourney.z19"
"9d67ebe4b3cbbb87a603bb6a66dbbb90","imagenet_midjourney.z15"
"e12232aebe99b2bf940891eee55b6c83","imagenet_midjourney.z14"
"81a0f8c9e4bc3638bbaa7f372479fc78","imagenet_midjourney.z13"
"cddf1bfa650e5371513819244053af0a","imagenet_midjourney.z12"
"a755a5c3cc58bc430ad27453627dd877","imagenet_midjourney.z11"
"caf588a5b6f750268e93a0bda8e08f46","imagenet_midjourney.z08"
"48c49eb7e336639e89b25068557bfd3f","imagenet_midjourney.z07"
"473a0e1ed164401c07bebf7c4172fb1a","imagenet_midjourney.z10"
"6279ab7ab7b1a5f0c41274efee465a23","imagenet_midjourney.z06"
"974569481ff502125b8d60fc1d745779","imagenet_midjourney.z01"
"141d8d749445ab1b0c935469fb36a233","imagenet_midjourney.z03"
"51f651e0c19c6c0391fe9d947ac30bb4","imagenet_midjourney.z04"
"172eb01af0733bd1e9001cd9f142b535","imagenet_midjourney.z05"
"fa1a242d9a02f2725d35b4d01b6e0257","imagenet_midjourney.z02"
"428de9eeab3154fe0db6882ed606eeee","imagenet_ai_0419_sdv4.z06"
"85700c74e89672f31735a28ff6ecfa67","imagenet_ai_0419_sdv4.z01"
"d9005baa539506c08a3a3c494c0cee70","imagenet_ai_0419_sdv4.z27"
"27a7228745b6b00d577999d895af4e46","imagenet_ai_0419_sdv4.z08"
"6d3809f1ffa1967a6cf131ffdfbd2e8c","imagenet_ai_0419_sdv4.z24"
"37e7c69763ddf7752c0a629ece94d5fe","imagenet_ai_0419_sdv4.z23"
"ff3ada9885eb6cf99672c50aa37ea38e","imagenet_ai_0419_sdv4.z10"
"fa588d46624c6965fda788637682f306","imagenet_ai_0419_sdv4.z02"
"f4abb9fd7c19fec8a6df24d8196e4f10","imagenet_ai_0419_sdv4.z05"
"eb7d38268bfc3168dab59cbdb72111a7","imagenet_ai_0419_sdv4.z29"
"531fd92856a9e2857da69c5c9f06fa06","imagenet_ai_0419_sdv4.zip"
"19298c2f807147e2a6e798017d69eb16","imagenet_ai_0419_sdv4.z28"
"02ae6fb73fd919a42c3633321ac64b9d","imagenet_ai_0419_sdv4.z26"
"c0ba5a7303ef40f14d5e9fa8f222c8ff","imagenet_ai_0419_sdv4.z25"
"94d1e50054310f153e7d9472c20af7c0","imagenet_ai_0419_sdv4.z22"
"a18a04fab2eb61550ccc8bf90535fd41","imagenet_ai_0419_sdv4.z21"
"ed0c7b164f8c97bfd3d9e34fd01df18d","imagenet_ai_0419_sdv4.z20"
"868fc79e62adeac9d6397e2a96a22a42","imagenet_ai_0419_sdv4.z19"
"4a1aa429072e8fc47b872b52caca9370","imagenet_ai_0419_sdv4.z18"
"11235a9422c6439d3a80a12f40043cb8","imagenet_ai_0419_sdv4.z16"
"16f991688b253142872156de43e71190","imagenet_ai_0419_sdv4.z17"
"dbce788e28443629b37d2de82d290536","imagenet_ai_0419_sdv4.z15"
"6e34cb00a2c6c30b733d134a342a60ae","imagenet_ai_0419_sdv4.z14"
"56d561c87906414cc3d4965fc256e98e","imagenet_ai_0419_sdv4.z13"
"0659316df324b0563cd342dbe1b929e6","imagenet_ai_0419_sdv4.z12"
"ebb3c65c7a38f4a79c91c3153a1dbf56","imagenet_ai_0419_sdv4.z11"
"059c888c444a976eb2692dfbc8b5144f","imagenet_ai_0419_sdv4.z07"
"c976259653a41c1062324a0a7db23d96","imagenet_ai_0419_sdv4.z09"
"ccfa4a9ef511f16614652932187d42e3","imagenet_ai_0419_sdv4.z04"
"9e690f3c8ee0ab86f85e817ad3bb3e37","imagenet_ai_0419_sdv4.z03"
"69e78d4d70d9f4db9a8464a63f7723df","imagenet_ai_0424_sdv5.z30"
"7a17abcdfcfbdbf70a6fa263a3d53958","imagenet_ai_0424_sdv5.z28"
"e856cb6d052f97ce502840b596c10f74","imagenet_ai_0424_sdv5.z29"
"2b51b8021b56db7baf9a5d3c930aa7b8","imagenet_ai_0424_sdv5.z27"
"837e3f5e9e577024423eb00e3a5da1b3","imagenet_ai_0424_sdv5.z25"
"6c58a22f29c61c0b6466895898c03762","imagenet_ai_0424_sdv5.z24"
"ab660dfca5510e555154419ee676b6e1","imagenet_ai_0424_sdv5.z23"
"4ed8e58b3b1040c2c9a62e2776dd9605","imagenet_ai_0424_sdv5.z22"
"0bd6a266153e5fb3e12223dbf32ba8f3","imagenet_ai_0424_sdv5.z21"
"332ffdf5ac65eaba4438815e89fe2acd","imagenet_ai_0424_sdv5.z20"
"6f9ed9f3714a7b2ea15a3ed436b24626","imagenet_ai_0424_sdv5.z19"
"a4424bd4c5b422f61f4e507556a2edcb","imagenet_ai_0424_sdv5.z26"
"a4424bd4c5b422f61f4e507556a2edcb","imagenet_ai_0424_sdv5.z26"
"922ec20e6a407020c0c8fbe86862b6c4","imagenet_ai_0424_sdv5.zip"
"4206fdd2a24bbb783ccb05477a0e3435","imagenet_ai_0424_sdv5.z18"
"d9f25eefd7232cd1e111bec25cce16e4","imagenet_ai_0424_sdv5.z17"
"fdb42d5595aa936b4642a1c354868c28","imagenet_ai_0424_sdv5.z16"
"7c69283c3b374c382264c934dfa83939","imagenet_ai_0424_sdv5.z15"
"2432c21eac40d5277176f3c4944335f6","imagenet_ai_0424_sdv5.z14"
"540413033e7964705a6f99929d2f1bf8","imagenet_ai_0424_sdv5.z13"
"3e8b935eeabf9888da1b66cb55c69344","imagenet_ai_0424_sdv5.z12"
"84a59f5fe83682d455bee8602be48434","imagenet_ai_0424_sdv5.z11"
"b344063e845097837cb04206fb5ce9e6","imagenet_ai_0424_sdv5.z10"
"f770e2f2c0e098b9f267abcc9f9cd59c","imagenet_ai_0424_sdv5.z09"
"b9094e2da3ae92dd8c0e35da4948921f","imagenet_ai_0424_sdv5.z08"
"e23d4974fd445765932d5d794c07e36a","imagenet_ai_0424_sdv5.z07"
"f9bf0edee690bab20d4b547336184f89","imagenet_ai_0424_sdv5.z06"
"64c388687ab9339cda0ec5bcd25cfbcf","imagenet_ai_0424_sdv5.z05"
"80c47527eb388ec300eb3dccaa765669","imagenet_ai_0424_sdv5.z04"
"ea8ead7549af217905b7732b2554b4c7","imagenet_ai_0424_sdv5.z03"
"f3d4ef66842b49c29e525104aa651c75","imagenet_ai_0424_sdv5.z02"
"ed5e6ea190e71cfb521ef780241bce60","imagenet_ai_0424_sdv5.z01"
"373b53891cd7440f0cf0379e6574a466","imagenet_ai_0419_vqdm.zip"
"b789f04bf7b9461627afa02f805bd1ed","imagenet_ai_0419_vqdm.z11"
"3ed0c0d35b2ecc29876b515f40cd3416","imagenet_ai_0419_vqdm.z10"
"487001439a65ae383bd5eb95a19b8920","imagenet_ai_0419_vqdm.z09"
"32c570a8051199055e4b31493dfe670b","imagenet_ai_0419_vqdm.z08"
"b7ab80396bf04d063474e0580e065e1e","imagenet_ai_0419_vqdm.z07"
"94bb180e456d5745837fc804e828db2d","imagenet_ai_0419_vqdm.z06"
"cd4ef83ab5113db99083e3b0d358df07","imagenet_ai_0419_vqdm.z05"
"ed5a5d834466e374481d34e5685011bd","imagenet_ai_0419_vqdm.z04"
"50f70cdc892ea00963a47d2126945199","imagenet_ai_0419_vqdm.z03"
"ad173fe1ee6cf388592b826e5a1530dc","imagenet_ai_0419_vqdm.z02"
"d9268c466f5ed02fb1137eb9a8ca236f","imagenet_ai_0419_vqdm.z01"
"bf58d04ea0fb2880a23b3e27044f6b12","imagenet_ai_0424_wukong.z26"
"b43504a4ef24b765462fa94072599012","imagenet_ai_0424_wukong.z30"
"f7a6ef7606ccf5dd9433b245bf4107f6","imagenet_ai_0424_wukong.zip"
"0186836839a07ae83f7beb604427659d","imagenet_ai_0424_wukong.z29"
"05d7a1aed06a76f9a23dcdef3bf15ae8","imagenet_ai_0424_wukong.z28"
"278113c937305ff2cf4f1f4ed896ade9","imagenet_ai_0424_wukong.z27"
"38f6e32581551b907518c55360d1e81d","imagenet_ai_0424_wukong.z19"
"735971eb370550e8282d0b5d0be4c47a","imagenet_ai_0424_wukong.z25"
"eb4cef3f820623032e16599d9236b806","imagenet_ai_0424_wukong.z24"
"e42b6f0edcae196484612ce24c6265a3","imagenet_ai_0424_wukong.z23"
"cd7a663c7e4d80557f7a68f2e8f8a576","imagenet_ai_0424_wukong.z22"
"c6e45cb75b4f4a30b97aa0e200808aad","imagenet_ai_0424_wukong.z12"
"9f4884c80502b9a24dcf468077895b52","imagenet_ai_0424_wukong.z21"
"25b57883eb3a816a23548511f45b1361","imagenet_ai_0424_wukong.z13"
"ab3b2864934d73239e4c053a7124dca3","imagenet_ai_0424_wukong.z20"
"e09030aaaf41e108cbded80c268b0875","imagenet_ai_0424_wukong.z18"
"cbd6d8556db3558e50ab08d5f31e013c","imagenet_ai_0424_wukong.z17"
"f2a049ec6c10a8f2fffd56ab175a0c1e","imagenet_ai_0424_wukong.z16"
"e94ade663c56402305f8c4cd960e3d00","imagenet_ai_0424_wukong.z15"
"d555c26a3676de30be176c49bc9be6f9","imagenet_ai_0424_wukong.z14"
"6464beadf9995460ff42af6d50db1fc9","imagenet_ai_0424_wukong.z08"
"fd908e7b970e91c3dba93bc94c214208","imagenet_ai_0424_wukong.z11"
"15e1f6289050aca68a50e32621bbc246","imagenet_ai_0424_wukong.z09"
"08ba53bf602d3f07bb52bc8cf91b08e3","imagenet_ai_0424_wukong.z10"
"b60452e85dfbc1551b05439b06159218","imagenet_ai_0424_wukong.z07"
"c897f976e261a34437a850f3a4e0a6b7","imagenet_ai_0424_wukong.z06"
"845f59a5382b3b08657561ca3b8e9223","imagenet_ai_0424_wukong.z05"
"75b498764d5dd9637e81cba5a6f5c7a6","imagenet_ai_0424_wukong.z02"
"2345dc3d5c3719b1cf1cfbab01e4e842","imagenet_ai_0424_wukong.z01"
"bad4a6df032ed8b329e2bdd72d5010cd","imagenet_ai_0424_wukong.z04"
"1efbdd424068333ce67ff8bec7038c8a","imagenet_ai_0424_wukong.z03"
4 changes: 4 additions & 0 deletions data/corrupted_files/imagenet_ai_0419_biggan.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
./imagenet_ai_0419_biggan/train/ai/116_biggan_00094.png
./imagenet_ai_0419_biggan/train/ai/116_biggan_00081.png
./imagenet_ai_0419_biggan/train/ai/116_biggan_00107.png
./imagenet_ai_0419_biggan/train/ai/116_biggan_00098.png
3 changes: 3 additions & 0 deletions data/corrupted_files/imagenet_ai_0419_sdv4.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
./imagenet_ai_0419_sdv4/train/ai/033_sdv4_00134.png
./imagenet_ai_0419_sdv4/train/ai/033_sdv4_00152.png
./imagenet_ai_0419_sdv4/train/ai/033_sdv4_00137.png
Loading