ncd.sh

#!/usr/bin/dash

# Normalized Compression Distance tool

# LICENSE:

# Copyright 2026, Noah Cashin <noahcashin@disroot.org>

# Permission to use, copy, modify, and/or distribute this software for any
# purpose with or without fee is hereby granted, provided that the above
# copyright notice and this permission notice appear in all copies.
#
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.

# End of LICENSE.

# MANUAL / DOCUMENTATION:
# This program aims to provide a portable normalized compression distance implementation.
# It should work on any UNIX-like OS. It works with any compressor program that can operate on files. 
# It only operates on files. It expects two arguments, paths. It will return the compressed size values as a table. Run it with no arguments, and pipe such a table into the program to compute NCD values. 
# This program supports arbitrary precision: iff the environment variable PRECISION is set to a valid integer, the NCD values will be computed to that precision. Otherwise, the default is 16.
# iff the environment variable NCD_SYMMETRIC is set(to anything, except nothing), it will return z(yx) in addition to z(xy) in the output table
# Also, the computed result will be done symmetrically if the aforementioned environment variable is set.
# Where z(x) is the byte length of x compressed with compressor z,
# computes, with a compressor z, and files x, and y: NCDz(x, y)
# which is equal to [(z(xy) - min{z(x), z(y)})/(max{z(x), z(y)})]
# if NCD_SYMMETRIC is set, z(xy) is substituted with min{z(xy), z(yx)}
# The NCD_COMPRESS should pass through a COMP_IN, and COMP_OUT path
# which are variables that allow this script to control the input and output files to a compressor.
# For example: `NCD_COMPRESS='gzip $COMP_IN --stdout >$COMP_OUT'`
# notice the single quotes, as to not actually evaluate those variables.
# But it should be evaluable shell for it to work!
# This requires a scratch directory, and will by default use $TMPDIR. That directory must have enough space.
# If doing large computations, it may be worth considering using a TMPDIR value that corresponds to 
# system memory, rather than an SSD, where it may cause wear.


usage() {
	echo "usage: $0 [path1 path2]" >&2
	echo "this returns raw compressed file size data" >&2
	echo "to compute NCD values, pipe the command into itself, for example:" >&2
	echo "$0 /path/to/file1 /path/to/file2 | $0" >&2
}

case $# in 
	0)
		if [ -t 0 ]; then
			usage
			exit 2;
		fi
		if [ -z "$PRECISION" ]; then
			PRECISION=16 # default
		fi
		# Main loop - calculate and return result one line at a time.
		while read line; do
			set -- $line
			if [ -n "$NCD_SYMMETRIC" ] && [ -n "$4" ]; then 
				if [ "$4" -lt "$3" ]; then
					set -- "$1" "$2" "$4" "$3"
				fi
			fi

			if [ "$1" -lt "$2" ]; then # true if 1 is min and 2 is max
				echo "scale=$PRECISION; ($3 - $1) / $2" | bc
			else
				echo "scale=$PRECISION; ($3 - $2) / $1" | bc
			fi
		done
		exit 0
		;;
	2)
		# Check if wc is available (should be!)
		if !  command -v wc >/dev/null 2>&1; then
			echo 'wc not found but required.' >&2
			exit 2;
		fi
		# Check if rm is available (should be!)
		if !  command -v rm >/dev/null 2>&1; then
			echo 'rm not found but required.' >&2
			exit 2;
		fi
		# TMPDIR should be present! Ideally, this is on a filesystem mapped to RAM when doing large operations, and with sufficient ram. Enough space is crucial.
		if [ -z "$TMPDIR" ]; then
			echo 'TMPDIR environment variable is not set.' >&2
			exit 2;
		fi
		# Check if the env var for the compression command is set.
		if [ -z "$NCD_COMPRESS" ]; then
			echo 'NCD_COMPRESS is not set' >&2
			exit 2;
		fi
		if [ ! -f  "$1" ] || [ ! -f "$2" ]; then
			echo "Error: paths $1 and/or $2 dont exist." >&2
			usage
			exit 2;
		fi
		COMP_IN="$1"
		COMP_OUT="$TMPDIR/ncdscratchpath.active_$$_file"
		eval $NCD_COMPRESS
		zx=$(wc -c < $COMP_OUT)
		rm $COMP_OUT
		COMP_IN="$2"
		eval $NCD_COMPRESS
		zy=$(wc -c < $COMP_OUT)
		rm $COMP_OUT
		COMP_IN="$TMPDIR/ncdscratchpath.active_$$_file2"
		cat $1 $2 > $COMP_IN
		eval $NCD_COMPRESS
		zxy=$(wc -c < $COMP_OUT)
		rm $COMP_OUT $COMP_IN
		zyx=
		if [ -n "$NCD_SYMMETRIC" ]; then
			cat $2 $1 > $COMP_IN
			eval $NCD_COMPRESS
			zyx=$(wc -c < $COMP_OUT)
			rm $COMP_OUT $COMP_IN
		fi
		echo $zx $zy $zxy $zyx
		exit 0
		;;
	*)
		usage
		exit 2
		;;
esac