Zlib decompression on the DPU

DOCKER.md

@@ -0,0 +1,15 @@
+# UPMEM SDK in Docker
+
+How to obtain the UPMEM SDK environment on top of an Ubuntu image
+
+## Pull from Docker Hub
+To use a pre-built docker image, use:
+`docker pull penjiboy/upmem_sdk_base`
+
+## Build from source
+Alternatively, you can also build (and modify) the docker image from source using the Dockerfile in this repository. To build, use: 
+`docker build . -t upmem_sdk_base`
+
+## Usage
+It is often useful to place code in a mounted volume on the docker container. To run the container and mount a directory into the container:
+`docker run -it -v /full/path/to/local/directory:/root /bin/bash`

Dockerfile

@@ -0,0 +1,16 @@
+FROM ubuntu:18.04 as base
+
+#SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+RUN apt-get --yes update && apt-get install --yes --no-install-recommends openjdk-8-jre-headless python python3 gcc build-essential wget ranger libedit-dev libxml2-dev libpython2.7-dev
+
+WORKDIR /root
+
+RUN wget http://sdk-releases.upmem.com/2020.1.0/ubuntu_18.04/upmem-2020.1.0-Linux-x86_64.tar.gz
+RUN tar -xvzf upmem-2020.1.0-Linux-x86_64.tar.gz
+RUN mv upmem-2020.1.0-Linux-x86_64 upmem-2020.1.0
+
+#COPY "upmem-2019.4.0" "upmem-2019.4.0"
+
+RUN echo ". $HOME/upmem-2020.1.0/upmem_env.sh" >> $HOME/.bashrc
+

upmem-zlib/.gitignore

@@ -0,0 +1,3 @@
+test/
+decompress.dpu
+host_driver

upmem-zlib/Makefile

@@ -0,0 +1,35 @@
+RM 						= rm -rf
+
+DPU_CC 					= dpu-upmem-dpurte-clang
+DPU_CFLAGS 				= -Os -flto -g -w -fno-builtin-memcpy 
+
+CC 						= gcc
+CFLAGS 					= -g -w `dpu-pkg-config --cflags --libs dpu`
+
+DPU_COMMON_SOURCES		= zlib/adler32.c common/zutil.c 
+#DPU_COMMON_INCLUDES		= -Icommon -Izlib/ -Lzlib/
+DPU_COMMON_INCLUDES		= -Icommon
+
+COMMON_SOURCES			=
+COMMON_INCLUDES 		= -Izlib/lib/ -Lzlib/lib/ -lz -Icommon
+
+DPU_DECOMPRESS_SOURCES 	= dpu_decompress.c $(DPU_COMMON_SOURCES) decompress/inflate.c zlib/inffast.c common/inftrees.c 
+DPU_DECOMPRESS_INCLUDES = -Idecompress $(DPU_COMMON_INCLUDES)
+DPU_DECOMPRESS_PROGRAM  = decompress.dpu 
+
+HOST_SOURCES			= host_simple_example.c
+HOST_PROGRAM			= host_driver
+
+default: all
+
+all: $(HOST_PROGRAM) $(DPU_DECOMPRESS_PROGRAM) 
+
+
+$(HOST_PROGRAM): $(HOST_SOURCES)
+	$(CC) $(HOST_SOURCES) -o $(HOST_PROGRAM) $(CFLAGS) $(COMMON_INCLUDES)
+
+$(DPU_DECOMPRESS_PROGRAM): $(DPU_DECOMPRESS_SOURCES)
+	$(DPU_CC) $(DPU_DECOMPRESS_SOURCES) -o $(DPU_DECOMPRESS_PROGRAM) $(DPU_DECOMPRESS_INCLUDES) $(DPU_CFLAGS)
+
+clean:
+	$(RM) $(HOST_PROGRAM) $(DPU_DECOMPRESS_PROGRAM) 

upmem-zlib/NOTES.md

@@ -0,0 +1,16 @@
+# Notes
+
+Summary of what has been done.
+
+* Worth noting from the RFC that the spec "does not attempt to allow random access to compressed data"
+* Took the Zlib library and tried to adapt it to the DPU
+* Was able to compile the library for the DPU  by copying and including missing header files which are in the `common` directory
+* We used the `-O2 -flto` optimization compilation flags early on because the `.text` region wouldn't fit in IRAM
+* Trimmed down various parts of inflate:
+  * Removed the `CRC32` code, we only need to use the `Adler` checksum code
+  * Removed several functions `inflate.c` that seemed to be only used for testing/verification
+* Ran into an issue with clang compiler not able to lower memory intrinsic, fixed this by (mostly) using `fno-builtin-memcpy` compilation flag and by doing manual copies whenever we see a `memcpy` 
+* Prefixed a lot of the regular pointers in `struct inflate_state` and `struct z_stream_s` with `__mram_ptr`
+* In `zutil.c`, we modified `zcalloc` and `zfree` functions to return MRAM pointers (manual heap memory management)
+* There's a function called `mram_memcpy` floating around in `inflate.c` that is probably not documented anywhere. It does a regular `memcpy` but for MRAM pointers
+

upmem-zlib/README.md

@@ -0,0 +1,21 @@
+# Zlib on DPU
+
+Port of Zlib for a DPU-based solution.
+
+## Contents
+
+- `Makefile` compile and build executables
+- `zlib` contains the original zlib library
+- `host_simple_example.c` contains a host harnest:
+- `dpu_decompress.c` the dpu entry point for decompression
+- `common` sources required for all dpu programs
+- `decompress` sources required for decompression on the dpu
+
+## How to build
+
+- Make sure to build the zlib library following the appropriate instructions which are found in `zlib/Makefile.in`
+- run `make`
+
+## How to run
+- The executable we're using is `host_driver`, it should be generated when you run make
+- Usage: `host_driver --cpu/dpu --compress/decompress <inputfile> <outputfile>`

upmem-zlib/common/bits/a.out.h

@@ -0,0 +1,11 @@
+#ifndef __A_OUT_GNU_H__
+# error "Never use <bits/a.out.h> directly; include <a.out.h> instead."
+#endif
+
+#ifdef __x86_64__
+
+/* Signal to users of this header that this architecture really doesn't
+   support a.out binary format.  */
+#define __NO_A_OUT_SUPPORT 1
+
+#endif

upmem-zlib/common/bits/argp-ldbl.h

@@ -0,0 +1,24 @@
+/* Redirections for argp functions for -mlong-double-64.
+   Copyright (C) 2019 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#ifndef _ARGP_H
+# error "Never include <bits/argp-ldbl.h> directly; use <argp.h> instead."
+#endif
+
+__LDBL_REDIR_DECL (argp_error)
+__LDBL_REDIR_DECL (argp_failure)

upmem-zlib/common/bits/byteswap.h

@@ -0,0 +1,79 @@
+/* Macros and inline functions to swap the order of bytes in integer values.
+   Copyright (C) 1997-2019 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+#if !defined _BYTESWAP_H && !defined _NETINET_IN_H && !defined _ENDIAN_H
+# error "Never use <bits/byteswap.h> directly; include <byteswap.h> instead."
+#endif
+
+#ifndef _BITS_BYTESWAP_H
+#define _BITS_BYTESWAP_H 1
+
+#include <features.h>
+#include <bits/types.h>
+
+/* Swap bytes in 16-bit value.  */
+#define __bswap_constant_16(x)					\
+  ((__uint16_t) ((((x) >> 8) & 0xff) | (((x) & 0xff) << 8)))
+
+static __inline __uint16_t
+__bswap_16 (__uint16_t __bsx)
+{
+#if __GNUC_PREREQ (4, 8)
+  return __builtin_bswap16 (__bsx);
+#else
+  return __bswap_constant_16 (__bsx);
+#endif
+}
+
+/* Swap bytes in 32-bit value.  */
+#define __bswap_constant_32(x)					\
+  ((((x) & 0xff000000u) >> 24) | (((x) & 0x00ff0000u) >> 8)	\
+   | (((x) & 0x0000ff00u) << 8) | (((x) & 0x000000ffu) << 24))
+
+static __inline __uint32_t
+__bswap_32 (__uint32_t __bsx)
+{
+#if __GNUC_PREREQ (4, 3)
+  return __builtin_bswap32 (__bsx);
+#else
+  return __bswap_constant_32 (__bsx);
+#endif
+}
+
+/* Swap bytes in 64-bit value.  */
+#define __bswap_constant_64(x)			\
+  ((((x) & 0xff00000000000000ull) >> 56)	\
+   | (((x) & 0x00ff000000000000ull) >> 40)	\
+   | (((x) & 0x0000ff0000000000ull) >> 24)	\
+   | (((x) & 0x000000ff00000000ull) >> 8)	\
+   | (((x) & 0x00000000ff000000ull) << 8)	\
+   | (((x) & 0x0000000000ff0000ull) << 24)	\
+   | (((x) & 0x000000000000ff00ull) << 40)	\
+   | (((x) & 0x00000000000000ffull) << 56))
+
+__extension__ static __inline __uint64_t
+__bswap_64 (__uint64_t __bsx)
+{
+#if __GNUC_PREREQ (4, 3)
+  return __builtin_bswap64 (__bsx);
+#else
+  return __bswap_constant_64 (__bsx);
+#endif
+}
+
+#endif /* _BITS_BYTESWAP_H */

upmem-zlib/common/bits/cmathcalls.h

@@ -0,0 +1,130 @@
+/* Prototype declarations for complex math functions;
+   helper file for <complex.h>.
+   Copyright (C) 1997-2019 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+
+/* NOTE: Because of the special way this file is used by <complex.h>, this
+   file must NOT be protected from multiple inclusion as header files
+   usually are.
+
+   This file provides prototype declarations for the math functions.
+   Most functions are declared using the macro:
+
+   __MATHCALL (NAME, (ARGS...));
+
+   This means there is a function `NAME' returning `double' and a function
+   `NAMEf' returning `float'.  Each place `_Mdouble_' appears in the
+   prototype, that is actually `double' in the prototype for `NAME' and
+   `float' in the prototype for `NAMEf'.  Reentrant variant functions are
+   called `NAME_r' and `NAMEf_r'.
+
+   Functions returning other types like `int' are declared using the macro:
+
+   __MATHDECL (TYPE, NAME, (ARGS...));
+
+   This is just like __MATHCALL but for a function returning `TYPE'
+   instead of `_Mdouble_'.  In all of these cases, there is still
+   both a `NAME' and a `NAMEf' that takes `float' arguments.  */
+
+#ifndef _COMPLEX_H
+#error "Never use <bits/cmathcalls.h> directly; include <complex.h> instead."
+#endif
+
+#ifndef _Mdouble_complex_
+# define _Mdouble_complex_ _Mdouble_ _Complex
+#endif
+
+
+/* Trigonometric functions.  */
+
+/* Arc cosine of Z.  */
+__MATHCALL (cacos, (_Mdouble_complex_ __z));
+/* Arc sine of Z.  */
+__MATHCALL (casin, (_Mdouble_complex_ __z));
+/* Arc tangent of Z.  */
+__MATHCALL (catan, (_Mdouble_complex_ __z));
+
+/* Cosine of Z.  */
+__MATHCALL (ccos, (_Mdouble_complex_ __z));
+/* Sine of Z.  */
+__MATHCALL (csin, (_Mdouble_complex_ __z));
+/* Tangent of Z.  */
+__MATHCALL (ctan, (_Mdouble_complex_ __z));
+
+
+/* Hyperbolic functions.  */
+
+/* Hyperbolic arc cosine of Z.  */
+__MATHCALL (cacosh, (_Mdouble_complex_ __z));
+/* Hyperbolic arc sine of Z.  */
+__MATHCALL (casinh, (_Mdouble_complex_ __z));
+/* Hyperbolic arc tangent of Z.  */
+__MATHCALL (catanh, (_Mdouble_complex_ __z));
+
+/* Hyperbolic cosine of Z.  */
+__MATHCALL (ccosh, (_Mdouble_complex_ __z));
+/* Hyperbolic sine of Z.  */
+__MATHCALL (csinh, (_Mdouble_complex_ __z));
+/* Hyperbolic tangent of Z.  */
+__MATHCALL (ctanh, (_Mdouble_complex_ __z));
+
+
+/* Exponential and logarithmic functions.  */
+
+/* Exponential function of Z.  */
+__MATHCALL (cexp, (_Mdouble_complex_ __z));
+
+/* Natural logarithm of Z.  */
+__MATHCALL (clog, (_Mdouble_complex_ __z));
+
+#ifdef __USE_GNU
+/* The base 10 logarithm is not defined by the standard but to implement
+   the standard C++ library it is handy.  */
+__MATHCALL (clog10, (_Mdouble_complex_ __z));
+#endif
+
+/* Power functions.  */
+
+/* Return X to the Y power.  */
+__MATHCALL (cpow, (_Mdouble_complex_ __x, _Mdouble_complex_ __y));
+
+/* Return the square root of Z.  */
+__MATHCALL (csqrt, (_Mdouble_complex_ __z));
+
+
+/* Absolute value, conjugates, and projection.  */
+
+/* Absolute value of Z.  */
+__MATHDECL (_Mdouble_,cabs, (_Mdouble_complex_ __z));
+
+/* Argument value of Z.  */
+__MATHDECL (_Mdouble_,carg, (_Mdouble_complex_ __z));
+
+/* Complex conjugate of Z.  */
+__MATHCALL (conj, (_Mdouble_complex_ __z));
+
+/* Projection of Z onto the Riemann sphere.  */
+__MATHCALL (cproj, (_Mdouble_complex_ __z));
+
+
+/* Decomposing complex values.  */
+
+/* Imaginary part of Z.  */
+__MATHDECL (_Mdouble_,cimag, (_Mdouble_complex_ __z));
+
+/* Real part of Z.  */
+__MATHDECL (_Mdouble_,creal, (_Mdouble_complex_ __z));