[File] [PATCH] Zstandard decompression using libzstd

[Christos Zoulas]

On 2022-09-13 8:09 pm, Martin Rodriguez Reboredo wrote:
> Uses the streaming decompression API of libzstd to obtain inflated info 
> for
> magic processing from Zstandard compressed data.

Committed, many thanks!

christos

> ---
>  configure.ac   | 19 ++++++++++++++
>  src/compress.c | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++
>  2 files changed, 87 insertions(+)
> 
> diff --git a/configure.ac b/configure.ac
> index ffbe0f69..9172be9d 100644
> --- a/configure.ac
> +++ b/configure.ac
> @@ -49,6 +49,11 @@ AC_ARG_ENABLE([xzlib],
>  [AS_HELP_STRING([--disable-xzlib], [disable liblzma/xz compression
> support @<:@default=auto@:>@])])
>  AC_MSG_RESULT($enable_xzlib)
> 
> +AC_MSG_CHECKING(for zstdlib support)
> +AC_ARG_ENABLE([zstdlib],
> +[AS_HELP_STRING([--disable-zstdlib], [disable zstdlib compression
> support @<:@default=auto@:>@])])
> +AC_MSG_RESULT($enable_zstdlib)
> +
>  AC_MSG_CHECKING(for libseccomp support)
>  AC_ARG_ENABLE([libseccomp],
>  [AS_HELP_STRING([--disable-libseccomp], [disable libseccomp
> sandboxing @<:@default=auto@:>@])])
> @@ -112,6 +117,9 @@ fi
>  if test "$enable_xzlib" != "no"; then
>    AC_CHECK_HEADERS(lzma.h)
>  fi
> +if test "$enable_zstdlib" != "no"; then
> +  AC_CHECK_HEADERS(zstd.h zstd_errors.h)
> +fi
>  AC_CHECK_TYPE([sig_t],[AC_DEFINE([HAVE_SIG_T],1,[Have sig_t
> type])],,[#include <signal.h>])
> 
>  dnl Checks for typedefs, structures, and compiler characteristics.
> @@ -180,6 +188,9 @@ fi
>  if test "$enable_xzlib" != "no"; then
>    AC_CHECK_LIB(lzma, lzma_stream_decoder)
>  fi
> +if test "$enable_zstdlib" != "no"; then
> +  AC_CHECK_LIB(zstd, ZSTD_createDStream)
> +fi
>  if test "$enable_libseccomp" != "no"; then
>      AC_CHECK_LIB(seccomp, seccomp_init)
>  fi
> @@ -215,6 +226,14 @@ fi
>  if  test "$ac_cv_header_lzma_h$ac_cv_lib_lzma_lzma_stream_decoder" =
> "yesyes"; then
>    AC_DEFINE([XZLIBSUPPORT], 1, [Enable xzlib compression support])
>  fi
> +if test "$enable_zstdlib" = "yes"; then
> +  if test "$ac_cv_header_zstd_h$ac_cv_lib_zstd_ZSTD_createDStream" !=
> "yesyes"; then
> +    AC_MSG_ERROR([zstdlib support requested but not found])
> +  fi
> +fi
> +if  test "$ac_cv_header_zstd_h$ac_cv_lib_zstd_ZSTD_createDStream" =
> "yesyes"; then
> +  AC_DEFINE([ZSTDLIBSUPPORT], 1, [Enable zstdlib compression support])
> +fi
> 
>  AC_CONFIG_FILES([Makefile src/Makefile magic/Makefile tests/Makefile
> doc/Makefile python/Makefile libmagic.pc])
>  AC_OUTPUT
> diff --git a/src/compress.c b/src/compress.c
> index 113077ae..abb8e766 100644
> --- a/src/compress.c
> +++ b/src/compress.c
> @@ -79,6 +79,12 @@ typedef void (*sig_t)(int);
>  #include <lzma.h>
>  #endif
> 
> +#if defined(HAVE_ZSTD_H) && defined(ZSTDLIBSUPPORT)
> +#define BUILTIN_ZSTDLIB
> +#include <zstd.h>
> +#include <zstd_errors.h>
> +#endif
> +
>  #ifdef DEBUG
>  int tty = -1;
>  #define DPRINTF(...)	do { \
> @@ -175,6 +181,7 @@ private const struct {
>  #define METH_FROZEN	2
>  #define METH_BZIP	7
>  #define METH_XZ		9
> +#define METH_ZSTD	12
>  #define METH_LZMA	13
>  #define METH_ZLIB	14
>      { { .magic = "\037\235" },	2, gzip_args, NULL },	/* 0, compressed 
> */
> @@ -223,6 +230,10 @@ private int uncompressbzlib(const unsigned char
> *, unsigned char **, size_t,
>  private int uncompressxzlib(const unsigned char *, unsigned char **, 
> size_t,
>      size_t *);
>  #endif
> +#ifdef BUILTIN_ZSTDLIB
> +private int uncompresszstd(const unsigned char *, unsigned char **, 
> size_t,
> +    size_t *);
> +#endif
> 
>  static int makeerror(unsigned char **, size_t *, const char *, ...)
>      __attribute__((__format__(__printf__, 3, 4)));
> @@ -697,6 +708,55 @@ err:
>  }
>  #endif
> 
> +#ifdef BUILTIN_ZSTDLIB
> +private int
> +uncompresszstd(const unsigned char *old, unsigned char **newch,
> +    size_t bytes_max, size_t *n)
> +{
> +	size_t rc;
> +	ZSTD_DStream *zstd;
> +	ZSTD_inBuffer in;
> +	ZSTD_outBuffer out;
> +
> +	if ((zstd = ZSTD_createDStream()) == NULL)
> +		return makeerror(newch, n, "No ZSTD decompression stream, %s",
> strerror(errno));
> +
> +	rc = ZSTD_DCtx_reset(zstd, ZSTD_reset_session_only);
> +	if (ZSTD_isError(rc))
> +		goto err;
> +
> +	if ((*newch = CAST(unsigned char *, malloc(bytes_max + 1))) == NULL) 
> {
> +		ZSTD_freeDStream(zstd);
> +		return makeerror(newch, n, "No buffer, %s", strerror(errno));
> +	}
> +
> +	in.src = CCAST(const void *, old);
> +	in.size = *n;
> +	in.pos = 0;
> +	out.dst = RCAST(void *, *newch);
> +	out.size = bytes_max;
> +	out.pos = 0;
> +
> +	rc = ZSTD_decompressStream(zstd, &out, &in);
> +	if (ZSTD_isError(rc))
> +		goto err;
> +
> +	*n = out.pos;
> +
> +	ZSTD_freeDStream(zstd);
> +
> +	/* let's keep the nul-terminate tradition */
> +	(*newch)[*n] = '\0';
> +
> +	return OKDATA;
> +err:
> +	ZSTD_freeDStream(zstd);
> +	snprintf(RCAST(char *, *newch), bytes_max, "zstd error %d",
> ZSTD_getErrorCode(rc));
> +	*n = strlen(RCAST(char *, *newch));
> +	return ERRDATA;
> +}
> +#endif
> +
> 
>  static int
>  makeerror(unsigned char **buf, size_t *len, const char *fmt, ...)
> @@ -863,6 +923,10 @@ methodname(size_t method)
>  	case METH_XZ:
>  	case METH_LZMA:
>  		return "xzlib";
> +#endif
> +#ifdef BUILTIN_ZSTDLIB
> +	case METH_ZSTD:
> +		return "zstd";
>  #endif
>  	default:
>  		return compr[method].argv[0];
> @@ -899,6 +963,10 @@ uncompressbuf(int fd, size_t bytes_max, size_t
> method, const unsigned char *old,
>  	case METH_XZ:
>  	case METH_LZMA:
>  		return uncompressxzlib(old, newch, bytes_max, n);
> +#endif
> +#ifdef BUILTIN_ZSTDLIB
> +	case METH_ZSTD:
> +		return uncompresszstd(old, newch, bytes_max, n);
>  #endif
>  	default:
>  		break;
> --
> 2.37.3

-- 
christos
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 195 bytes
Desc: OpenPGP digital signature
URL: <https://mailman.astron.com/pipermail/file/attachments/20220914/3c26103a/attachment.asc>