[File] Excessive memory allocation in CDF chain parsing

[Acts1631]

Hello,

A crafted Composite Document File can make libmagic request
multi-gigabyte allocations while parsing attacker-controlled input.

The issue is in the CDF parser's sector-chain handling. The header
accepts sector sizes up to 1 MiB, while cdf_count_chain() limits only
the number of sectors to 10,000. cdf_read_dir() then multiplies that
count by the number of directory entries per sector and passes the
result to CDF_CALLOC() without a memory bound. The same unbounded
chain length also reaches SSAT and stream allocations.

A 2 MiB crafted input with a 1 MiB sector size and a 10,000-sector
directory chain caused the unpatched libFuzzer harness to request
malloc(11141120000) in cdf_read_dir(). An attacker who can make a
process call magic_buffer() or magic_descriptor() on the file can
therefore cause excessive memory consumption or process termination,
depending on allocator and resource limits.

The patch adds a shared 16 MiB CDF memory limit, checks the
sat_len * size calculation before deriving the sector bound, limits
all cdf_count_chain() consumers, and checks the directory allocation
before calling CDF_CALLOC().

diff --git a/src/cdf.c b/src/cdf.c
index 2da240c..1deba80 100644
--- a/src/cdf.c
+++ b/src/cdf.c
@@ -490,8 +490,7 @@ cdf_read_sat(const cdf_info_t *info, cdf_header_t *h, cdf_sat_t *sat)
 	}
 
 	sat->sat_len = h->h_num_sectors_in_master_sat * nsatpersec + i;
-#define CDF_SAT_LIMIT	(16 * 1024 * 1024)
-	if (ss != 0 && sat->sat_len > CDF_SAT_LIMIT / ss) {
+	if (ss != 0 && sat->sat_len > CDF_MEMORY_LIMIT / ss) {
 		errno = EFTYPE;
 		return -1;
 	}
@@ -563,8 +562,12 @@ size_t
 cdf_count_chain(const cdf_sat_t *sat, cdf_secid_t sid, size_t size)
 {
 	size_t i, j;
-	cdf_secid_t maxsector = CAST(cdf_secid_t, (sat->sat_len * size)
-	    / sizeof(maxsector));
+	cdf_secid_t maxsector;
+
+	if (size == 0 || sat->sat_len > SIZE_T_MAX / size)
+		goto out;
+	maxsector = CAST(cdf_secid_t, (sat->sat_len * size) /
+	    sizeof(maxsector));
 
 	DPRINTF(("Chain:"));
 	if (sid == CDF_SECID_END_OF_CHAIN) {
@@ -579,6 +582,10 @@ cdf_count_chain(const cdf_sat_t *sat, cdf_secid_t sid, size_t size)
 			DPRINTF(("Counting chain loop limit"));
 			goto out;
 		}
+		if (i >= CDF_MEMORY_LIMIT / size) {
+			DPRINTF(("Counting chain size limit"));
+			goto out;
+		}
 		if (sid >= maxsector) {
 			DPRINTF(("Sector %d >= %d\n", sid, maxsector));
 			goto out;
@@ -715,6 +722,12 @@ cdf_read_dir(const cdf_info_t *info, const cdf_header_t *h,
 		return -1;
 
 	nd = ss / CDF_DIRECTORY_SIZE;
+	if (nd != 0 && ns > CDF_MEMORY_LIMIT / nd /
+	    sizeof(dir->dir_tab[0])) {
+		DPRINTF(("Directory size limit"));
+		errno = EFTYPE;
+		return -1;
+	}
 
 	dir->dir_len = ns * nd;
 	dir->dir_tab = CAST(cdf_directory_t *,
diff --git a/src/cdf.h b/src/cdf.h
index 6dddb3d..7f47555 100644
--- a/src/cdf.h
+++ b/src/cdf.h
@@ -49,6 +49,7 @@ typedef int32_t cdf_secid_t;
 
 #define CDF_LOOP_LIMIT					10000
 #define CDF_ELEMENT_LIMIT				100000
+#define CDF_MEMORY_LIMIT				(16 * 1024 * 1024)
 
 #define CDF_SECID_NULL					0
 #define CDF_SECID_FREE					-1