Add an experimental feature to automatically detect Base64 encoded content in the input stream and optimize compression by skipping dictionary and history lookups for these regions. This is controlled by a new parameter BROTLI_PARAM_BASE64_MODE.

c/enc/backward_references.c

@@ -18,6 +18,68 @@
 #include "params.h"
 #include "quality.h"  /* IWYU pragma: keep for inc */
 
+BROTLI_INTERNAL const BROTLI_MODEL("small")
+    uint8_t kIsBase64[256] = {
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1,
+        0, 0, 0, 1, /* 43 '+', 47 '/' (45 '-' is 0) */
+        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, /* 48-57 '0'-'9' (61 '='
+                                                           is 0) */
+        0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 65-79 'A'-'O' */
+        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 80-90 'P'-'Z' */
+        0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, /* 97-111 'a'-'o' */
+        1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, /* 112-122 'p'-'z' */
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+
+static const size_t kBase64TriggerLen = 8;
+
+static BROTLI_INLINE BROTLI_BOOL IsBase64Char(uint8_t c) {
+  return TO_BROTLI_BOOL(kIsBase64[c]);
+}
+
+static BROTLI_INLINE BROTLI_BOOL MatchTrigger(const uint8_t* ringbuffer,
+                                              size_t mask, size_t pos) {
+  const char* trigger = ";base64,";
+  size_t i;
+  for (i = 0; i < kBase64TriggerLen; ++i) {
+    if (ringbuffer[(pos + i) & mask] != trigger[i]) return BROTLI_FALSE;
+  }
+  return BROTLI_TRUE;
+}
+
+static size_t FindNextBase64Trigger(const uint8_t* ringbuffer, size_t mask,
+                                    size_t pos, size_t end) {
+  while (pos + kBase64TriggerLen <= end) {
+    size_t ringbuffer_size = mask + 1;
+    size_t pos_index = pos & mask;
+    size_t contiguous_len = ringbuffer_size - pos_index;
+    size_t max_scan_len = end - pos;
+    size_t scan_len = BROTLI_MIN(size_t, contiguous_len, max_scan_len);
+
+    const uint8_t* p =
+        (const uint8_t*)memchr(&ringbuffer[pos_index], ';', scan_len);
+    if (p != NULL) {
+      size_t offset = (size_t)(p - &ringbuffer[pos_index]);
+      if (pos + offset + kBase64TriggerLen <= end) {
+        if (MatchTrigger(ringbuffer, mask, pos + offset)) {
+          return pos + offset;
+        }
+      } else {
+        return end;
+      }
+      pos += offset + 1;
+    } else {
+      pos += scan_len;
+    }
+  }
+  return end;
+}
+
 #if defined(__cplusplus) || defined(c_plusplus)
 extern "C" {
 #endif

c/enc/backward_references_inc.h

@@ -35,7 +35,60 @@ static BROTLI_NOINLINE void EXPORT_FN(CreateBackwardReferences)(
 
   FN(PrepareDistanceCache)(privat, dist_cache);
 
+  size_t next_base64_pos = pos_end;
+  if (params->base64_mode &&
+      hasher->common.num_base64_regions < params->max_base64_regions) {
+    next_base64_pos =
+        FindNextBase64Trigger(ringbuffer, ringbuffer_mask, position, pos_end);
+  }
   while (position + FN(HashTypeLength)() < pos_end) {
+    if (position >= next_base64_pos) {
+      /* Find where it ends */
+      size_t scan_pos = position + kBase64TriggerLen;
+      size_t first_equal_pos = 0;
+      while (scan_pos < pos_end) {
+        uint8_t c = ringbuffer[scan_pos & ringbuffer_mask];
+        if (IsBase64Char(c)) {
+          if (first_equal_pos != 0) {
+            scan_pos = first_equal_pos;
+            break;
+          }
+          scan_pos++;
+        } else if (c == '=') {
+          if (first_equal_pos == 0) {
+            first_equal_pos = scan_pos;
+          }
+          scan_pos++;
+        } else {
+          break;
+        }
+      }
+      /* Jump directly to the end of base64 block */
+      /* Skip the ';base64,' trigger */
+      size_t start_pos = position + kBase64TriggerLen;
+      size_t length = scan_pos - start_pos;
+      /* Exclude '=' characters from the flat 6-bit entropy block */
+      while (length > 0 &&
+             ringbuffer[(start_pos + length - 1) & ringbuffer_mask] == '=') {
+        length--;
+      }
+      if (length > 0) {
+        hasher->common.base64_regions[hasher->common.num_base64_regions]
+            .start_literal_pos = start_pos;
+        hasher->common.base64_regions[hasher->common.num_base64_regions]
+            .length = length;
+        hasher->common.num_base64_regions++;
+      }
+      insert_length += (scan_pos - position);
+      position = scan_pos;
+      if (hasher->common.num_base64_regions < params->max_base64_regions) {
+        next_base64_pos = FindNextBase64Trigger(ringbuffer, ringbuffer_mask,
+                                                position, pos_end);
+      } else {
+        next_base64_pos = pos_end;
+      }
+      continue;
+    }
     size_t max_length = pos_end - position;
     size_t max_distance = BROTLI_MIN(size_t, position, max_backward_limit);
     size_t dictionary_start = BROTLI_MIN(size_t,

c/enc/block_encoder_inc.h

@@ -1,20 +1,14 @@
-/* NOLINT(build/header_guard) */
-/* Copyright 2014 Google Inc. All Rights Reserved.
-
-   Distributed under MIT license.
-   See file LICENSE for detail or copy at https://opensource.org/licenses/MIT
-*/
-
-/* template parameters: FN */
+#if !defined(BROTLI_BASE64_LUT_)
+#define BROTLI_BASE64_LUT_
+BROTLI_INTERNAL extern const BROTLI_MODEL("small") uint8_t kIsBase64[256];
+#endif
 
 #define HistogramType FN(Histogram)
 
-/* Creates entropy codes for all block types and stores them to the bit
-   stream. */
-static void FN(BuildAndStoreEntropyCodes)(MemoryManager* m, BlockEncoder* self,
-    const HistogramType* histograms, const size_t histograms_size,
-    const size_t alphabet_size, HuffmanTree* tree,
-    size_t* storage_ix, uint8_t* storage) {
+static void FN(BuildAndStoreEntropyCodes)(
+    MemoryManager* m, BlockEncoder* self, const HistogramType* histograms,
+    const size_t histograms_size, const size_t alphabet_size, HuffmanTree* tree,
+    const uint8_t* is_base64_histogram, size_t* storage_ix, uint8_t* storage) {
   const size_t table_size = histograms_size * self->histogram_length_;
   self->depths_ = BROTLI_ALLOC(m, uint8_t, table_size);
   self->bits_ = BROTLI_ALLOC(m, uint16_t, table_size);
@@ -24,9 +18,24 @@ static void FN(BuildAndStoreEntropyCodes)(MemoryManager* m, BlockEncoder* self,
     size_t i;
     for (i = 0; i < histograms_size; ++i) {
       size_t ix = i * self->histogram_length_;
-      BuildAndStoreHuffmanTree(&histograms[i].data_[0], self->histogram_length_,
-          alphabet_size, tree, &self->depths_[ix], &self->bits_[ix],
-          storage_ix, storage);
+      if (self->histogram_length_ == 256 && is_base64_histogram &&
+          is_base64_histogram[i]) {
+        size_t k;
+        memset(&self->depths_[ix], 0, 256);
+        for (k = 0; k < 256; ++k) {
+          if (kIsBase64[k]) {
+            self->depths_[ix + k] = 6;
+          }
+        }
+        BrotliConvertBitDepthsToSymbols(&self->depths_[ix], 256,
+                                        &self->bits_[ix]);
+        BrotliStoreHuffmanTree(&self->depths_[ix], 256, tree, storage_ix,
+                               storage);
+      } else {
+        BuildAndStoreHuffmanTree(
+            &histograms[i].data_[0], self->histogram_length_, alphabet_size,
+            tree, &self->depths_[ix], &self->bits_[ix], storage_ix, storage);
+      }
     }
   }
 }

c/enc/brotli_bit_stream.c

@@ -1017,16 +1017,30 @@ void BrotliStoreMetaBlock(MemoryManager* m,
     if (BROTLI_IS_OOM(m)) return;
   }
 
-  BuildAndStoreEntropyCodesLiteral(m, literal_enc, mb->literal_histograms,
-      mb->literal_histograms_size, BROTLI_NUM_LITERAL_SYMBOLS, tree,
-      storage_ix, storage);
+  {
+    uint8_t is_base64_histogram[256] = {0};
+    if (mb->literal_split.num_types > 0) {
+      size_t b64_type_id = mb->literal_split.num_types - 1;
+      if (b64_type_id < 256 && (mb->literal_is_base64[b64_type_id >> 3] & (1u << (b64_type_id & 7)))) {
+        uint32_t b64_histo_id = mb->literal_context_map ? mb->literal_context_map[b64_type_id << 6] : (uint32_t)b64_type_id;
+        is_base64_histogram[b64_histo_id] = 1;
+      }
+    }
+
+    BuildAndStoreEntropyCodesLiteral(m, literal_enc, mb->literal_histograms,
+        mb->literal_histograms_size, BROTLI_NUM_LITERAL_SYMBOLS, tree,
+        is_base64_histogram,
+        storage_ix, storage);
+  }
   if (BROTLI_IS_OOM(m)) return;
   BuildAndStoreEntropyCodesCommand(m, command_enc, mb->command_histograms,
       mb->command_histograms_size, BROTLI_NUM_COMMAND_SYMBOLS, tree,
+      NULL,
       storage_ix, storage);
   if (BROTLI_IS_OOM(m)) return;
   BuildAndStoreEntropyCodesDistance(m, distance_enc, mb->distance_histograms,
       mb->distance_histograms_size, num_distance_symbols, tree,
+      NULL,
       storage_ix, storage);
   if (BROTLI_IS_OOM(m)) return;
   BROTLI_FREE(m, tree);

c/enc/encode.c

@@ -105,6 +105,14 @@ BROTLI_BOOL BrotliEncoderSetParameter(
       state->params.stream_offset = value;
       return BROTLI_TRUE;
 
+    case BROTLI_PARAM_BASE64_MODE:
+      state->params.base64_mode = (int)(value & 1);
+      return BROTLI_TRUE;
+
+    case BROTLI_PARAM_MAX_BASE64_REGIONS:
+      state->params.max_base64_regions = value;
+      return BROTLI_TRUE;
+
     default: return BROTLI_FALSE;
   }
 }
@@ -488,6 +496,8 @@ static void WriteMetaBlockInternal(MemoryManager* m,
                                    const uint64_t last_flush_pos,
                                    const size_t bytes,
                                    const BROTLI_BOOL is_last,
+                                   const Base64Region* base64_regions,
+                                   size_t num_base64_regions,
                                    ContextType literal_context_mode,
                                    const BrotliEncoderParams* params,
                                    const uint8_t prev_byte,
@@ -499,7 +509,6 @@ static void WriteMetaBlockInternal(MemoryManager* m,
                                    int* dist_cache,
                                    size_t* storage_ix,
                                    uint8_t* storage) {
-  const uint32_t wrapped_last_flush_pos = WrapPosition(last_flush_pos);
   uint16_t last_bytes;
   uint8_t last_bytes_bits;
   ContextLut literal_context_lut = BROTLI_CONTEXT_LUT(literal_context_mode);
@@ -518,7 +527,7 @@ static void WriteMetaBlockInternal(MemoryManager* m,
        CreateBackwardReferences is now unused. */
     memcpy(dist_cache, saved_dist_cache, 4 * sizeof(dist_cache[0]));
     BrotliStoreUncompressedMetaBlock(is_last, data,
-                                     wrapped_last_flush_pos, mask, bytes,
+                                     (size_t)last_flush_pos, mask, bytes,
                                      storage_ix, storage);
     return;
   }
@@ -527,13 +536,13 @@ static void WriteMetaBlockInternal(MemoryManager* m,
   last_bytes = (uint16_t)((storage[1] << 8) | storage[0]);
   last_bytes_bits = (uint8_t)(*storage_ix);
   if (params->quality <= MAX_QUALITY_FOR_STATIC_ENTROPY_CODES) {
-    BrotliStoreMetaBlockFast(m, data, wrapped_last_flush_pos,
+    BrotliStoreMetaBlockFast(m, data, (size_t)last_flush_pos,
                              bytes, mask, is_last, params,
                              commands, num_commands,
                              storage_ix, storage);
     if (BROTLI_IS_OOM(m)) return;
   } else if (params->quality < MIN_QUALITY_FOR_BLOCK_SPLIT) {
-    BrotliStoreMetaBlockTrivial(m, data, wrapped_last_flush_pos,
+    BrotliStoreMetaBlockTrivial(m, data, (size_t)last_flush_pos,
                                 bytes, mask, is_last, params,
                                 commands, num_commands,
                                 storage_ix, storage);
@@ -550,17 +559,20 @@ static void WriteMetaBlockInternal(MemoryManager* m,
             BROTLI_ALLOC(m, uint32_t, 32 * (BROTLI_MAX_STATIC_CONTEXTS + 1));
         if (BROTLI_IS_OOM(m) || BROTLI_IS_NULL(arena)) return;
         DecideOverLiteralContextModeling(
-            data, wrapped_last_flush_pos, bytes, mask, params->quality,
+            data, (size_t)last_flush_pos, bytes, mask, params->quality,
             params->size_hint, &num_literal_contexts,
             &literal_context_map, arena);
         BROTLI_FREE(m, arena);
       }
-      BrotliBuildMetaBlockGreedy(m, data, wrapped_last_flush_pos, mask,
+      BrotliBuildMetaBlockGreedy(m, data, (size_t)last_flush_pos, mask,
+          base64_regions, num_base64_regions,
           prev_byte, prev_byte2, literal_context_lut, num_literal_contexts,
           literal_context_map, commands, num_commands, &mb);
       if (BROTLI_IS_OOM(m)) return;
     } else {
-      BrotliBuildMetaBlock(m, data, wrapped_last_flush_pos, mask, &block_params,
+      BrotliBuildMetaBlock(m, data, (size_t)last_flush_pos, mask,
+                           base64_regions, num_base64_regions,
+                           &block_params,
                            prev_byte, prev_byte2,
                            commands, num_commands,
                            literal_context_mode,
@@ -573,7 +585,7 @@ static void WriteMetaBlockInternal(MemoryManager* m,
          for "Large Window Brotli" (32-bit). */
       BrotliOptimizeHistograms(block_params.dist.alphabet_size_limit, &mb);
     }
-    BrotliStoreMetaBlock(m, data, wrapped_last_flush_pos, bytes, mask,
+    BrotliStoreMetaBlock(m, data, (size_t)last_flush_pos, bytes, mask,
                          prev_byte, prev_byte2,
                          is_last,
                          &block_params,
@@ -591,7 +603,7 @@ static void WriteMetaBlockInternal(MemoryManager* m,
     storage[1] = (uint8_t)(last_bytes >> 8);
     *storage_ix = last_bytes_bits;
     BrotliStoreUncompressedMetaBlock(is_last, data,
-                                     wrapped_last_flush_pos, mask,
+                                     (size_t)last_flush_pos, mask,
                                      bytes, storage_ix, storage);
   }
 }
@@ -688,6 +700,8 @@ static void BrotliEncoderInitParams(BrotliEncoderParams* params) {
   params->size_hint = 0;
   params->disable_literal_context_modeling = BROTLI_FALSE;
   BrotliInitSharedEncoderDictionary(&params->dictionary);
+  params->base64_mode = (int)BROTLI_DEFAULT_BASE64_MODE;
+  params->max_base64_regions = BROTLI_DEFAULT_MAX_BASE64_REGIONS;
   params->dist.distance_postfix_bits = 0;
   params->dist.num_direct_distance_codes = 0;
   params->dist.alphabet_size_max =
@@ -757,6 +771,7 @@ static void BrotliEncoderInitState(BrotliEncoderState* s) {
   /* Save the state of the distance cache in case we need to restore it for
      emitting an uncompressed block. */
   memcpy(s->saved_dist_cache_, s->dist_cache_, sizeof(s->saved_dist_cache_));
+  s->hasher_.common.num_base64_regions = 0;
 }
 
 BrotliEncoderState* BrotliEncoderCreateInstance(
@@ -1170,10 +1185,12 @@ static BROTLI_BOOL EncodeData(
     storage[1] = (uint8_t)(s->last_bytes_ >> 8);
     WriteMetaBlockInternal(
         m, data, mask, s->last_flush_pos_, metablock_size, is_last,
+        s->hasher_.common.base64_regions, s->hasher_.common.num_base64_regions,
         literal_context_mode, &s->params, s->prev_byte_, s->prev_byte2_,
         s->num_literals_, s->num_commands_, s->commands_, s->saved_dist_cache_,
         s->dist_cache_, &storage_ix, storage);
     if (BROTLI_IS_OOM(m)) return BROTLI_FALSE;
+    s->hasher_.common.num_base64_regions = 0;
     s->last_bytes_ = (uint16_t)(storage[storage_ix >> 3]);
     s->last_bytes_bits_ = storage_ix & 7u;
     s->last_flush_pos_ = s->input_pos_;