Support gz/zstd/lz4 compression/decompression when writing/reading to files

[rigtorp]

In particular when doing long term captures and using the -G option it's useful to compress on the fly.

It's already possible to compress files using the post-rotate command, but it leads to write amplification. Capture files compresses well in my experience and with fast compressors such as LZ4 or zstd that can compress faster than disk bandwidth it can also increase maximum capture rate.

I have already implemented this in my own capture tool that focuses on efficient long term capture. The additional code is quite low, using fopencookie with pcap_dump_fopen. I'm happy to provide this under BSD license:

struct zstd_cookie {
  char *buf;
  size_t bufSize;
  ZSTD_CCtx *cctx;
  FILE *file;
};

static ssize_t zstd_cookie_write(void *cookie_, const char *buf, size_t size) {
  errno = 0;
  struct zstd_cookie *cookie = (struct zstd_cookie *)cookie_;
  ZSTD_inBuffer input = {buf, size, 0};
  for (;;) {
    ZSTD_outBuffer output = {cookie->buf, cookie->bufSize, 0};
    size_t const remaining =
        ZSTD_compressStream2(cookie->cctx, &output, &input, ZSTD_e_continue);
    if (ZSTD_isError(remaining) != 0) {
      goto err;
    }
    size_t const writtenSize = fwrite(output.dst, 1, output.pos, cookie->file);
    if (writtenSize != output.pos) {
      goto err;
    }
    if (remaining == 0) {
      break;
    }
  }
  return size;

err:
  if (errno == 0) {
    errno = EIO;
  }
  return 0;
}

static int zstd_cookie_close(void *cookie_) {
  errno = 0;
  struct zstd_cookie *cookie = (struct zstd_cookie *)cookie_;
  ZSTD_inBuffer input = {NULL, 0, 0};
  int rc = 0;
  for (;;) {
    ZSTD_outBuffer output = {cookie->buf, cookie->bufSize, 0};
    size_t const remaining =
        ZSTD_compressStream2(cookie->cctx, &output, &input, ZSTD_e_end);
    if (ZSTD_isError(remaining) != 0) {
      if (errno == 0) {
        errno = EIO;
      }
      rc = EOF;
      break;
    }
    size_t const writtenSize = fwrite(output.dst, 1, output.pos, cookie->file);
    if (writtenSize != output.pos) {
      if (errno == 0) {
        errno = EIO;
      }
      rc = EOF;
      break;
    }
    if (remaining == 0) {
      break;
    }
  }
  ZSTD_freeCCtx(cookie->cctx);
  free(cookie->buf);
  if (int rc2 = fclose(cookie->file); rc2 != 0) {
    rc = rc2;
  }
  free(cookie);
  return rc;
}

static const cookie_io_functions_t zstd_io_funcs = {
    .read = NULL,
    .write = zstd_cookie_write,
    .seek = NULL,
    .close = zstd_cookie_close,
};

static FILE *zstd_cookie_create(FILE *in, const char *mode) {
  struct zstd_cookie *cookie = NULL;

  cookie = (struct zstd_cookie *)malloc(sizeof(struct zstd_cookie));
  if (cookie == NULL) {
    goto err;
  }
  memset(cookie, 0, sizeof(struct zstd_cookie));
  cookie->file = in;
  cookie->bufSize = ZSTD_DStreamOutSize();
  cookie->buf = (char *)malloc(cookie->bufSize);
  if (cookie->buf == NULL) {
    goto err;
  }
  cookie->cctx = ZSTD_createCCtx();
  if (cookie->cctx == NULL) {
    goto err;
  }

  ZSTD_CCtx_setParameter(cookie->cctx, ZSTD_c_checksumFlag, 1);

  return fopencookie(cookie, mode, zstd_io_funcs);

err:
  if (cookie != nullptr) {
    free(cookie->buf);
    ZSTD_freeCCtx(cookie->cctx);
    free(cookie);
  }
  return NULL;
}

There's some funkiness with the error handling because pcap_dump doesn't return the error code from fwrite.