GR-raytracing/src/fast_psf_fftw.c

#include "fast_psf_fftw.h"

#include <fftw3.h>
#include <errno.h>
#include <limits.h>
#include <math.h>
#include <omp.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

/* Process-global FFTW threading lifecycle.  Plans are created only from the
 * serial control thread, and global cleanup happens only after the last live
 * state is gone.  This keeps the individual states from racing on FFTW's
 * process-wide planner state. */
static int g_threading_initialized = 0;
static int g_live_states = 0;
static FastPsfFftwPlanMode g_plan_mode = FAST_PSF_FFTW_PLAN_ESTIMATE;
static const char *g_wisdom_path = NULL;
static int g_wisdom_imported = 0;
static int g_wisdom_available = 0;

void fast_psf_fftw_set_plan_mode(int measure)
{
  g_plan_mode = measure ? FAST_PSF_FFTW_PLAN_MEASURE
                        : FAST_PSF_FFTW_PLAN_ESTIMATE;
  g_wisdom_path = NULL;
  g_wisdom_imported = 0;
  g_wisdom_available = 0;
}

int fast_psf_fftw_configure(FastPsfFftwPlanMode mode, const char *wisdom_path)
{
  if ((mode == FAST_PSF_FFTW_PLAN_WISDOM ||
       mode == FAST_PSF_FFTW_PLAN_WISDOM_UPDATE) &&
      (wisdom_path == NULL || wisdom_path[0] == '\0')) {
    fprintf(stderr,
            "Fast FFTW: --fast-fftw-plan wisdom modes require "
            "--fast-fftw-wisdom FILE.\n");
    return -1;
  }
  g_plan_mode = mode;
  g_wisdom_path = wisdom_path;
  g_wisdom_imported = 0;
  g_wisdom_available = 0;
  return 0;
}

static const char *plan_mode_name(void)
{
  switch (g_plan_mode) {
  case FAST_PSF_FFTW_PLAN_MEASURE:
    return "measure";
  case FAST_PSF_FFTW_PLAN_WISDOM:
    return "wisdom";
  case FAST_PSF_FFTW_PLAN_WISDOM_UPDATE:
    return "wisdom-update";
  case FAST_PSF_FFTW_PLAN_ESTIMATE:
  default:
    return "estimate";
  }
}

static unsigned plan_flags(void)
{
  switch (g_plan_mode) {
  case FAST_PSF_FFTW_PLAN_MEASURE:
  case FAST_PSF_FFTW_PLAN_WISDOM_UPDATE:
    return FFTW_MEASURE;
  case FAST_PSF_FFTW_PLAN_WISDOM:
    return FFTW_WISDOM_ONLY;
  case FAST_PSF_FFTW_PLAN_ESTIMATE:
  default:
    return FFTW_ESTIMATE;
  }
}

/* Wisdom identity sidecar.  FFTW's own wisdom already encodes the transform
 * size and precision, but not this build's supersampling or worker count, so a
 * validated <wisdom>.meta accompanies every exported file. */
static int wisdom_meta_path(char *out, size_t cap, const char *wisdom_path)
{
  const int written = snprintf(out, cap, "%s.meta", wisdom_path);
  return written < 0 || (size_t)written >= cap ? -1 : 0;
}

struct FastPsfFftwState {
  int fft_width, fft_height;
  int ss_width, ss_height;
  int final_width, final_height;
  int supersample;
  int kernel_radius;
  int workers;
  int registered;
  int plan_measure;
  size_t scratch_bytes;
  double setup_seconds;
  double kernel_seconds;
  double fft_scale;
  double box_scale;

  double *real_rgb;          /* 3 planes of fft_height * fft_width */
  fftw_complex *freq_rgb;    /* 3 planes of fft_height * (fft_width/2 + 1) */
  fftw_complex *kernel_freq; /* one plane of fft_height * (fft_width/2 + 1) */
  fftw_plan forward_plan;
  fftw_plan inverse_plan;
};

/* Required identity fields.  Every one must appear exactly once and match; a
 * missing or bad field is a miss, not an implicit default. */
enum {
  WISDOM_META_VERSION = 1u << 0,
  WISDOM_META_PRECISION = 1u << 1,
  WISDOM_META_SS_WIDTH = 1u << 2,
  WISDOM_META_SS_HEIGHT = 1u << 3,
  WISDOM_META_FFT_WIDTH = 1u << 4,
  WISDOM_META_FFT_HEIGHT = 1u << 5,
  WISDOM_META_SUPERSAMPLE = 1u << 6,
  WISDOM_META_RADIUS = 1u << 7,
  WISDOM_META_WORKERS = 1u << 8,
  WISDOM_META_ALL = (1u << 9) - 1
};

static int strict_long(const char *text, long *out)
{
  char *end;
  errno = 0;
  const long value = strtol(text, &end, 10);
  if (errno != 0 || end == text || *end != '\0')
    return -1;
  *out = value;
  return 0;
}

static int wisdom_meta_matches(const FastPsfFftwState *state)
{
  if (g_wisdom_path == NULL)
    return 0;
  char path[PATH_MAX];
  if (wisdom_meta_path(path, sizeof path, g_wisdom_path))
    return 0;
  FILE *file = fopen(path, "r");
  if (file == NULL)
    return 0;
  char line[256];
  unsigned seen = 0;
  int ok = 1;
  while (ok && fgets(line, sizeof line, file) != NULL) {
    char key[64], value[192];
    if (sscanf(line, "%63[^=]=%191s", key, value) != 2)
      continue;
    long expected = 0;
    unsigned bit = 0;
    int numeric = 0;
    if (!strcmp(key, "fftw_version")) {
      ok = strcmp(value, fftw_version) == 0;
      bit = WISDOM_META_VERSION;
    } else if (!strcmp(key, "precision")) {
      ok = strcmp(value, "double") == 0;
      bit = WISDOM_META_PRECISION;
    } else if (!strcmp(key, "ss_width")) {
      expected = state->ss_width; numeric = 1; bit = WISDOM_META_SS_WIDTH;
    } else if (!strcmp(key, "ss_height")) {
      expected = state->ss_height; numeric = 1; bit = WISDOM_META_SS_HEIGHT;
    } else if (!strcmp(key, "fft_width")) {
      expected = state->fft_width; numeric = 1; bit = WISDOM_META_FFT_WIDTH;
    } else if (!strcmp(key, "fft_height")) {
      expected = state->fft_height; numeric = 1; bit = WISDOM_META_FFT_HEIGHT;
    } else if (!strcmp(key, "supersample")) {
      expected = state->supersample; numeric = 1; bit = WISDOM_META_SUPERSAMPLE;
    } else if (!strcmp(key, "kernel_radius")) {
      expected = state->kernel_radius; numeric = 1; bit = WISDOM_META_RADIUS;
    } else if (!strcmp(key, "workers")) {
      expected = state->workers; numeric = 1; bit = WISDOM_META_WORKERS;
    } else {
      continue;
    }
    if (ok && numeric) {
      long parsed = 0;
      ok = strict_long(value, &parsed) == 0 && parsed == expected;
    }
    if (ok && (seen & bit) != 0)
      ok = 0; /* duplicate field: reject rather than accept twice */
    if (ok)
      seen |= bit;
  }
  fclose(file);
  return ok && (seen & WISDOM_META_ALL) == WISDOM_META_ALL;
}

static int wisdom_write_meta(const FastPsfFftwState *state, const char *path)
{
  FILE *file = fopen(path, "w");
  if (file == NULL)
    return -1;
  const int written = fprintf(
      file,
      "fftw_version=%s\nprecision=double\nss_width=%d\nss_height=%d\n"
      "fft_width=%d\nfft_height=%d\nsupersample=%d\nkernel_radius=%d\n"
      "workers=%d\n",
      fftw_version, state->ss_width, state->ss_height, state->fft_width,
      state->fft_height, state->supersample, state->kernel_radius,
      state->workers);
  const int closed = fclose(file);
  return written < 0 || closed != 0 ? -1 : 0;
}

static int wisdom_import_and_verify(const FastPsfFftwState *state)
{
  if (!g_wisdom_imported) {
    g_wisdom_imported = 1;
    if (g_wisdom_path != NULL &&
        fftw_import_wisdom_from_filename(g_wisdom_path) != 0)
      g_wisdom_available = 1;
  }
  if (!g_wisdom_available) {
    fprintf(stderr,
            "Fast FFTW: --fast-fftw-plan wisdom could not import wisdom from "
            "%s.\n",
            g_wisdom_path != NULL ? g_wisdom_path : "(unset)");
    return 0;
  }
  if (!wisdom_meta_matches(state)) {
    fprintf(stderr,
            "Fast FFTW: wisdom in %s does not match this render "
            "(size, supersample, worker count, version, or precision).\n",
            g_wisdom_path);
    return 0;
  }
  return 1;
}

/* Writes the wisdom and its identity sidecar next to the requested path.
 * Each file is written to a unique temporary and renamed into place, so each
 * rename is individually atomic.  The wisdom file and its sidecar are two
 * separate files and are not transactionally updated: the sidecar is committed
 * last, so a loader that sees a wisdom without a matching sidecar treats it as
 * a miss.  Any failure returns -1 and makes wisdom-update initialization fail. */
static int wisdom_export(const FastPsfFftwState *state)
{
  if (g_plan_mode != FAST_PSF_FFTW_PLAN_WISDOM_UPDATE || g_wisdom_path == NULL)
    return 0;
  char meta_path[PATH_MAX];
  char wisdom_temp[PATH_MAX];
  char meta_temp[PATH_MAX];
  if (wisdom_meta_path(meta_path, sizeof meta_path, g_wisdom_path))
    return -1;
  const long pid = (long)getpid();
  if (snprintf(wisdom_temp, sizeof wisdom_temp, "%s.tmp.%ld", g_wisdom_path,
               pid) < 0 ||
      snprintf(meta_temp, sizeof meta_temp, "%s.tmp.%ld", meta_path, pid) < 0)
    return -1;
  if (wisdom_write_meta(state, meta_temp) != 0) {
    fprintf(stderr, "Fast FFTW: failed to write wisdom metadata %s.\n",
            meta_temp);
    unlink(meta_temp);
    return -1;
  }
  if (fftw_export_wisdom_to_filename(wisdom_temp) == 0) {
    fputs("Fast FFTW: failed to export FFTW wisdom.\n", stderr);
    unlink(wisdom_temp);
    unlink(meta_temp);
    return -1;
  }
  if (rename(wisdom_temp, g_wisdom_path) != 0) {
    fprintf(stderr, "Fast FFTW: failed to commit wisdom to %s.\n",
            g_wisdom_path);
    unlink(wisdom_temp);
    unlink(meta_temp);
    return -1;
  }
  if (rename(meta_temp, meta_path) != 0) {
    fprintf(stderr, "Fast FFTW: failed to commit wisdom metadata to %s.\n",
            meta_path);
    unlink(meta_temp);
    return -1;
  }
  return 0;
}

static int checked_mul_size(size_t a, size_t b, size_t *out)
{
  if (a != 0 && b > SIZE_MAX / a)
    return -1;
  *out = a * b;
  return 0;
}

int fast_psf_fftw_next_smooth_size(size_t min_extent, size_t *out)
{
  if (out == NULL || min_extent == 0)
    return -1;
  if (min_extent > (size_t)INT_MAX)
    return -1;
  static const unsigned factors[] = {2, 3, 5, 7};
  for (size_t candidate = min_extent; candidate <= (size_t)INT_MAX;
       ++candidate) {
    size_t remaining = candidate;
    for (size_t i = 0; i < sizeof factors / sizeof factors[0]; ++i)
      while (remaining % factors[i] == 0)
        remaining /= factors[i];
    if (remaining == 1) {
      *out = candidate;
      return 0;
    }
  }
  return -1;
}

void fast_psf_fftw_destroy(FastPsfFftwState *state)
{
  if (state == NULL)
    return;
  if (state->forward_plan != NULL)
    fftw_destroy_plan(state->forward_plan);
  if (state->inverse_plan != NULL)
    fftw_destroy_plan(state->inverse_plan);
  if (state->real_rgb != NULL)
    fftw_free(state->real_rgb);
  if (state->freq_rgb != NULL)
    fftw_free(state->freq_rgb);
  if (state->kernel_freq != NULL)
    fftw_free(state->kernel_freq);
  const int registered = state->registered;
  free(state);

  if (!registered)
    return;
  g_live_states--;
  if (g_live_states <= 0) {
    g_live_states = 0;
    if (g_threading_initialized) {
      fftw_cleanup_threads();
      g_threading_initialized = 0;
    }
  }
}

FastPsfFftwState *fast_psf_fftw_create(int ss_width, int ss_height,
                                       int final_width, int final_height,
                                       int supersample, int kernel_radius,
                                       const float *weights,
                                       const int *row_span, int fft_workers,
                                       double *setup_seconds,
                                       double *kernel_seconds)
{
  if (setup_seconds != NULL)
    *setup_seconds = 0.0;
  if (kernel_seconds != NULL)
    *kernel_seconds = 0.0;
  if (ss_width <= 0 || ss_height <= 0 || final_width <= 0 ||
      final_height <= 0 || supersample < 1 || kernel_radius < 0 ||
      weights == NULL || row_span == NULL || fft_workers < 1) {
    fprintf(stderr,
            "Fast FFTW init failed (arguments): ss=%dx%d final=%dx%d "
            "supersample=%d kernel_radius=%d workers=%d\n",
            ss_width, ss_height, final_width, final_height, supersample,
            kernel_radius, fft_workers);
    return NULL;
  }
  if ((size_t)final_width * (size_t)supersample != (size_t)ss_width ||
      (size_t)final_height * (size_t)supersample != (size_t)ss_height) {
    fprintf(stderr,
            "Fast FFTW init failed (dimensions): ss=%dx%d is not "
            "final=%dx%d times supersample=%d\n",
            ss_width, ss_height, final_width, final_height, supersample);
    return NULL;
  }

  const char *stage = "state allocation";
  FastPsfFftwState *state = calloc(1, sizeof *state);
  if (state == NULL) {
    fprintf(stderr, "Fast FFTW init failed (state allocation): %zu bytes\n",
            sizeof *state);
    return NULL;
  }
  state->ss_width = ss_width;
  state->ss_height = ss_height;
  state->final_width = final_width;
  state->final_height = final_height;
  state->supersample = supersample;
  state->kernel_radius = kernel_radius;
  state->workers = fft_workers;
  state->plan_measure =
      g_plan_mode == FAST_PSF_FFTW_PLAN_MEASURE ||
      g_plan_mode == FAST_PSF_FFTW_PLAN_WISDOM_UPDATE;

  size_t real_bytes = 0, freq_bytes = 0, kernel_bytes = 0;
  size_t fft_width = 0, fft_height = 0;
  const size_t radius = (size_t)kernel_radius;
  size_t width_min, height_min;
  stage = "kernel extent";
  if (checked_mul_size(radius, 2, &width_min) ||
      checked_mul_size(radius, 2, &height_min))
    goto fail;
  if ((size_t)ss_width > SIZE_MAX - width_min ||
      (size_t)ss_height > SIZE_MAX - height_min)
    goto fail;
  width_min += (size_t)ss_width;
  height_min += (size_t)ss_height;
  stage = "smooth FFT size";
  if (fast_psf_fftw_next_smooth_size(width_min, &fft_width) ||
      fast_psf_fftw_next_smooth_size(height_min, &fft_height))
    goto fail;
  state->fft_width = (int)fft_width;
  state->fft_height = (int)fft_height;

  const size_t complex_width = fft_width / 2 + 1;
  size_t real_plane, real_count;
  size_t freq_plane, freq_count;
  stage = "allocation size";
  if (checked_mul_size(fft_height, fft_width, &real_plane) ||
      checked_mul_size(real_plane, 3, &real_count) ||
      checked_mul_size(real_count, sizeof(double), &real_bytes) ||
      checked_mul_size(fft_height, complex_width, &freq_plane) ||
      checked_mul_size(freq_plane, 3, &freq_count) ||
      checked_mul_size(freq_count, sizeof(fftw_complex), &freq_bytes) ||
      checked_mul_size(freq_plane, sizeof(fftw_complex), &kernel_bytes) ||
      real_bytes > SIZE_MAX - freq_bytes - kernel_bytes)
    goto fail;
  /* FFTW's plan_many interface takes idist/odist as int, so the per-plane
   * element counts must fit before the cast below even when each axis is a
   * valid int. */
  stage = "plan_many stride";
  if (real_plane > (size_t)INT_MAX || freq_plane > (size_t)INT_MAX)
    goto fail;
  state->scratch_bytes = real_bytes + freq_bytes + kernel_bytes;

  stage = "FFTW threading initialization";
  if (g_live_states == 0) {
    if (!fftw_init_threads())
      goto fail;
    g_threading_initialized = 1;
  }
  g_live_states++;
  state->registered = 1;
  fftw_plan_with_nthreads(fft_workers);

  if (g_plan_mode == FAST_PSF_FFTW_PLAN_WISDOM &&
      !wisdom_import_and_verify(state))
    goto fail;

  const double plan_start = omp_get_wtime();
  stage = "FFTW scratch allocation";
  state->real_rgb = fftw_alloc_real(real_count);
  state->freq_rgb = fftw_alloc_complex(freq_count);
  state->kernel_freq = fftw_alloc_complex(freq_plane);
  if (state->real_rgb == NULL || state->freq_rgb == NULL ||
      state->kernel_freq == NULL)
    goto fail;

  int n[2] = {state->fft_height, state->fft_width};
  int real_embed[2] = {state->fft_height, state->fft_width};
  int complex_embed[2] = {state->fft_height, (int)complex_width};
  const int real_dist = (int)real_plane;
  const int complex_dist = (int)freq_plane;
  stage = "FFTW plan creation";
  state->forward_plan = fftw_plan_many_dft_r2c(
      2, n, 3, state->real_rgb, real_embed, 1, real_dist, state->freq_rgb,
      complex_embed, 1, complex_dist, plan_flags());
  state->inverse_plan = fftw_plan_many_dft_c2r(
      2, n, 3, state->freq_rgb, complex_embed, 1, complex_dist,
      state->real_rgb, real_embed, 1, real_dist, plan_flags());
  if (state->forward_plan == NULL || state->inverse_plan == NULL)
    goto fail;
  state->setup_seconds = omp_get_wtime() - plan_start;
  if (setup_seconds != NULL)
    *setup_seconds = state->setup_seconds;

  /* One-time kernel transform.  The kernel is stored unshifted at padded origin
   * indices (dy + R, dx + R), retaining only the circular mask the spatial
   * reference traverses; square corners and FFT padding stay zero.  The plan is
   * created before the kernel is written because FFTW_MEASURE overwrites its
   * input during planning. */
  stage = "kernel scratch allocation";
  double *kernel_real = fftw_alloc_real(real_plane);
  fftw_plan kernel_plan = NULL;
  if (kernel_real == NULL)
    goto fail;
  const double kernel_start = omp_get_wtime();
  stage = "kernel plan creation";
  kernel_plan = fftw_plan_dft_r2c_2d(state->fft_height, state->fft_width,
                                     kernel_real, state->kernel_freq,
                                     plan_flags());
  if (kernel_plan == NULL) {
    fftw_free(kernel_real);
    goto fail;
  }
  memset(kernel_real, 0, real_plane * sizeof *kernel_real);
  const size_t side = 2 * radius + 1;
  for (int dy = -kernel_radius; dy <= kernel_radius; ++dy) {
    const int span = row_span[dy + kernel_radius];
    for (int dx = -span; dx <= span; ++dx)
      kernel_real[(size_t)(dy + kernel_radius) * fft_width +
                  (size_t)(dx + kernel_radius)] =
          (double)weights[(size_t)(dy + kernel_radius) * side +
                          (size_t)(dx + kernel_radius)];
  }
  fftw_execute(kernel_plan);
  state->kernel_seconds = omp_get_wtime() - kernel_start;
  if (kernel_seconds != NULL)
    *kernel_seconds = state->kernel_seconds;
  fftw_destroy_plan(kernel_plan);
  fftw_free(kernel_real);

  state->fft_scale = 1.0 / ((double)fft_width * (double)fft_height);
  state->box_scale = 1.0 / ((double)supersample * (double)supersample);
  stage = "wisdom export";
  if (wisdom_export(state) != 0)
    goto fail;
  return state;

fail:
  fprintf(stderr,
          "Fast FFTW init failed at '%s': ss=%dx%d, linear min=%zux%zu, "
          "fft=%zux%zu, plan=%s, workers=%d, bytes real=%zu freq=%zu "
          "kernel=%zu total=%zu\n",
          stage, ss_width, ss_height,
          (size_t)ss_width + 2 * radius,
          (size_t)ss_height + 2 * radius, fft_width, fft_height,
          plan_mode_name(), fft_workers,
          real_bytes, freq_bytes, kernel_bytes,
          real_bytes + freq_bytes + kernel_bytes);
  fast_psf_fftw_destroy(state);
  return NULL;
}

static int fast_psf_fftw_resolve_impl(FastPsfFftwState *state,
                                      const double *interleaved_ss_rgb,
                                      int clear_source,
                                      double *interleaved_hdr_rgb,
                                      FastPsfFftwFrameTiming *timing)
{
  if (state == NULL || interleaved_ss_rgb == NULL ||
      interleaved_hdr_rgb == NULL)
    return -1;
  FastPsfFftwFrameTiming local = {0};
  const double total_start = omp_get_wtime();

  const size_t plane = (size_t)state->fft_height * state->fft_width;
  const size_t complex_width = (size_t)state->fft_width / 2 + 1;
  const size_t complex_plane = (size_t)state->fft_height * complex_width;
  const size_t real_count = 3 * plane;
  const size_t freq_count = 3 * complex_plane;

  /* The pack pass writes every FFT cell and, when clear_source is set, both
   * consumes and zeroes its own source triplet.  Each supersampled triplet is
   * touched by exactly one worker, so zeroing needs no atomic and no second
   * serial memset.  Clearing is deliberately unconditional in the consuming
   * variant: even a later failure must not leave stale deposits behind. */
  double *source_mut = (double *)(uintptr_t)interleaved_ss_rgb;
  double start = omp_get_wtime();
  /* The padded border is never overwritten by packing, so it must be zeroed
   * every frame regardless of the consuming variant. */
#pragma omp parallel for schedule(static) num_threads(state->workers) \
    if (state->workers > 1)
  for (size_t i = 0; i < real_count; ++i)
    state->real_rgb[i] = 0.0;
#pragma omp parallel for schedule(static) num_threads(state->workers) \
    if (state->workers > 1)
  for (int y = 0; y < state->ss_height; ++y) {
    for (int x = 0; x < state->ss_width; ++x) {
      const size_t source =
          3 * ((size_t)y * state->ss_width + (size_t)x);
      const size_t target = (size_t)y * state->fft_width + (size_t)x;
      state->real_rgb[target] = source_mut[source];
      state->real_rgb[plane + target] = source_mut[source + 1];
      state->real_rgb[2 * plane + target] = source_mut[source + 2];
      if (clear_source) {
        source_mut[source] = 0.0;
        source_mut[source + 1] = 0.0;
        source_mut[source + 2] = 0.0;
      }
    }
  }
  local.zero_pack_seconds = omp_get_wtime() - start;

  start = omp_get_wtime();
  fftw_execute(state->forward_plan);
  local.forward_seconds = omp_get_wtime() - start;

  start = omp_get_wtime();
#pragma omp parallel for schedule(static) num_threads(state->workers) \
    if (state->workers > 1)
  for (size_t i = 0; i < freq_count; ++i) {
    const double ar = state->freq_rgb[i][0];
    const double ai = state->freq_rgb[i][1];
    const double br = state->kernel_freq[i % complex_plane][0];
    const double bi = state->kernel_freq[i % complex_plane][1];
    state->freq_rgb[i][0] = ar * br - ai * bi;
    state->freq_rgb[i][1] = ar * bi + ai * br;
  }
  local.multiply_seconds = omp_get_wtime() - start;

  start = omp_get_wtime();
  fftw_execute(state->inverse_plan);
  local.inverse_seconds = omp_get_wtime() - start;

  start = omp_get_wtime();
  const int radius = state->kernel_radius;
  const int supersample = state->supersample;
  const double scale = state->fft_scale * state->box_scale;
#pragma omp parallel for schedule(static) num_threads(state->workers) \
    if (state->workers > 1)
  for (int row = 0; row < state->final_height; ++row) {
    for (int column = 0; column < state->final_width; ++column) {
      double sum[3] = {0.0, 0.0, 0.0};
      const int base_y = radius + supersample * row;
      const int base_x = radius + supersample * column;
      for (int j = 0; j < supersample; ++j)
        for (int i = 0; i < supersample; ++i) {
          const size_t offset =
              (size_t)(base_y + j) * state->fft_width +
              (size_t)(base_x + i);
          sum[0] += state->real_rgb[offset];
          sum[1] += state->real_rgb[plane + offset];
          sum[2] += state->real_rgb[2 * plane + offset];
        }
      const size_t out =
          3 * ((size_t)row * state->final_width + (size_t)column);
      interleaved_hdr_rgb[out] += sum[0] * scale;
      interleaved_hdr_rgb[out + 1] += sum[1] * scale;
      interleaved_hdr_rgb[out + 2] += sum[2] * scale;
    }
  }
  local.crop_downsample_seconds = omp_get_wtime() - start;
  local.total_seconds = omp_get_wtime() - total_start;
  if (timing != NULL)
    *timing = local;
  return 0;
}

int fast_psf_fftw_resolve(FastPsfFftwState *state,
                          const double *interleaved_ss_rgb,
                          double *interleaved_hdr_rgb,
                          FastPsfFftwFrameTiming *timing)
{
  return fast_psf_fftw_resolve_impl(state, interleaved_ss_rgb, 0,
                                    interleaved_hdr_rgb, timing);
}

int fast_psf_fftw_resolve_and_clear(FastPsfFftwState *state,
                                    double *interleaved_ss_rgb,
                                    double *interleaved_hdr_rgb,
                                    FastPsfFftwFrameTiming *timing)
{
  if (state == NULL || interleaved_ss_rgb == NULL)
    return -1;
  const int result = fast_psf_fftw_resolve_impl(state, interleaved_ss_rgb, 1,
                                                interleaved_hdr_rgb, timing);
  if (result != 0) {
    /* The pack pass may not have run, so guarantee a clean source even on the
     * error path rather than letting a later frame inherit stale deposits. */
    const size_t count = (size_t)state->ss_width * state->ss_height * 3;
    memset(interleaved_ss_rgb, 0, count * sizeof *interleaved_ss_rgb);
  }
  return result;
}

void fast_psf_fftw_report(const FastPsfFftwState *state, FILE *stream)
{
  if (state == NULL || stream == NULL)
    return;
  fprintf(stream,
          "Fast FFTW: linear min=%dx%d, fft=%dx%d, workers=%d, "
          "plan=%s, plan=%.6f s, kernel_fft=%.6f s, scratch=%.1f MiB\n",
          state->ss_height + 2 * state->kernel_radius,
          state->ss_width + 2 * state->kernel_radius, state->fft_height,
          state->fft_width, state->workers,
          plan_mode_name(), state->setup_seconds,
          state->kernel_seconds,
          (double)state->scratch_bytes / (1024.0 * 1024.0));
}