path: root/src/freedreno/vulkan/tu_util.cc

/*
 * Copyright © 2015 Intel Corporation
 * SPDX-License-Identifier: MIT
 */

#include "tu_util.h"

#include <errno.h>
#include <stdarg.h>

#include "common/freedreno_rd_output.h"
#include "util/u_math.h"
#include "util/timespec.h"
#include "util/os_file_notify.h"
#include "util/os_file.h"
#include "vk_enum_to_str.h"

#include "tu_device.h"
#include "tu_pass.h"

static const struct debug_control tu_debug_options[] = {
   { "startup", TU_DEBUG_STARTUP },
   { "nir", TU_DEBUG_NIR },
   { "nobin", TU_DEBUG_NOBIN },
   { "sysmem", TU_DEBUG_SYSMEM },
   { "gmem", TU_DEBUG_GMEM },
   { "forcebin", TU_DEBUG_FORCEBIN },
   { "layout", TU_DEBUG_LAYOUT },
   { "noubwc", TU_DEBUG_NOUBWC },
   { "nomultipos", TU_DEBUG_NOMULTIPOS },
   { "nolrz", TU_DEBUG_NOLRZ },
   { "nolrzfc", TU_DEBUG_NOLRZFC },
   { "perf", TU_DEBUG_PERF },
   { "perfc", TU_DEBUG_PERFC },
   { "flushall", TU_DEBUG_FLUSHALL },
   { "syncdraw", TU_DEBUG_SYNCDRAW },
   { "push_consts_per_stage", TU_DEBUG_PUSH_CONSTS_PER_STAGE },
   { "rast_order", TU_DEBUG_RAST_ORDER },
   { "unaligned_store", TU_DEBUG_UNALIGNED_STORE },
   { "log_skip_gmem_ops", TU_DEBUG_LOG_SKIP_GMEM_OPS },
   { "dynamic", TU_DEBUG_DYNAMIC },
   { "bos", TU_DEBUG_BOS },
   { "3d_load", TU_DEBUG_3D_LOAD },
   { "fdm", TU_DEBUG_FDM },
   { "noconform", TU_DEBUG_NOCONFORM },
   { "rd", TU_DEBUG_RD },
   { "hiprio", TU_DEBUG_HIPRIO },
   { "noconcurrentresolves", TU_DEBUG_NO_CONCURRENT_RESOLVES },
   { "noconcurrentunresolves", TU_DEBUG_NO_CONCURRENT_UNRESOLVES },
   { "dumpas", TU_DEBUG_DUMPAS },
   { "nobinmerging", TU_DEBUG_NO_BIN_MERGING },
   { "perfcraw", TU_DEBUG_PERFCRAW },
   { "fdmoffset", TU_DEBUG_FDM_OFFSET },
   { "check_cmd_buffer_status", TU_DEBUG_CHECK_CMD_BUFFER_STATUS },
   { "comm", TU_DEBUG_COMM },
   { "nofdm", TU_DEBUG_NOFDM },
   { "nocb", TU_DEBUG_NO_CONCURRENT_BINNING },
   { "forcecb", TU_DEBUG_FORCE_CONCURRENT_BINNING },
   { NULL, 0 }
};

/*
 * The runtime debug flags are a subset of the debug flags that can be set at
 * runtime. Flags which depend on running state of the driver, the application
 * or the hardware and would otherwise break when toggled should not be set here.
 * Note: Keep in sync with the list of flags in 'docs/drivers/freedreno.rst'.
 */
const uint64_t tu_runtime_debug_flags =
   TU_DEBUG_NIR | TU_DEBUG_NOBIN | TU_DEBUG_SYSMEM | TU_DEBUG_GMEM |
   TU_DEBUG_FORCEBIN | TU_DEBUG_LAYOUT | TU_DEBUG_NOLRZ | TU_DEBUG_NOLRZFC |
   TU_DEBUG_PERF | TU_DEBUG_FLUSHALL | TU_DEBUG_SYNCDRAW |
   TU_DEBUG_RAST_ORDER | TU_DEBUG_UNALIGNED_STORE |
   TU_DEBUG_LOG_SKIP_GMEM_OPS | TU_DEBUG_3D_LOAD | TU_DEBUG_FDM |
   TU_DEBUG_NO_CONCURRENT_RESOLVES | TU_DEBUG_NO_CONCURRENT_UNRESOLVES |
   TU_DEBUG_NO_BIN_MERGING;

os_file_notifier_t tu_debug_notifier;
struct tu_env tu_env;

static uint64_t
tu_env_get_file_flags(const char *path)
{
   char *str = os_read_file(path, NULL);
   if (str) {
      uint64_t flags = parse_debug_string(str, tu_debug_options);
      free(str);
      return flags;
   }
   return 0;
}

static void
tu_env_notify(
   void *data, const char *path, bool created, bool deleted, bool dir_deleted)
{
   uint64_t file_flags = 0;
   if (!deleted) {
      file_flags = tu_env_get_file_flags(path);
   }

   uint64_t runtime_flags = file_flags & tu_runtime_debug_flags;
   if ((tu_env.debug.load(std::memory_order_acquire) & tu_runtime_debug_flags) ^ runtime_flags) {
      mesa_logd("TU_DEBUG_FILE: Runtime debug flags change detected. Flags set:");
      for (unsigned i = 0; i < ARRAY_SIZE(tu_debug_options); i++) {
         if (runtime_flags & tu_debug_options[i].flag)
            mesa_logd("TU_DEBUG_FILE:   %s", tu_debug_options[i].string);
      }

      if (runtime_flags == 0)
         mesa_logd("TU_DEBUG_FILE:   None");
   }

   tu_env.debug.store(runtime_flags | tu_env.start_debug, std::memory_order_release);

   if (unlikely(dir_deleted))
      mesa_logw(
         "Directory containing TU_DEBUG_FILE (%s) was deleted, stopping watching",
         path);
}

static void
tu_env_deinit(void)
{
   if (tu_debug_notifier)
      os_file_notifier_destroy(tu_debug_notifier);
}

static void
tu_env_init_once(void)
{
   tu_env.start_debug = tu_env.debug = parse_debug_string(os_get_option("TU_DEBUG"), tu_debug_options);

   if (TU_DEBUG(STARTUP))
      mesa_logi("TU_DEBUG=0x%" PRIx64, tu_env.debug.load());

   /* TU_DEBUG=rd functionality was moved to fd_rd_output. This debug option
    * should translate to the basic-level FD_RD_DUMP_ENABLE option.
    */
   if (TU_DEBUG(RD))
      fd_rd_dump_env.flags |= FD_RD_DUMP_ENABLE;

   const char *debug_file = os_get_option("TU_DEBUG_FILE");
   if (debug_file) {
      if ((tu_env.debug & tu_runtime_debug_flags) != 0) {
         mesa_logw("TU_DEBUG_FILE is set (%s), but TU_DEBUG is also set. "
                   "Any runtime options (0x%" PRIx64 ") set in TU_DEBUG cannot be changed at runtime.",
                   debug_file, tu_env.debug & tu_runtime_debug_flags);
      }

      uint64_t file_flags = tu_env_get_file_flags(debug_file);
      tu_env.start_debug |= file_flags & ~tu_runtime_debug_flags;
      tu_env.debug = file_flags | tu_env.start_debug;

      if (TU_DEBUG(STARTUP))
         mesa_logi("Watching TU_DEBUG_FILE: %s", debug_file);

      const char* error_str = "Unknown error";
      tu_debug_notifier =
         os_file_notifier_create(debug_file, tu_env_notify, NULL, &error_str);
      if (!tu_debug_notifier)
         mesa_logw("Failed to watch TU_DEBUG_FILE (%s): %s", debug_file, error_str);
   } else {
      tu_debug_notifier = NULL;
   }

   atexit(tu_env_deinit);
}

void
tu_env_init(void)
{
   fd_rd_dump_env_init();

   static once_flag once = ONCE_FLAG_INIT;
   call_once(&once, tu_env_init_once);
}

const char *
tu_env_debug_as_string(void)
{
   static thread_local char debug_string[96];
   dump_debug_control_string(debug_string, sizeof(debug_string),
                             tu_debug_options,
                             tu_env.debug.load(std::memory_order_acquire));
   return debug_string;
}

void PRINTFLIKE(3, 4)
   __tu_finishme(const char *file, int line, const char *format, ...)
{
   va_list ap;
   char buffer[256];

   va_start(ap, format);
   vsnprintf(buffer, sizeof(buffer), format, ap);
   va_end(ap);

   mesa_loge("%s:%d: FINISHME: %s\n", file, line, buffer);
}

VkResult
__vk_startup_errorf(struct tu_instance *instance,
                    VkResult error,
                    const char *file,
                    int line,
                    const char *format,
                    ...)
{
   va_list ap;
   char buffer[256];

   const char *error_str = vk_Result_to_str(error);

   if (format) {
      va_start(ap, format);
      vsnprintf(buffer, sizeof(buffer), format, ap);
      va_end(ap);

      mesa_loge("%s:%d: %s (%s)\n", file, line, buffer, error_str);
   } else {
      mesa_loge("%s:%d: %s\n", file, line, error_str);
   }

   return error;
}

static void
tu_tiling_config_update_tile_layout(struct tu_framebuffer *fb,
                                    const struct tu_device *dev,
                                    const struct tu_render_pass *pass,
                                    enum tu_gmem_layout gmem_layout)
{
   const uint32_t tile_align_w = pass->tile_align_w;
   uint32_t tile_align_h = dev->physical_device->info->tile_align_h;
   struct tu_tiling_config *tiling = &fb->tiling[gmem_layout];

   *tiling = (struct tu_tiling_config) {
      /* Put in dummy values that will assertion fail in register setup using
       * them, since you shouldn't be doing gmem work if gmem is not possible.
       */
      .tile0 = (VkExtent2D) { ~0, ~0 },
      .possible = false,
      .vsc = {
         .tile_count = (VkExtent2D) { .width = 1, .height = 1 },
      },
   };

   /* From the Vulkan 1.3.232 spec, under VkFramebufferCreateInfo:
    *
    *   If the render pass uses multiview, then layers must be one and each
    *   attachment requires a number of layers that is greater than the
    *   maximum bit index set in the view mask in the subpasses in which it is
    *   used.
    */

   uint32_t layers = MAX2(fb->layers, pass->num_views);

   /* If there is more than one layer, we need to make sure that the layer
    * stride is expressible as an offset in RB_RESOLVE_GMEM_BUFFER_BASE which ignores
    * the low 12 bits. The layer stride seems to be implicitly calculated from
    * the tile width and height so we need to adjust one of them.
    */
   const uint32_t gmem_align_log2 = 12;
   const uint32_t gmem_align = 1 << gmem_align_log2;
   uint32_t min_layer_stride = tile_align_h * tile_align_w * pass->min_cpp;
   if (layers > 1 && align(min_layer_stride, gmem_align) != min_layer_stride) {
      /* Make sure that min_layer_stride is a multiple of gmem_align. Because
       * gmem_align is a power of two and min_layer_stride isn't already a
       * multiple of gmem_align, this is equivalent to shifting tile_align_h
       * until the number of 0 bits at the bottom of min_layer_stride is at
       * least gmem_align_log2.
       */
      tile_align_h <<= gmem_align_log2 - (ffs(min_layer_stride) - 1);

      /* Check that we did the math right. */
      min_layer_stride = tile_align_h * tile_align_w * pass->min_cpp;
      assert(util_is_aligned(min_layer_stride, gmem_align));
   }

   /* will force to sysmem, don't bother trying to have a valid tile config
    * TODO: just skip all GMEM stuff when sysmem is forced?
    */
   if (!pass->gmem_pixels[gmem_layout])
      return;

   uint32_t best_tile_count = ~0;
   VkExtent2D tile_count;
   VkExtent2D tile_size;
   /* There aren't that many different tile widths possible, so just walk all
    * of them finding which produces the lowest number of bins.
    */
   const uint32_t max_tile_width = MIN2(
      dev->physical_device->info->tile_max_w, util_align_npot(fb->width, tile_align_w));
   const uint32_t max_tile_height =
      MIN2(dev->physical_device->info->tile_max_h,
           align(fb->height, tile_align_h));
   for (tile_size.width = tile_align_w; tile_size.width <= max_tile_width;
        tile_size.width += tile_align_w) {
      tile_size.height = pass->gmem_pixels[gmem_layout] / (tile_size.width * layers);
      tile_size.height = MIN2(tile_size.height, max_tile_height);
      tile_size.height = ROUND_DOWN_TO(tile_size.height, tile_align_h);
      if (!tile_size.height)
         continue;

      /* When using FDM, we need approximately square tiles to maintain
       * proper density distribution across the framebuffer.
       * Way to wide or tall tiles would distort the density mapping, causing
       * areas intended for low density to receive higher density and vice
       * versa.
       */
      uint32_t fdm_penalty = 0;
      if (pass->has_fdm &&
          (tile_size.width > tile_size.height * 2 ||
           tile_size.height > tile_size.width * 2)) {
         fdm_penalty = 1000;
      }

      tile_count.width = DIV_ROUND_UP(fb->width, tile_size.width);
      tile_count.height = DIV_ROUND_UP(fb->height, tile_size.height);

      /* Drop the height of the tile down to split tiles more evenly across the
       * screen for a given tile count.
       */
      tile_size.height =
         align(DIV_ROUND_UP(fb->height, tile_count.height), tile_align_h);

      /* Pick the layout with the minimum number of bins (lowest CP overhead
       * and amount of cache flushing), but the most square tiles in the case
       * of a tie (likely highest cache locality).
       */
      uint32_t total_tiles = tile_count.width * tile_count.height + fdm_penalty;
      if (total_tiles < best_tile_count ||
          (total_tiles == best_tile_count &&
           abs((int)(tile_size.width - tile_size.height)) <
              abs((int)(tiling->tile0.width - tiling->tile0.height)))) {
         tiling->possible = true;
         tiling->tile0 = tile_size;
         tiling->vsc.tile_count = tile_count;
         best_tile_count = total_tiles;
      }
   }

   /* If forcing binning, try to get at least 2 tiles in each direction. */
   if (TU_DEBUG(FORCEBIN) && tiling->possible) {
      if (tiling->vsc.tile_count.width == 1 && tiling->tile0.width != tile_align_w) {
         tiling->tile0.width = util_align_npot(DIV_ROUND_UP(tiling->tile0.width, 2), tile_align_w);
         tiling->vsc.tile_count.width = 2;
      }
      if (tiling->vsc.tile_count.height == 1 && tiling->tile0.height != tile_align_h) {
         tiling->tile0.height = align(DIV_ROUND_UP(tiling->tile0.height, 2), tile_align_h);
         tiling->vsc.tile_count.height = 2;
      }
   }
}

static bool
is_hw_binning_possible(const struct tu_vsc_config *vsc)
{
   /* Similar to older gens, # of tiles per pipe cannot be more than 32.
    * But there are no hangs with 16 or more tiles per pipe in either
    * X or Y direction, so that limit does not seem to apply.
    */
   uint32_t tiles_per_pipe = vsc->pipe0.width * vsc->pipe0.height;
   return tiles_per_pipe <= 32;
}

static void
tu_tiling_config_update_pipe_layout(struct tu_vsc_config *vsc,
                                    const struct tu_device *dev,
                                    bool fdm)
{
   const uint32_t max_pipe_count =
      dev->physical_device->info->num_vsc_pipes;

   /* If there is a fragment density map and bin merging is enabled, we will
    * likely be able to merge some bins. Bins can only be merged if they are
    * in the same visibility stream, so making the pipes cover too small an
    * area can prevent bin merging from happening. Maximize the size of each
    * pipe instead of minimizing it.
    */
   if (fdm && dev->physical_device->info->a6xx.has_bin_mask &&
       !TU_DEBUG(NO_BIN_MERGING)) {
      vsc->pipe0.width = 4;
      vsc->pipe0.height = 8;
      vsc->pipe_count.width =
         DIV_ROUND_UP(vsc->tile_count.width, vsc->pipe0.width);
      vsc->pipe_count.height =
         DIV_ROUND_UP(vsc->tile_count.height, vsc->pipe0.height);
      vsc->binning_possible =
         vsc->pipe_count.width * vsc->pipe_count.height <= max_pipe_count;
      return;
   }

   /* start from 1 tile per pipe */
   vsc->pipe0 = (VkExtent2D) {
      .width = 1,
      .height = 1,
   };
   vsc->pipe_count = vsc->tile_count;

   while (vsc->pipe_count.width * vsc->pipe_count.height > max_pipe_count) {
      if (vsc->pipe0.width < vsc->pipe0.height) {
         vsc->pipe0.width += 1;
         vsc->pipe_count.width =
            DIV_ROUND_UP(vsc->tile_count.width, vsc->pipe0.width);
      } else {
         vsc->pipe0.height += 1;
         vsc->pipe_count.height =
            DIV_ROUND_UP(vsc->tile_count.height, vsc->pipe0.height);
      }
   }

   vsc->binning_possible = is_hw_binning_possible(vsc);
}

static void
tu_tiling_config_update_pipes(struct tu_vsc_config *vsc,
                              const struct tu_device *dev)
{
   const uint32_t max_pipe_count =
      dev->physical_device->info->num_vsc_pipes;
   const uint32_t used_pipe_count =
      vsc->pipe_count.width * vsc->pipe_count.height;
   const VkExtent2D last_pipe = {
      .width = (vsc->tile_count.width - 1) % vsc->pipe0.width + 1,
      .height = (vsc->tile_count.height - 1) % vsc->pipe0.height + 1,
   };

   if (!vsc->binning_possible)
      return;

   assert(used_pipe_count <= max_pipe_count);
   assert(max_pipe_count <= ARRAY_SIZE(vsc->pipe_config));

   for (uint32_t y = 0; y < vsc->pipe_count.height; y++) {
      for (uint32_t x = 0; x < vsc->pipe_count.width; x++) {
         const uint32_t pipe_x = vsc->pipe0.width * x;
         const uint32_t pipe_y = vsc->pipe0.height * y;
         const uint32_t pipe_w = (x == vsc->pipe_count.width - 1)
                                    ? last_pipe.width
                                    : vsc->pipe0.width;
         const uint32_t pipe_h = (y == vsc->pipe_count.height - 1)
                                    ? last_pipe.height
                                    : vsc->pipe0.height;
         const uint32_t n = vsc->pipe_count.width * y + x;

         vsc->pipe_config[n] = A6XX_VSC_PIPE_CONFIG_REG_X(pipe_x) |
                                  A6XX_VSC_PIPE_CONFIG_REG_Y(pipe_y) |
                                  A6XX_VSC_PIPE_CONFIG_REG_W(pipe_w) |
                                  A6XX_VSC_PIPE_CONFIG_REG_H(pipe_h);
         vsc->pipe_sizes[n] = CP_SET_BIN_DATA5_0_VSC_SIZE(pipe_w * pipe_h);
      }
   }

   memset(vsc->pipe_config + used_pipe_count, 0,
          sizeof(uint32_t) * (max_pipe_count - used_pipe_count));
}

static void
tu_tiling_config_update_binning(struct tu_vsc_config *vsc, const struct tu_device *device)
{
   if (vsc->binning_possible) {
      vsc->binning = (vsc->tile_count.width * vsc->tile_count.height) > 2;

      if (TU_DEBUG(FORCEBIN))
         vsc->binning = true;
      if (TU_DEBUG(NOBIN))
         vsc->binning = false;
   } else {
      vsc->binning = false;
   }
}

void
tu_framebuffer_tiling_config(struct tu_framebuffer *fb,
                             const struct tu_device *device,
                             const struct tu_render_pass *pass)
{
   for (int gmem_layout = 0; gmem_layout < TU_GMEM_LAYOUT_COUNT; gmem_layout++) {
      struct tu_tiling_config *tiling = &fb->tiling[gmem_layout];
      tu_tiling_config_update_tile_layout(fb, device, pass,
                                          (enum tu_gmem_layout) gmem_layout);
      if (!tiling->possible)
         continue;

      struct tu_vsc_config *vsc = &tiling->vsc;
      tu_tiling_config_update_pipe_layout(vsc, device, pass->has_fdm);
      tu_tiling_config_update_pipes(vsc, device);
      tu_tiling_config_update_binning(vsc, device);

      if (pass->has_fdm) {
         struct tu_vsc_config *fdm_offset_vsc = &tiling->fdm_offset_vsc;
         fdm_offset_vsc->tile_count = (VkExtent2D) {
            vsc->tile_count.width + 1, vsc->tile_count.height + 1
         };
         tu_tiling_config_update_pipe_layout(fdm_offset_vsc, device, true);
         tu_tiling_config_update_pipes(fdm_offset_vsc, device);
         tu_tiling_config_update_binning(fdm_offset_vsc, device);
      }
   }
}

void
tu_dbg_log_gmem_load_store_skips(struct tu_device *device)
{
   static uint32_t last_skipped_loads = 0;
   static uint32_t last_skipped_stores = 0;
   static uint32_t last_total_loads = 0;
   static uint32_t last_total_stores = 0;
   static struct timespec last_time = {};

   pthread_mutex_lock(&device->submit_mutex);

   struct timespec current_time;
   clock_gettime(CLOCK_MONOTONIC, &current_time);

   if (timespec_sub_to_nsec(&current_time, &last_time) > 1000 * 1000 * 1000) {
      last_time = current_time;
   } else {
      pthread_mutex_unlock(&device->submit_mutex);
      return;
   }

   struct tu6_global *global = device->global_bo_map;

   uint32_t current_taken_loads = global->dbg_gmem_taken_loads;
   uint32_t current_taken_stores = global->dbg_gmem_taken_stores;
   uint32_t current_total_loads = global->dbg_gmem_total_loads;
   uint32_t current_total_stores = global->dbg_gmem_total_stores;

   uint32_t skipped_loads = current_total_loads - current_taken_loads;
   uint32_t skipped_stores = current_total_stores - current_taken_stores;

   uint32_t current_time_frame_skipped_loads = skipped_loads - last_skipped_loads;
   uint32_t current_time_frame_skipped_stores = skipped_stores - last_skipped_stores;

   uint32_t current_time_frame_total_loads = current_total_loads - last_total_loads;
   uint32_t current_time_frame_total_stores = current_total_stores - last_total_stores;

   mesa_logi("[GMEM] loads total: %u skipped: %.1f%%\n",
         current_time_frame_total_loads,
         current_time_frame_skipped_loads / (float) current_time_frame_total_loads * 100.f);
   mesa_logi("[GMEM] stores total: %u skipped: %.1f%%\n",
         current_time_frame_total_stores,
         current_time_frame_skipped_stores / (float) current_time_frame_total_stores * 100.f);

   last_skipped_loads = skipped_loads;
   last_skipped_stores = skipped_stores;
   last_total_loads = current_total_loads;
   last_total_stores = current_total_stores;

   pthread_mutex_unlock(&device->submit_mutex);
}