/* * Copyright © 2020 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ #include "util/u_vector.h" #include "nir.h" #include "nir_worklist.h" static bool combine_all_barriers(nir_intrinsic_instr *a, nir_intrinsic_instr *b, void *_) { nir_intrinsic_set_memory_modes( a, nir_intrinsic_memory_modes(a) | nir_intrinsic_memory_modes(b)); nir_intrinsic_set_memory_semantics( a, nir_intrinsic_memory_semantics(a) | nir_intrinsic_memory_semantics(b)); nir_intrinsic_set_memory_scope( a, MAX2(nir_intrinsic_memory_scope(a), nir_intrinsic_memory_scope(b))); nir_intrinsic_set_execution_scope( a, MAX2(nir_intrinsic_execution_scope(a), nir_intrinsic_execution_scope(b))); return true; } static bool nir_opt_combine_barriers_impl(nir_function_impl *impl, nir_combine_barrier_cb combine_cb, void *data) { bool progress = false; nir_foreach_block(block, impl) { nir_intrinsic_instr *prev = NULL; nir_foreach_instr_safe(instr, block) { if (instr->type != nir_instr_type_intrinsic) { prev = NULL; continue; } nir_intrinsic_instr *current = nir_instr_as_intrinsic(instr); if (current->intrinsic != nir_intrinsic_barrier) { prev = NULL; continue; } if (prev && combine_cb(prev, current, data)) { nir_instr_remove(¤t->instr); progress = true; } else { prev = current; } } } return nir_progress(progress, impl, nir_metadata_control_flow | nir_metadata_live_defs); } /* Combine adjacent scoped barriers. */ bool nir_opt_combine_barriers(nir_shader *shader, nir_combine_barrier_cb combine_cb, void *data) { /* Default to combining everything. Only some backends can do better. */ if (!combine_cb) combine_cb = combine_all_barriers; bool progress = false; nir_foreach_function_impl(impl, shader) { if (nir_opt_combine_barriers_impl(impl, combine_cb, data)) { progress = true; } } return progress; } /** If \p instr is a nir_intrinsic_barrier, returns it, else NULL. */ static nir_intrinsic_instr * instr_as_barrier(nir_instr *instr) { if (instr && instr->type == nir_instr_type_intrinsic) { nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); return intrin->intrinsic == nir_intrinsic_barrier ? intrin : NULL; } return NULL; } /** * Return true if \p atomic is surrounded by a pattern: * * 1. Release barrier * 2. Atomic operation * 3. Acquire barrier * * where all three have the same mode, both barriers have the same scope, * and that scope is \p max_scope or narrower. * * For simplicity, we require the barriers to have exactly the one mode * used by the atomic, so that we don't have to compare many barriers for * other side effects they may have. nir_opt_barrier_modes() can be used * to help reduce unnecessary barrier modes. */ static bool is_acquire_release_atomic(nir_intrinsic_instr *atomic, mesa_scope max_scope) { assert(atomic->intrinsic == nir_intrinsic_deref_atomic || atomic->intrinsic == nir_intrinsic_deref_atomic_swap); nir_deref_instr *atomic_deref = nir_src_as_deref(atomic->src[0]); nir_intrinsic_instr *prev = instr_as_barrier(nir_instr_prev(&atomic->instr)); nir_intrinsic_instr *next = instr_as_barrier(nir_instr_next(&atomic->instr)); if (!prev || !next) return false; return nir_intrinsic_memory_semantics(prev) == NIR_MEMORY_RELEASE && nir_intrinsic_memory_semantics(next) == NIR_MEMORY_ACQUIRE && nir_intrinsic_memory_modes(prev) == atomic_deref->modes && nir_intrinsic_memory_modes(next) == atomic_deref->modes && nir_intrinsic_memory_scope(prev) <= max_scope && nir_intrinsic_memory_scope(prev) == nir_intrinsic_memory_scope(next); } /** * Remove redundant barriers between sequences of atomics. * * Some shaders contain back-to-back atomic accesses in SPIR-V with * AcquireRelease semantics. In NIR, we translate these to a release * memory barrier, the atomic, then an acquire memory barrier. * * This results in a lot of unnecessary memory barriers in the * middle of the sequence of atomics: * * 1a. Release memory barrier * 1b. Atomic * 1c. Acquire memory barrier * ... * 2a. Release memory barrier * 2b. Atomic * 2c. Acquire memory barrier * ... * * We pattern match for instruction triplets, * and when we find back-to-back occurrences of that pattern, we eliminate * the barriers in-between the atomics (1c and 2a above): * * 1. Release memory barrier * 2. Atomic * ... * m. Atomic * n. Acquire memory barrier * * Some requirements: * - The atomics' destinations must be unused (so their only effect is * to update the associated memory store) * - Matched barriers must impact the atomic's memory mode. * - All barriers must have have identical scope no wider than \p max_scope * (beyond that, removing synchronization could be observable). * * And for simplicity: * - Barrier modes must be exactly the mode of the atomics (otherwise we'd * have to take care to preserve side-effects for other modes). * - Barriers must appear directly before/after the instruction (easier * pattern matching, and it's what we generate for the SPIR-V construct) * * Other instructions are allowed to be present between the atomics, so * long as they don't affect the relevant memory mode. Loads/stores or * atomics not matching this pattern are not allowed (we stop matching). * For example, this allows calculating the value to be used as the next * atomic's operand to appear in-between the two. */ static bool nir_opt_acquire_release_barriers_impl(nir_function_impl *impl, mesa_scope max_scope) { bool progress = false; nir_intrinsic_instr *last_atomic = NULL; nir_foreach_block(block, impl) { last_atomic = NULL; nir_foreach_instr_safe(instr, block) { if (instr->type != nir_instr_type_intrinsic) continue; nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); switch (intrin->intrinsic) { case nir_intrinsic_load_deref: case nir_intrinsic_load_deref_block_intel: case nir_intrinsic_store_deref: case nir_intrinsic_store_deref_block_intel: if (last_atomic) { /* If there is a load/store of the same mode as our matched * atomic, then abandon our pattern match. */ nir_deref_instr *ref = nir_src_as_deref(intrin->src[0]); nir_deref_instr *lastdr = nir_src_as_deref(last_atomic->src[0]); if (nir_deref_mode_may_be(ref, lastdr->modes)) last_atomic = NULL; } break; case nir_intrinsic_deref_atomic: case nir_intrinsic_deref_atomic_swap: if (nir_def_is_unused(&intrin->def) && is_acquire_release_atomic(intrin, max_scope)) { if (!last_atomic) { last_atomic = intrin; } else { nir_intrinsic_instr *last_acquire = nir_instr_as_intrinsic(nir_instr_next(&last_atomic->instr)); nir_intrinsic_instr *this_release = nir_instr_as_intrinsic(nir_instr_prev(&intrin->instr)); assert(last_acquire->intrinsic == nir_intrinsic_barrier); assert(this_release->intrinsic == nir_intrinsic_barrier); /* Verify that this atomic's barrier modes/scopes match * the last atomic's modes/scope. (Note that we already * verified that each atomic's pair of barriers match * each other, so we can compare against either here.) */ if (nir_intrinsic_memory_modes(last_acquire) == nir_intrinsic_memory_modes(this_release) && nir_intrinsic_memory_scope(last_acquire) == nir_intrinsic_memory_scope(this_release)) { progress = true; nir_instr_remove(&last_acquire->instr); nir_instr_remove(&this_release->instr); } /* Regardless of progress, continue matching from here */ last_atomic = intrin; } } else { /* Abandon our pattern match, this is another kind of access */ last_atomic = NULL; } break; default: /* Ignore instructions that don't affect this kind of memory */ break; } } } nir_progress(progress, impl, nir_metadata_control_flow | nir_metadata_live_defs); return progress; } bool nir_opt_acquire_release_barriers(nir_shader *shader, mesa_scope max_scope) { bool progress = false; nir_foreach_function_impl(impl, shader) { progress |= nir_opt_acquire_release_barriers_impl(impl, max_scope); } return progress; } static bool barrier_happens_before(const nir_instr *a, const nir_instr *b) { if (a->block == b->block) return a->index < b->index; return nir_block_dominates(a->block, b->block); } static bool nir_opt_barrier_modes_impl(nir_function_impl *impl) { bool progress = false; nir_instr_worklist barriers; if (!nir_instr_worklist_init(&barriers)) return false; struct u_vector mem_derefs; if (!u_vector_init(&mem_derefs, 32, sizeof(struct nir_instr *))) { nir_instr_worklist_fini(&barriers); return false; } const unsigned all_memory_modes = nir_var_image | nir_var_mem_ssbo | nir_var_mem_shared | nir_var_mem_global; nir_foreach_block_safe(block, impl) { nir_foreach_instr_safe(instr, block) { if (instr->type == nir_instr_type_intrinsic) { nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr); if (intrin->intrinsic == nir_intrinsic_barrier) nir_instr_worklist_push_tail(&barriers, instr); } else if (instr->type == nir_instr_type_deref) { nir_deref_instr *deref = nir_instr_as_deref(instr); if (nir_deref_mode_may_be(deref, all_memory_modes) || glsl_contains_atomic(deref->type)) { nir_deref_instr **tail = u_vector_add(&mem_derefs); *tail = deref; } } } } nir_foreach_instr_in_worklist(instr, &barriers) { nir_intrinsic_instr *barrier = nir_instr_as_intrinsic(instr); const unsigned barrier_modes = nir_intrinsic_memory_modes(barrier); unsigned new_modes = barrier_modes & ~all_memory_modes; /* If a barrier dominates all memory accesses for a particular mode (or * there are none), then the barrier cannot affect those accesses. We * can drop that mode from the barrier. * * For each barrier, we look at the list of memory derefs, and see if * the barrier fails to dominate the deref. If so, then there's at * least one memory access that may happen before the barrier, so we * need to keep the mode. Any modes not kept are discarded. */ nir_deref_instr **p_deref; u_vector_foreach(p_deref, &mem_derefs) { nir_deref_instr *deref = *p_deref; const unsigned atomic_mode = glsl_contains_atomic(deref->type) ? nir_var_mem_ssbo : 0; const unsigned deref_modes = (deref->modes | atomic_mode) & barrier_modes; if (deref_modes && !barrier_happens_before(&barrier->instr, &deref->instr)) new_modes |= deref_modes; } /* If we don't need all the modes, update the barrier. */ if (barrier_modes != new_modes) { nir_intrinsic_set_memory_modes(barrier, new_modes); progress = true; } /* Shared memory only exists within a workgroup, so synchronizing it * beyond workgroup scope is nonsense. */ if (nir_intrinsic_execution_scope(barrier) == SCOPE_NONE && new_modes == nir_var_mem_shared) { nir_intrinsic_set_memory_scope(barrier, MIN2(nir_intrinsic_memory_scope(barrier), SCOPE_WORKGROUP)); progress = true; } } nir_instr_worklist_fini(&barriers); u_vector_finish(&mem_derefs); return progress; } /** * Reduce barriers to remove unnecessary modes and scope. * * This pass must be called before nir_lower_explicit_io lowers derefs! * * Many shaders issue full memory barriers, which may need to synchronize * access to images, SSBOs, shared local memory, or global memory. However, * many of them only use a subset of those memory types - say, only SSBOs. * * Shaders may also have patterns such as: * * 1. shared local memory access * 2. barrier with full variable modes * 3. more shared local memory access * 4. image access * * In this case, the barrier is needed to ensure synchronization between the * various shared memory operations. Image reads and writes do also exist, * but they are all on one side of the barrier, so it is a no-op for image * access. We can drop the image mode from the barrier in this case too. * * In addition, we can reduce the memory scope of shared-only barriers, as * shared local memory only exists within a workgroup. */ bool nir_opt_barrier_modes(nir_shader *shader) { bool progress = false; nir_foreach_function_impl(impl, shader) { nir_metadata_require(impl, nir_metadata_dominance | nir_metadata_instr_index); bool impl_progress = nir_opt_barrier_modes_impl(impl); progress |= nir_progress(impl_progress, impl, nir_metadata_control_flow | nir_metadata_live_defs); } return progress; }