diff options
Diffstat (limited to 'src/compiler/nir/nir_lower_shader_calls.c')
| -rw-r--r-- | src/compiler/nir/nir_lower_shader_calls.c | 205 |
1 files changed, 83 insertions, 122 deletions
diff --git a/src/compiler/nir/nir_lower_shader_calls.c b/src/compiler/nir/nir_lower_shader_calls.c index 2374b3bb022..816c39c6fd1 100644 --- a/src/compiler/nir/nir_lower_shader_calls.c +++ b/src/compiler/nir/nir_lower_shader_calls.c @@ -72,51 +72,16 @@ instr_is_shader_call(nir_instr *instr) intrin->intrinsic == nir_intrinsic_execute_callable; } -/* Previously named bitset, it had to be renamed as FreeBSD defines a struct - * named bitset in sys/_bitset.h required by pthread_np.h which is included - * from src/util/u_thread.h that is indirectly included by this file. - */ -struct sized_bitset { - BITSET_WORD *set; - unsigned size; -}; - -static struct sized_bitset -bitset_create(void *mem_ctx, unsigned size) -{ - return (struct sized_bitset){ - .set = rzalloc_array(mem_ctx, BITSET_WORD, BITSET_WORDS(size)), - .size = size, - }; -} - static bool src_is_in_bitset(nir_src *src, void *_set) { - struct sized_bitset *set = _set; - - /* Any SSA values which were added after we generated liveness information - * are things generated by this pass and, while most of it is arithmetic - * which we could re-materialize, we don't need to because it's only used - * for a single load/store and so shouldn't cross any shader calls. - */ - if (src->ssa->index >= set->size) - return false; - - return BITSET_TEST(set->set, src->ssa->index); -} - -static void -add_ssa_def_to_bitset(nir_def *def, struct sized_bitset *set) -{ - if (def->index >= set->size) - return; + struct u_sparse_bitset *set = _set; - BITSET_SET(set->set, def->index); + return u_sparse_bitset_test(set, src->ssa->index); } static bool -can_remat_instr(nir_instr *instr, struct sized_bitset *remat) +can_remat_instr(nir_instr *instr, struct u_sparse_bitset *remat) { /* Set of all values which are trivially re-materializable and we shouldn't * ever spill them. This includes: @@ -203,21 +168,21 @@ can_remat_instr(nir_instr *instr, struct sized_bitset *remat) } static bool -can_remat_ssa_def(nir_def *def, struct sized_bitset *remat) +can_remat_ssa_def(nir_def *def, struct u_sparse_bitset *remat) { return can_remat_instr(def->parent_instr, remat); } struct add_instr_data { struct util_dynarray *buf; - struct sized_bitset *remat; + struct u_sparse_bitset *remat; }; static bool add_src_instr(nir_src *src, void *state) { struct add_instr_data *data = state; - if (BITSET_TEST(data->remat->set, src->ssa->index)) + if (u_sparse_bitset_test(data->remat, src->ssa->index)) return true; util_dynarray_foreach(data->buf, nir_instr *, instr_ptr) { @@ -243,7 +208,7 @@ compare_instr_indexes(const void *_inst1, const void *_inst2) } static bool -can_remat_chain_ssa_def(nir_def *def, struct sized_bitset *remat, struct util_dynarray *buf) +can_remat_chain_ssa_def(nir_def *def, struct u_sparse_bitset *remat, struct util_dynarray *buf) { assert(util_dynarray_num_elements(buf, nir_instr *) == 0); @@ -274,14 +239,14 @@ can_remat_chain_ssa_def(nir_def *def, struct sized_bitset *remat, struct util_dy * through values that might not be in that set but that we can * rematerialize. */ - struct sized_bitset potential_remat = bitset_create(mem_ctx, remat->size); - memcpy(potential_remat.set, remat->set, BITSET_WORDS(remat->size) * sizeof(BITSET_WORD)); + struct u_sparse_bitset potential_remat; + u_sparse_bitset_dup(&potential_remat, remat); util_dynarray_foreach(buf, nir_instr *, instr_ptr) { nir_def *instr_ssa_def = nir_instr_def(*instr_ptr); /* If already in the potential rematerializable, nothing to do. */ - if (BITSET_TEST(potential_remat.set, instr_ssa_def->index)) + if (u_sparse_bitset_test(&potential_remat, instr_ssa_def->index)) continue; if (!can_remat_instr(*instr_ptr, &potential_remat)) @@ -290,7 +255,7 @@ can_remat_chain_ssa_def(nir_def *def, struct sized_bitset *remat, struct util_dy /* All the sources are rematerializable and the instruction is also * rematerializable, mark it as rematerializable too. */ - BITSET_SET(potential_remat.set, instr_ssa_def->index); + u_sparse_bitset_set(&potential_remat, instr_ssa_def->index); } ralloc_free(mem_ctx); @@ -313,8 +278,9 @@ remat_ssa_def(nir_builder *b, nir_def *def, struct hash_table *remap_table) static nir_def * remat_chain_ssa_def(nir_builder *b, struct util_dynarray *buf, - struct sized_bitset *remat, nir_def ***fill_defs, - unsigned call_idx, struct hash_table *remap_table) + struct u_sparse_bitset *remat, nir_def ***fill_defs, + unsigned call_idx, unsigned num_ssa_defs, + struct hash_table *remap_table) { nir_def *last_def = NULL; @@ -331,14 +297,14 @@ remat_chain_ssa_def(nir_builder *b, struct util_dynarray *buf, if (fill_defs[ssa_index] == NULL) { fill_defs[ssa_index] = - rzalloc_array(fill_defs, nir_def *, remat->size); + rzalloc_array(fill_defs, nir_def *, num_ssa_defs); } /* Add the new ssa_def to the list fill_defs and flag it as * rematerialized */ fill_defs[ssa_index][call_idx] = last_def = clone_ssa_def; - BITSET_SET(remat->set, ssa_index); + u_sparse_bitset_set(remat, ssa_index); _mesa_hash_table_insert(remap_table, instr_ssa_def, last_def); } @@ -414,9 +380,9 @@ spill_fill(nir_builder *before, nir_builder *after, nir_def *def, static bool add_src_to_call_live_bitset(nir_src *src, void *state) { - BITSET_WORD *call_live = state; + struct u_sparse_bitset *call_live = state; - BITSET_SET(call_live, src->ssa->index); + u_sparse_bitset_set(call_live, src->ssa->index); return true; } @@ -453,8 +419,8 @@ spill_ssa_defs_and_lower_shader_calls(nir_shader *shader, uint32_t num_calls, void *mem_ctx = ralloc_context(shader); const unsigned num_ssa_defs = impl->ssa_alloc; - const unsigned live_words = BITSET_WORDS(num_ssa_defs); - struct sized_bitset trivial_remat = bitset_create(mem_ctx, num_ssa_defs); + struct u_sparse_bitset trivial_remat; + u_sparse_bitset_init(&trivial_remat, num_ssa_defs, mem_ctx); /* Array of all live SSA defs which are spill candidates */ nir_def **spill_defs = @@ -467,8 +433,8 @@ spill_ssa_defs_and_lower_shader_calls(nir_shader *shader, uint32_t num_calls, rzalloc_array(mem_ctx, nir_def **, num_ssa_defs); /* For each call instruction, the liveness set at the call */ - const BITSET_WORD **call_live = - rzalloc_array(mem_ctx, const BITSET_WORD *, num_calls); + struct u_sparse_bitset **call_live = + rzalloc_array(mem_ctx, struct u_sparse_bitset *, num_calls); /* For each call instruction, the block index of the block it lives in */ uint32_t *call_block_indices = rzalloc_array(mem_ctx, uint32_t, num_calls); @@ -512,8 +478,8 @@ spill_ssa_defs_and_lower_shader_calls(nir_shader *shader, uint32_t num_calls, * spilled/filled sources. */ if (options->should_remat_callback) { - BITSET_WORD **updated_call_live = - rzalloc_array(mem_ctx, BITSET_WORD *, num_calls); + struct u_sparse_bitset **updated_call_live = + rzalloc_array(mem_ctx, struct u_sparse_bitset *, num_calls); nir_foreach_block(block, impl) { nir_foreach_instr(instr, block) { @@ -522,7 +488,7 @@ spill_ssa_defs_and_lower_shader_calls(nir_shader *shader, uint32_t num_calls, continue; for (unsigned c = 0; c < num_calls; c++) { - if (!BITSET_TEST(call_live[c], def->index)) + if (!u_sparse_bitset_test(call_live[c], def->index)) continue; if (!options->should_remat_callback(def->parent_instr, @@ -530,9 +496,8 @@ spill_ssa_defs_and_lower_shader_calls(nir_shader *shader, uint32_t num_calls, continue; if (updated_call_live[c] == NULL) { - const unsigned bitset_words = BITSET_WORDS(impl->ssa_alloc); - updated_call_live[c] = ralloc_array(mem_ctx, BITSET_WORD, bitset_words); - memcpy(updated_call_live[c], call_live[c], bitset_words * sizeof(BITSET_WORD)); + updated_call_live[c] = ralloc_size(mem_ctx, sizeof(struct u_sparse_bitset)); + u_sparse_bitset_dup(updated_call_live[c], call_live[c]); } nir_foreach_src(instr, add_src_to_call_live_bitset, updated_call_live[c]); @@ -557,7 +522,7 @@ spill_ssa_defs_and_lower_shader_calls(nir_shader *shader, uint32_t num_calls, nir_def *def = nir_instr_def(instr); if (def != NULL) { if (can_remat_ssa_def(def, &trivial_remat)) { - add_ssa_def_to_bitset(def, &trivial_remat); + u_sparse_bitset_set(&trivial_remat, def->index); _mesa_hash_table_insert(trivial_remap_table, def, def); } else { spill_defs[def->index] = def; @@ -567,7 +532,7 @@ spill_ssa_defs_and_lower_shader_calls(nir_shader *shader, uint32_t num_calls, if (!instr_is_shader_call(instr)) continue; - const BITSET_WORD *live = call_live[call_idx]; + struct u_sparse_bitset *live = call_live[call_idx]; struct hash_table *remap_table = _mesa_hash_table_clone(trivial_remap_table, mem_ctx); @@ -575,8 +540,8 @@ spill_ssa_defs_and_lower_shader_calls(nir_shader *shader, uint32_t num_calls, /* Make a copy of trivial_remat that we'll update as we crawl through * the live SSA defs and unspill them. */ - struct sized_bitset remat = bitset_create(mem_ctx, num_ssa_defs); - memcpy(remat.set, trivial_remat.set, live_words * sizeof(BITSET_WORD)); + struct u_sparse_bitset remat; + u_sparse_bitset_dup(&remat, &trivial_remat); /* Before the two builders are always separated by the call * instruction, it won't break anything to have two of them. @@ -596,63 +561,58 @@ spill_ssa_defs_and_lower_shader_calls(nir_shader *shader, uint32_t num_calls, util_dynarray_init_from_stack(&remat_chain, remat_chain_instrs, sizeof(remat_chain_instrs)); unsigned offset = shader->scratch_size; - for (unsigned w = 0; w < live_words; w++) { - BITSET_WORD spill_mask = live[w] & ~trivial_remat.set[w]; - while (spill_mask) { - int i = u_bit_scan(&spill_mask); - assert(i >= 0); - unsigned index = w * BITSET_WORDBITS + i; - assert(index < num_ssa_defs); - - def = spill_defs[index]; - nir_def *original_def = def, *new_def; - if (can_remat_ssa_def(def, &remat)) { - /* If this SSA def is re-materializable or based on other - * things we've already spilled, re-materialize it rather - * than spilling and filling. Anything which is trivially - * re-materializable won't even get here because we take - * those into account in spill_mask above. - */ - new_def = remat_ssa_def(&after, def, remap_table); - } else if (can_remat_chain_ssa_def(def, &remat, &remat_chain)) { - new_def = remat_chain_ssa_def(&after, &remat_chain, &remat, - fill_defs, call_idx, - remap_table); - util_dynarray_clear(&remat_chain); - } else { - bool is_bool = def->bit_size == 1; - if (is_bool) - def = nir_b2b32(&before, def); - - const unsigned comp_size = def->bit_size / 8; - offset = ALIGN(offset, comp_size); - - new_def = spill_fill(&before, &after, def, - index, call_idx, - offset, options->stack_alignment); - - if (is_bool) - new_def = nir_b2b1(&after, new_def); - - offset += def->num_components * comp_size; - } + U_SPARSE_BITSET_FOREACH_SET(live, index) { + if (u_sparse_bitset_test(&trivial_remat, index)) + continue; - /* Mark this SSA def as available in the remat set so that, if - * some other SSA def we need is computed based on it, we can - * just re-compute instead of fetching from memory. + def = spill_defs[index]; + nir_def *original_def = def, *new_def; + if (can_remat_ssa_def(def, &remat)) { + /* If this SSA def is re-materializable or based on other + * things we've already spilled, re-materialize it rather + * than spilling and filling. Anything which is trivially + * re-materializable won't even get here because we take + * those into account in spill_mask above. */ - BITSET_SET(remat.set, index); + new_def = remat_ssa_def(&after, def, remap_table); + } else if (can_remat_chain_ssa_def(def, &remat, &remat_chain)) { + new_def = remat_chain_ssa_def(&after, &remat_chain, &remat, + fill_defs, call_idx, num_ssa_defs, + remap_table); + util_dynarray_clear(&remat_chain); + } else { + bool is_bool = def->bit_size == 1; + if (is_bool) + def = nir_b2b32(&before, def); - /* For now, we just make a note of this new SSA def. We'll - * fix things up with the phi builder as a second pass. - */ - if (fill_defs[index] == NULL) { - fill_defs[index] = - rzalloc_array(fill_defs, nir_def *, num_calls); - } - fill_defs[index][call_idx] = new_def; - _mesa_hash_table_insert(remap_table, original_def, new_def); + const unsigned comp_size = def->bit_size / 8; + offset = ALIGN(offset, comp_size); + + new_def = spill_fill(&before, &after, def, + index, call_idx, + offset, options->stack_alignment); + + if (is_bool) + new_def = nir_b2b1(&after, new_def); + + offset += def->num_components * comp_size; + } + + /* Mark this SSA def as available in the remat set so that, if + * some other SSA def we need is computed based on it, we can + * just re-compute instead of fetching from memory. + */ + u_sparse_bitset_set(&remat, index); + + /* For now, we just make a note of this new SSA def. We'll + * fix things up with the phi builder as a second pass. + */ + if (fill_defs[index] == NULL) { + fill_defs[index] = + rzalloc_array(fill_defs, nir_def *, num_calls); } + fill_defs[index][call_idx] = new_def; + _mesa_hash_table_insert(remap_table, original_def, new_def); } nir_builder *b = &before; @@ -1005,7 +965,7 @@ flatten_resume_if_ladder(nir_builder *b, struct exec_list *child_list, bool child_list_contains_cursor, nir_instr *resume_instr, - struct sized_bitset *remat) + struct u_sparse_bitset *remat) { nir_cf_list cf_list; @@ -1044,7 +1004,7 @@ flatten_resume_if_ladder(nir_builder *b, b->cursor = nir_after_instr(instr); nir_def *def = nir_instr_def(instr); - BITSET_SET(remat->set, def->index); + u_sparse_bitset_set(remat, def->index); } } if (b->cursor.option == nir_cursor_after_block && @@ -1276,7 +1236,8 @@ lower_resume(nir_shader *shader, int call_idx) /* Used to track which things may have been assumed to be re-materialized * by the spilling pass and which we shouldn't delete. */ - struct sized_bitset remat = bitset_create(mem_ctx, impl->ssa_alloc); + struct u_sparse_bitset remat; + u_sparse_bitset_init(&remat, impl->ssa_alloc, mem_ctx); /* Create a nop instruction to use as a cursor as we extract and re-insert * stuff into the CFG. |