switch (p_device.vendor) {

case RenderingContextDriver::Vendor::VENDOR_AMD:

return "AMD";

case RenderingContextDriver::Vendor::VENDOR_IMGTEC:

return "ImgTec";

case RenderingContextDriver::Vendor::VENDOR_APPLE:

return "Apple";

case RenderingContextDriver::Vendor::VENDOR_NVIDIA:

return "NVIDIA";

case RenderingContextDriver::Vendor::VENDOR_ARM:

return "ARM";

case RenderingContextDriver::Vendor::VENDOR_MICROSOFT:

return "Microsoft";

case RenderingContextDriver::Vendor::VENDOR_QUALCOMM:

return "Qualcomm";

case RenderingContextDriver::Vendor::VENDOR_INTEL:

return "Intel";

default:

return "Unknown";

}

switch (p_device.type) {

case RenderingContextDriver::DEVICE_TYPE_INTEGRATED_GPU:

return "Integrated";

case RenderingContextDriver::DEVICE_TYPE_DISCRETE_GPU:

return "Discrete";

case RenderingContextDriver::DEVICE_TYPE_VIRTUAL_GPU:

return "Virtual";

case RenderingContextDriver::DEVICE_TYPE_CPU:

return "CPU";

case RenderingContextDriver::DEVICE_TYPE_OTHER:

default:

return "Other";

}

static const bool prefer_integrated = OS::get_singleton()->get_user_prefers_integrated_gpu();

switch (p_device.type) {

case RenderingContextDriver::DEVICE_TYPE_INTEGRATED_GPU:

return prefer_integrated ? 5 : 4;

case RenderingContextDriver::DEVICE_TYPE_DISCRETE_GPU:

return prefer_integrated ? 4 : 5;

case RenderingContextDriver::DEVICE_TYPE_VIRTUAL_GPU:

return 3;

case RenderingContextDriver::DEVICE_TYPE_CPU:

return 2;

case RenderingContextDriver::DEVICE_TYPE_OTHER:

default:

return 1;

}

_THREAD_SAFE_METHOD_

HashSet<RID> *set = dependency_map.getptr(p_depends_on);

if (set == nullptr) {

set = &dependency_map.insert(p_depends_on, HashSet<RID>())->value;

}

set->insert(p_id);

set = reverse_dependency_map.getptr(p_id);

if (set == nullptr) {

set = &reverse_dependency_map.insert(p_id, HashSet<RID>())->value;

}

set->insert(p_depends_on);

_THREAD_SAFE_METHOD_

// Direct dependencies must be freed.

HashMap<RID, HashSet<RID>>::Iterator E = dependency_map.find(p_id);

if (E) {

while (E->value.size()) {

free_rid(*E->value.begin());

}

dependency_map.remove(E);

}

// Reverse dependencies must be unreferenced.

E = reverse_dependency_map.find(p_id);

if (E) {

for (const RID &F : E->value) {

HashMap<RID, HashSet<RID>>::Iterator G = dependency_map.find(F);

ERR_CONTINUE(!G);

ERR_CONTINUE(!G->value.has(p_id));

G->value.erase(p_id);

}

reverse_dependency_map.remove(E);

}

switch (p_language) {

#ifdef MODULE_GLSLANG_ENABLED

case ShaderLanguage::SHADER_LANGUAGE_GLSL: {

ShaderLanguageVersion language_version = driver->get_shader_container_format().get_shader_language_version();

ShaderSpirvVersion spirv_version = driver->get_shader_container_format().get_shader_spirv_version();

return compile_glslang_shader(p_stage, ShaderIncludeDB::parse_include_files(p_source_code), language_version, spirv_version, r_error);

}

#endif

default:

ERR_FAIL_V_MSG(Vector<uint8_t>(), "Shader language is not supported.");

}

Vector<uint8_t> bytecode = shader_compile_binary_from_spirv(p_spirv, p_shader_name);

ERR_FAIL_COND_V(bytecode.is_empty(), RID());

return shader_create_from_bytecode(bytecode);

ERR_FAIL_COND_V_MSG(!has_feature(SUPPORTS_RAYTRACING_PIPELINE) && !has_feature(SUPPORTS_RAY_QUERY), RID(), "The current rendering device has neither raytracing pipeline nor ray query support.");

VertexArray *vertex_array = vertex_array_owner.get_or_null(p_vertex_array);

ERR_FAIL_NULL_V(vertex_array, RID());

uint32_t position_binding = p_position_attribute_location;

RDD::VertexFormatID vertex_format;

if (vertex_array->description != INVALID_ID) {

ERR_FAIL_COND_V(!vertex_formats.has(vertex_array->description), RID());

const VertexDescriptionCache &vd_cache = vertex_formats[vertex_array->description];

vertex_format = vd_cache.driver_id;

const VertexAttribute *position_attribute = nullptr;

for (int i = 0; i < vd_cache.vertex_formats.size(); i++) {

const VertexAttribute &attr = vd_cache.vertex_formats[i];

if (attr.location == p_position_attribute_location) {

position_attribute = &attr;

break;

}

}

ERR_FAIL_NULL_V_MSG(position_attribute, RID(), vformat("Vertex array is missing a position attribute at location %u.", p_position_attribute_location));

ERR_FAIL_COND_V_MSG(position_attribute->frequency != VERTEX_FREQUENCY_VERTEX, RID(), vformat("Position attribute at location %u must use vertex frequency.", p_position_attribute_location));

if (position_attribute->binding != UINT32_MAX) {

position_binding = position_attribute->binding;

}

}

ERR_FAIL_COND_V_MSG(position_binding >= (uint32_t)vertex_array->buffers.size(), RID(), vformat("Vertex array is missing a buffer for binding %u.", position_binding));

RDD::BufferID vertex_buffer = vertex_array->buffers[position_binding];

uint64_t vertex_offset = vertex_array->offsets[position_binding];

// Indices are optional.

IndexArray *index_array = index_array_owner.get_or_null(p_index_array);

RDD::BufferID index_buffer = RDD::BufferID();

IndexBufferFormat index_format = IndexBufferFormat::INDEX_BUFFER_FORMAT_UINT32;

uint32_t index_offset_bytes = 0;

uint32_t index_count = 0;

if (index_array) {

index_buffer = index_array->driver_id;

index_format = index_array->format;

index_offset_bytes = index_array->offset * (index_array->format == INDEX_BUFFER_FORMAT_UINT16 ? sizeof(uint16_t) : sizeof(uint32_t));

index_count = index_array->indices;

}

AccelerationStructure acceleration_structure;

acceleration_structure.type = RDD::ACCELERATION_STRUCTURE_TYPE_BLAS;

BitField<RDD::AccelerationStructureGeometryBits> geometry_bits = 0;

if (p_geometry_bits.has_flag(ACCELERATION_STRUCTURE_GEOMETRY_OPAQUE)) {

geometry_bits.set_flag(RDD::ACCELERATION_STRUCTURE_GEOMETRY_OPAQUE);

}

if (p_geometry_bits.has_flag(ACCELERATION_STRUCTURE_GEOMETRY_NO_DUPLICATE_ANY_HIT_INVOCATION)) {

geometry_bits.set_flag(RDD::ACCELERATION_STRUCTURE_GEOMETRY_NO_DUPLICATE_ANY_HIT_INVOCATION);

}

acceleration_structure.driver_id = driver->blas_create(vertex_buffer, vertex_offset, vertex_format, vertex_array->vertex_count, p_position_attribute_location, index_buffer, index_format, index_offset_bytes, index_count, geometry_bits);

ERR_FAIL_COND_V_MSG(!acceleration_structure.driver_id, RID(), "Failed to create BLAS.");

acceleration_structure.vertex_array = p_vertex_array;

acceleration_structure.index_array = p_index_array;

acceleration_structure.draw_tracker = RDG::resource_tracker_create();

acceleration_structure.draw_tracker->acceleration_structure_driver_id = acceleration_structure.driver_id;

// Assume we are going to build this acceleration structure

acceleration_structure.draw_tracker->usage = RDG::RESOURCE_USAGE_ACCELERATION_STRUCTURE_READ_WRITE;

for (int i = 0; i < vertex_array->draw_trackers.size(); i++) {

acceleration_structure.draw_trackers.push_back(vertex_array->draw_trackers[i]);

}

_check_transfer_worker_vertex_array(vertex_array);

if (index_array && index_array->draw_tracker) {

acceleration_structure.draw_trackers.push_back(index_array->draw_tracker);

}

_check_transfer_worker_index_array(index_array);

RID id = acceleration_structure_owner.make_rid(acceleration_structure);

#ifdef DEV_ENABLED

set_resource_name(id, "RID:" + itos(id.get_id()));

#endif

return id;

BitField<RDD::BufferUsageBits> usage = 0;

if (p_creation_bits.has_flag(BUFFER_CREATION_AS_STORAGE_BIT)) {

usage.set_flag(RDD::BUFFER_USAGE_STORAGE_BIT);

}

if (p_creation_bits.has_flag(BUFFER_CREATION_DEVICE_ADDRESS_BIT)) {

#ifdef DEBUG_ENABLED

ERR_FAIL_COND_V_MSG(!has_feature(SUPPORTS_BUFFER_DEVICE_ADDRESS), 0,

"The GPU doesn't support buffer address flag.");

#endif

usage.set_flag(RDD::BUFFER_USAGE_DEVICE_ADDRESS_BIT);

}

if (p_creation_bits.has_flag(BUFFER_CREATION_ACCELERATION_STRUCTURE_BUILD_INPUT_READ_ONLY_BIT)) {

#ifdef DEBUG_ENABLED

ERR_FAIL_COND_V_MSG(!has_feature(SUPPORTS_RAYTRACING_PIPELINE) && !has_feature(SUPPORTS_RAY_QUERY), 0,

"The GPU doesn't support acceleration structure build input flag.");

#endif

usage.set_flag(RDD::BUFFER_USAGE_ACCELERATION_STRUCTURE_BUILD_INPUT_READ_ONLY_BIT);

}

return usage;

ERR_FAIL_COND_V_MSG(!has_feature(SUPPORTS_RAYTRACING_PIPELINE) && !has_feature(SUPPORTS_RAY_QUERY), RID(), "The current rendering device has neither raytracing pipeline nor ray query support.");

ERR_FAIL_COND_V(p_instance_count == 0, RID());

uint32_t instances_buffer_size_bytes = driver->tlas_instances_buffer_get_size_bytes(p_instance_count);

InstancesBuffer instances_buffer;

instances_buffer.instance_count = p_instance_count;

instances_buffer.buffer.size = instances_buffer_size_bytes;

instances_buffer.buffer.usage = _creation_to_usage_bits(p_creation_bits) | RDD::BUFFER_USAGE_TRANSFER_FROM_BIT | RDD::BUFFER_USAGE_DEVICE_ADDRESS_BIT | RDD::BUFFER_USAGE_ACCELERATION_STRUCTURE_BUILD_INPUT_READ_ONLY_BIT;

instances_buffer.buffer.driver_id = driver->buffer_create(instances_buffer.buffer.size, instances_buffer.buffer.usage, RDD::MEMORY_ALLOCATION_TYPE_CPU, frames_drawn);

ERR_FAIL_COND_V_MSG(!instances_buffer.buffer.driver_id, RID(), "Failed to create instances buffer.");

_THREAD_SAFE_LOCK_

buffer_memory += instances_buffer.buffer.size;

_THREAD_SAFE_UNLOCK_

RID id = instances_buffer_owner.make_rid(instances_buffer);

#ifdef DEV_ENABLED

set_resource_name(id, "RID:" + itos(id.get_id()));

#endif

return id;

ERR_FAIL_COND_MSG(!has_feature(SUPPORTS_RAYTRACING_PIPELINE) && !has_feature(SUPPORTS_RAY_QUERY), "The current rendering device has neither raytracing pipeline nor ray query support.");

InstancesBuffer *instances_buffer = instances_buffer_owner.get_or_null(p_instances_buffer);

ERR_FAIL_NULL_MSG(instances_buffer, "Instances buffer input is not valid.");

uint32_t blases_count = p_blases.size();

ERR_FAIL_COND_MSG(blases_count != instances_buffer->instance_count, "The number of blases is not equal to the instance count of the instances buffer.");

ERR_FAIL_COND_MSG(blases_count != p_transforms.size(), "Blases and transforms vectors must have the same size.");

thread_local LocalVector<RDD::AccelerationStructureID> blases;

blases.resize(blases_count);

for (uint32_t i = 0; i < blases_count; i++) {

const AccelerationStructure *blas = acceleration_structure_owner.get_or_null(p_blases[i]);

ERR_FAIL_NULL_MSG(blas, "BLAS input is not valid.");

ERR_FAIL_COND_MSG(blas->type != RDD::ACCELERATION_STRUCTURE_TYPE_BLAS, "Acceleration structure input is not a BLAS.");

blases[i] = blas->driver_id;

}

instances_buffer->blases = p_blases;

driver->tlas_instances_buffer_fill(instances_buffer->buffer.driver_id, blases, p_transforms);

ERR_FAIL_COND_V_MSG(!has_feature(SUPPORTS_RAYTRACING_PIPELINE) && !has_feature(SUPPORTS_RAY_QUERY), RID(), "The current rendering device has neither raytracing pipeline nor ray query support.");

const InstancesBuffer *instances_buffer = instances_buffer_owner.get_or_null(p_instances_buffer);

ERR_FAIL_NULL_V_MSG(instances_buffer, RID(), "Instances buffer input is not valid.");

AccelerationStructure acceleration_structure;

acceleration_structure.type = RDD::ACCELERATION_STRUCTURE_TYPE_TLAS;

acceleration_structure.driver_id = driver->tlas_create(instances_buffer->buffer.driver_id);

ERR_FAIL_COND_V_MSG(!acceleration_structure.driver_id, RID(), "Failed to create TLAS.");

acceleration_structure.instances_buffer = p_instances_buffer;

acceleration_structure.draw_tracker = RDG::resource_tracker_create();

acceleration_structure.draw_tracker->acceleration_structure_driver_id = acceleration_structure.driver_id;

// Assume we are going to build this acceleration structure

acceleration_structure.draw_tracker->usage = RDG::RESOURCE_USAGE_ACCELERATION_STRUCTURE_READ_WRITE;

for (Vector<RID>::ConstIterator itr = instances_buffer->blases.begin(); itr != instances_buffer->blases.end(); ++itr) {

const AccelerationStructure *blas = acceleration_structure_owner.get_or_null(*itr);

ERR_FAIL_NULL_V_MSG(blas, RID(), "BLAS input is not valid.");

if (blas->draw_tracker) {

acceleration_structure.draw_trackers.push_back(blas->draw_tracker);

}

}

RID id = acceleration_structure_owner.make_rid(acceleration_structure);

#ifdef DEV_ENABLED

set_resource_name(id, "RID:" + itos(id.get_id()));

#endif

return id;

ERR_RENDER_THREAD_GUARD_V(ERR_UNAVAILABLE);

ERR_FAIL_COND_V_MSG(draw_list.active, ERR_INVALID_PARAMETER,

"Building acceleration structures is forbidden during creation of a draw list.");

ERR_FAIL_COND_V_MSG(compute_list.active, ERR_INVALID_PARAMETER,

"Building acceleration structures is forbidden during creation of a compute list.");

ERR_FAIL_COND_V_MSG(raytracing_list.active, ERR_INVALID_PARAMETER,

"Building acceleration structures is forbidden during creation of a raytracing list.");

AccelerationStructure *accel = acceleration_structure_owner.get_or_null(p_acceleration_structure);

ERR_FAIL_NULL_V_MSG(accel, ERR_INVALID_PARAMETER, "Acceleration structure argument is not valid.");

uint64_t scratch_size = driver->acceleration_structure_get_scratch_size_bytes(accel->driver_id);

if (accel->scratch_buffer) {

uint64_t scratch_buffer_size = driver->buffer_get_allocation_size(accel->scratch_buffer);

if (scratch_buffer_size < scratch_size) {

Buffer to_dispose = Buffer();

to_dispose.driver_id = accel->scratch_buffer;

DEV_ASSERT(scratch_buffer_size <= UINT32_MAX);

to_dispose.size = scratch_buffer_size;

frames[frame].buffers_to_dispose_of.push_back(to_dispose);

accel->scratch_buffer = RDD::BufferID();

}

}

if (!accel->scratch_buffer) {

accel->scratch_buffer = driver->buffer_create(scratch_size, RDD::BUFFER_USAGE_STORAGE_BIT | RDD::BUFFER_USAGE_DEVICE_ADDRESS_BIT, RDD::MEMORY_ALLOCATION_TYPE_GPU, frames_drawn);

ERR_FAIL_COND_V(!accel->scratch_buffer, ERR_CANT_CREATE);

_THREAD_SAFE_LOCK_

buffer_memory += scratch_size;

_THREAD_SAFE_UNLOCK_

}

draw_graph.add_acceleration_structure_build(accel->driver_id, accel->scratch_buffer, accel->draw_tracker, accel->draw_trackers);

return OK;

uint32_t transfer_worker_offset;

TransferWorker *transfer_worker = _acquire_transfer_worker(p_data.size(), p_required_align, transfer_worker_offset);

p_buffer->transfer_worker_index = transfer_worker->index;

{

MutexLock lock(transfer_worker->operations_mutex);

p_buffer->transfer_worker_operation = ++transfer_worker->operations_counter;

}

// Copy to the worker's staging buffer.

uint8_t *data_ptr = driver->buffer_map(transfer_worker->staging_buffer);

ERR_FAIL_NULL_V(data_ptr, ERR_CANT_CREATE);

memcpy(data_ptr + transfer_worker_offset, p_data.ptr(), p_data.size());

driver->buffer_unmap(transfer_worker->staging_buffer);

// Copy from the staging buffer to the real buffer.

RDD::BufferCopyRegion region;

region.src_offset = transfer_worker_offset;

region.dst_offset = 0;

region.size = p_data.size();

driver->command_copy_buffer(transfer_worker->command_buffer, transfer_worker->staging_buffer, p_buffer->driver_id, region);

_release_transfer_worker(transfer_worker);

return OK;

StagingBufferBlock block;

block.driver_id = driver->buffer_create(p_staging_buffers.block_size, p_staging_buffers.usage_bits, RDD::MEMORY_ALLOCATION_TYPE_CPU, frames_drawn);

ERR_FAIL_COND_V(!block.driver_id, ERR_CANT_CREATE);

block.frame_used = 0;

block.fill_amount = 0;

block.data_ptr = driver->buffer_map(block.driver_id);

if (block.data_ptr == nullptr) {

driver->buffer_free(block.driver_id);

return ERR_CANT_CREATE;

}

p_staging_buffers.blocks.insert(p_staging_buffers.current, block);

return OK;

// Determine a block to use.

r_alloc_size = p_amount;

r_required_action = STAGING_REQUIRED_ACTION_NONE;

while (true) {

r_alloc_offset = 0;

// See if we can use current block.

if (p_staging_buffers.blocks[p_staging_buffers.current].frame_used == frames_drawn) {

// We used this block this frame, let's see if there is still room.

uint32_t write_from = p_staging_buffers.blocks[p_staging_buffers.current].fill_amount;

{

uint32_t align_remainder = write_from % p_required_align;

if (align_remainder != 0) {

write_from += p_required_align - align_remainder;

}

}

int32_t available_bytes = int32_t(p_staging_buffers.block_size) - int32_t(write_from);

if ((int32_t)p_amount < available_bytes) {

// All is good, we should be ok, all will fit.

r_alloc_offset = write_from;

} else if (p_can_segment && available_bytes >= (int32_t)p_required_align) {

// Ok all won't fit but at least we can fit a chunkie.

// All is good, update what needs to be written to.

r_alloc_offset = write_from;

r_alloc_size = available_bytes - (available_bytes % p_required_align);

} else {

// Can't fit it into this buffer.

// Will need to try next buffer.

p_staging_buffers.current = (p_staging_buffers.current + 1) % p_staging_buffers.blocks.size();

// Before doing anything, though, let's check that we didn't manage to fill all blocks.

// Possible in a single frame.

if (p_staging_buffers.blocks[p_staging_buffers.current].frame_used == frames_drawn) {

// Guess we did.. ok, let's see if we can insert a new block.

if ((uint64_t)p_staging_buffers.blocks.size() * p_staging_buffers.block_size < p_staging_buffers.max_size) {

// We can, so we are safe.

Error err = _insert_staging_block(p_staging_buffers);

if (err) {

return err;

}

// Claim for this frame.

p_staging_buffers.blocks.write[p_staging_buffers.current].frame_used = frames_drawn;

} else {

// Ok, worst case scenario, all the staging buffers belong to this frame

// and this frame is not even done.

// If this is the main thread, it means the user is likely loading a lot of resources at once,.

// Otherwise, the thread should just be blocked until the next frame (currently unimplemented).

r_required_action = STAGING_REQUIRED_ACTION_FLUSH_AND_STALL_ALL;

}

} else {

// Not from current frame, so continue and try again.

continue;

}

}

} else if (p_staging_buffers.blocks[p_staging_buffers.current].frame_used <= frames_drawn - frames.size()) {

// This is an old block, which was already processed, let's reuse.

p_staging_buffers.blocks.write[p_staging_buffers.current].frame_used = frames_drawn;

p_staging_buffers.blocks.write[p_staging_buffers.current].fill_amount = 0;

} else {

// This block may still be in use, let's not touch it unless we have to, so.. can we create a new one?

if ((uint64_t)p_staging_buffers.blocks.size() * p_staging_buffers.block_size < p_staging_buffers.max_size) {

// We are still allowed to create a new block, so let's do that and insert it for current pos.

Error err = _insert_staging_block(p_staging_buffers);

if (err) {

return err;

}

// Claim for this frame.

p_staging_buffers.blocks.write[p_staging_buffers.current].frame_used = frames_drawn;

} else {

// Oops, we are out of room and we can't create more.

// Let's flush older frames.

// The logic here is that if a game is loading a lot of data from the main thread, it will need to be stalled anyway.

// If loading from a separate thread, we can block that thread until next frame when more room is made (not currently implemented, though).

r_required_action = STAGING_REQUIRED_ACTION_STALL_PREVIOUS;

}

}

// All was good, break.

break;

}

p_staging_buffers.used = true;

return OK;

switch (p_required_action) {

case STAGING_REQUIRED_ACTION_NONE: {

// Do nothing.

} break;

case STAGING_REQUIRED_ACTION_FLUSH_AND_STALL_ALL: {

_flush_and_stall_for_all_frames();

// Clear the whole staging buffer.

for (int i = 0; i < p_staging_buffers.blocks.size(); i++) {

p_staging_buffers.blocks.write[i].frame_used = 0;

p_staging_buffers.blocks.write[i].fill_amount = 0;

}

// Claim for current frame.

p_staging_buffers.blocks.write[p_staging_buffers.current].frame_used = frames_drawn;

} break;

case STAGING_REQUIRED_ACTION_STALL_PREVIOUS: {

_stall_for_previous_frames();

for (int i = 0; i < p_staging_buffers.blocks.size(); i++) {

// Clear all blocks but the ones from this frame.

int block_idx = (i + p_staging_buffers.current) % p_staging_buffers.blocks.size();

if (p_staging_buffers.blocks[block_idx].frame_used == frames_drawn) {

break; // Ok, we reached something from this frame, abort.

}

p_staging_buffers.blocks.write[block_idx].frame_used = 0;

p_staging_buffers.blocks.write[block_idx].fill_amount = 0;

}

// Claim for current frame.

p_staging_buffers.blocks.write[p_staging_buffers.current].frame_used = frames_drawn;

} break;

default: {

DEV_ASSERT(false && "Unknown required action.");

} break;

}

ERR_RENDER_THREAD_GUARD_V(ERR_UNAVAILABLE);

ERR_FAIL_COND_V_MSG(draw_list.active, ERR_INVALID_PARAMETER,

"Copying buffers is forbidden during creation of a draw list.");

ERR_FAIL_COND_V_MSG(compute_list.active, ERR_INVALID_PARAMETER,

"Copying buffers is forbidden during creation of a compute list.");

ERR_FAIL_COND_V_MSG(raytracing_list.active, ERR_INVALID_PARAMETER,

"Copying buffers is forbidden during creation of a raytracing list.");

Buffer *src_buffer = _get_buffer_from_owner(p_src_buffer);

if (!src_buffer) {

ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, "Source buffer argument is not a valid buffer of any type.");

}

Buffer *dst_buffer = _get_buffer_from_owner(p_dst_buffer);

if (!dst_buffer) {

ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, "Destination buffer argument is not a valid buffer of any type.");

}

// Validate the copy's dimensions for both buffers.

ERR_FAIL_COND_V_MSG((p_size + p_src_offset) > src_buffer->size, ERR_INVALID_PARAMETER, "Size is larger than the source buffer.");

ERR_FAIL_COND_V_MSG((p_size + p_dst_offset) > dst_buffer->size, ERR_INVALID_PARAMETER, "Size is larger than the destination buffer.");

_check_transfer_worker_buffer(src_buffer);

_check_transfer_worker_buffer(dst_buffer);

// Perform the copy.

RDD::BufferCopyRegion region;

region.src_offset = p_src_offset;

region.dst_offset = p_dst_offset;

region.size = p_size;

if (_buffer_make_mutable(dst_buffer, p_dst_buffer)) {

// The destination buffer must be mutable to be used as a copy destination.

draw_graph.add_synchronization();

}

draw_graph.add_buffer_copy(src_buffer->driver_id, src_buffer->draw_tracker, dst_buffer->driver_id, dst_buffer->draw_tracker, region);

return OK;

ERR_RENDER_THREAD_GUARD_V(ERR_UNAVAILABLE);

copy_bytes_count += p_size;

ERR_FAIL_COND_V_MSG(draw_list.active && !p_skip_check, ERR_INVALID_PARAMETER,

"Updating buffers is forbidden during creation of a draw list.");

ERR_FAIL_COND_V_MSG(compute_list.active && !p_skip_check, ERR_INVALID_PARAMETER,

"Updating buffers is forbidden during creation of a compute list.");

ERR_FAIL_COND_V_MSG(raytracing_list.active && !p_skip_check, ERR_INVALID_PARAMETER,

"Updating buffers is forbidden during creation of a raytracing list.");

Buffer *buffer = _get_buffer_from_owner(p_buffer);

ERR_FAIL_NULL_V_MSG(buffer, ERR_INVALID_PARAMETER, "Buffer argument is not a valid buffer of any type.");

ERR_FAIL_COND_V_MSG(p_offset + p_size > buffer->size, ERR_INVALID_PARAMETER, "Attempted to write buffer (" + itos((p_offset + p_size) - buffer->size) + " bytes) past the end.");

if (buffer->usage.has_flag(RDD::BUFFER_USAGE_DYNAMIC_PERSISTENT_BIT)) {

uint8_t *dst_data = driver->buffer_persistent_map_advance(buffer->driver_id, frames_drawn);

memcpy(dst_data + p_offset, p_data, p_size);

direct_copy_count++;

buffer_flush(p_buffer);

return OK;

}

_check_transfer_worker_buffer(buffer);

// Submitting may get chunked for various reasons, so convert this to a task.

size_t to_submit = p_size;

size_t submit_from = 0;

thread_local LocalVector<RDG::RecordedBufferCopy> command_buffer_copies_vector;

command_buffer_copies_vector.clear();

const uint8_t *src_data = reinterpret_cast<const uint8_t *>(p_data);

const uint32_t required_align = 32;

while (to_submit > 0) {

uint32_t block_write_offset;

uint32_t block_write_amount;

StagingRequiredAction required_action;

Error err = _staging_buffer_allocate(upload_staging_buffers, MIN(to_submit, upload_staging_buffers.block_size), required_align, block_write_offset, block_write_amount, required_action);

if (err) {

return err;

}

if (!command_buffer_copies_vector.is_empty() && required_action == STAGING_REQUIRED_ACTION_FLUSH_AND_STALL_ALL) {

if (_buffer_make_mutable(buffer, p_buffer)) {

// The buffer must be mutable to be used as a copy destination.

draw_graph.add_synchronization();

}

draw_graph.add_buffer_update(buffer->driver_id, buffer->draw_tracker, command_buffer_copies_vector);

command_buffer_copies_vector.clear();

}

_staging_buffer_execute_required_action(upload_staging_buffers, required_action);

// Copy to staging buffer.

memcpy(upload_staging_buffers.blocks[upload_staging_buffers.current].data_ptr + block_write_offset, src_data + submit_from, block_write_amount);

// Insert a command to copy this.

RDD::BufferCopyRegion region;

region.src_offset = block_write_offset;

region.dst_offset = submit_from + p_offset;

region.size = block_write_amount;

RDG::RecordedBufferCopy buffer_copy;

buffer_copy.source = upload_staging_buffers.blocks[upload_staging_buffers.current].driver_id;

buffer_copy.region = region;

command_buffer_copies_vector.push_back(buffer_copy);

upload_staging_buffers.blocks.write[upload_staging_buffers.current].fill_amount = block_write_offset + block_write_amount;

to_submit -= block_write_amount;

submit_from += block_write_amount;

}

if (!command_buffer_copies_vector.is_empty()) {

if (_buffer_make_mutable(buffer, p_buffer)) {

// The buffer must be mutable to be used as a copy destination.

draw_graph.add_synchronization();

}

draw_graph.add_buffer_update(buffer->driver_id, buffer->draw_tracker, command_buffer_copies_vector);

}

gpu_copy_count++;

return OK;

ERR_RENDER_THREAD_GUARD_V(ERR_UNAVAILABLE);

ERR_FAIL_COND_V_MSG(draw_list.active, ERR_INVALID_PARAMETER,

"Driver callback is forbidden during creation of a draw list.");

ERR_FAIL_COND_V_MSG(compute_list.active, ERR_INVALID_PARAMETER,

"Driver callback is forbidden during creation of a compute list.");

ERR_FAIL_COND_V_MSG(raytracing_list.active, ERR_INVALID_PARAMETER,

"Driver callback is forbidden during creation of a raytracing list.");

thread_local LocalVector<RDG::ResourceTracker *> trackers;

thread_local LocalVector<RDG::ResourceUsage> usages;

uint32_t resource_count = p_resources.size();

trackers.resize(resource_count);

usages.resize(resource_count);

if (resource_count > 0) {

for (uint32_t i = 0; i < p_resources.size(); i++) {

const CallbackResource &cr = p_resources[i];

switch (cr.type) {

case CALLBACK_RESOURCE_TYPE_BUFFER: {

Buffer *buffer = _get_buffer_from_owner(cr.rid);

if (!buffer) {

ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, vformat("Argument %d is not a valid buffer of any type.", i));

}

if (_buffer_make_mutable(buffer, cr.rid)) {

draw_graph.add_synchronization();

}

trackers[i] = buffer->draw_tracker;

usages[i] = (RDG::ResourceUsage)cr.usage;

} break;

case CALLBACK_RESOURCE_TYPE_TEXTURE: {

Texture *texture = texture_owner.get_or_null(cr.rid);

if (!texture) {

ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, vformat("Argument %d is not a valid texture.", i));

}

if (_texture_make_mutable(texture, cr.rid)) {

draw_graph.add_synchronization();

}

trackers[i] = texture->draw_tracker;

usages[i] = (RDG::ResourceUsage)cr.usage;

} break;

default: {

CRASH_NOW_MSG("Invalid callback resource type.");

} break;

}

}

}

draw_graph.add_driver_callback(p_callback, p_userdata, trackers, usages);

return OK;

String perf_report_text;

perf_report_text += " gpu:" + String::num_int64(gpu_copy_count);

perf_report_text += " direct:" + String::num_int64(direct_copy_count);

perf_report_text += " bytes:" + String::num_int64(copy_bytes_count);

perf_report_text += " lazily alloc:" + String::num_int64(driver->get_lazily_memory_used());

return perf_report_text;

prev_gpu_copy_count = gpu_copy_count;

prev_copy_bytes_count = copy_bytes_count;

gpu_copy_count = 0;

direct_copy_count = 0;

copy_bytes_count = 0;

ERR_RENDER_THREAD_GUARD_V(ERR_UNAVAILABLE);

ERR_FAIL_COND_V_MSG((p_size % 4) != 0, ERR_INVALID_PARAMETER,

"Size must be a multiple of four.");

ERR_FAIL_COND_V_MSG(draw_list.active, ERR_INVALID_PARAMETER,

"Updating buffers in is forbidden during creation of a draw list.");

ERR_FAIL_COND_V_MSG(compute_list.active, ERR_INVALID_PARAMETER,

"Updating buffers is forbidden during creation of a compute list.");

ERR_FAIL_COND_V_MSG(raytracing_list.active, ERR_INVALID_PARAMETER,

"Updating buffers is forbidden during creation of a raytracing list.");

Buffer *buffer = _get_buffer_from_owner(p_buffer);

if (!buffer) {

ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, "Buffer argument is not a valid buffer of any type.");

}

ERR_FAIL_COND_V_MSG(p_offset + p_size > buffer->size, ERR_INVALID_PARAMETER,

"Attempted to write buffer (" + itos((p_offset + p_size) - buffer->size) + " bytes) past the end.");

_check_transfer_worker_buffer(buffer);

if (_buffer_make_mutable(buffer, p_buffer)) {

// The destination buffer must be mutable to be used as a clear destination.

draw_graph.add_synchronization();

}

draw_graph.add_buffer_clear(buffer->driver_id, buffer->draw_tracker, p_offset, p_size);

return OK;

ERR_RENDER_THREAD_GUARD_V(Vector<uint8_t>());

Buffer *buffer = _get_buffer_from_owner(p_buffer);

if (!buffer) {

ERR_FAIL_V_MSG(Vector<uint8_t>(), "Buffer is either invalid or this type of buffer can't be retrieved.");

}

// Size of buffer to retrieve.

if (!p_size) {

p_size = buffer->size;

} else {

ERR_FAIL_COND_V_MSG(p_size + p_offset > buffer->size, Vector<uint8_t>(),

"Size is larger than the buffer.");

}

_check_transfer_worker_buffer(buffer);

RDD::BufferID tmp_buffer = driver->buffer_create(buffer->size, RDD::BUFFER_USAGE_TRANSFER_TO_BIT, RDD::MEMORY_ALLOCATION_TYPE_CPU, frames_drawn);

ERR_FAIL_COND_V(!tmp_buffer, Vector<uint8_t>());

RDD::BufferCopyRegion region;

region.src_offset = p_offset;

region.size = p_size;

draw_graph.add_buffer_get_data(buffer->driver_id, buffer->draw_tracker, tmp_buffer, region);

// Flush everything so memory can be safely mapped.

_flush_and_stall_for_all_frames();

uint8_t *buffer_mem = driver->buffer_map(tmp_buffer);

ERR_FAIL_NULL_V(buffer_mem, Vector<uint8_t>());

Vector<uint8_t> buffer_data;

{

buffer_data.resize(p_size);

uint8_t *w = buffer_data.ptrw();

memcpy(w, buffer_mem, p_size);

}

driver->buffer_unmap(tmp_buffer);

driver->buffer_free(tmp_buffer);

return buffer_data;

ERR_RENDER_THREAD_GUARD_V(ERR_UNAVAILABLE);

Buffer *buffer = _get_buffer_from_owner(p_buffer);

if (buffer == nullptr) {

ERR_FAIL_V_MSG(ERR_INVALID_PARAMETER, "Buffer is either invalid or this type of buffer can't be retrieved.");

}

if (p_size == 0) {

p_size = buffer->size;

}

ERR_FAIL_COND_V_MSG(p_size + p_offset > buffer->size, ERR_INVALID_PARAMETER, "Size is larger than the buffer.");

ERR_FAIL_COND_V_MSG(!p_callback.is_valid(), ERR_INVALID_PARAMETER, "Callback must be valid.");

_check_transfer_worker_buffer(buffer);

BufferGetDataRequest get_data_request;

get_data_request.callback = p_callback;

get_data_request.frame_local_index = frames[frame].download_buffer_copy_regions.size();

get_data_request.size = p_size;

const uint32_t required_align = 32;

uint32_t block_write_offset;

uint32_t block_write_amount;

StagingRequiredAction required_action;

uint32_t to_submit = p_size;

uint32_t submit_from = 0;

while (to_submit > 0) {

Error err = _staging_buffer_allocate(download_staging_buffers, MIN(to_submit, download_staging_buffers.block_size), required_align, block_write_offset, block_write_amount, required_action);

if (err) {

return err;

}

const bool flush_frames = (get_data_request.frame_local_count > 0) && required_action == STAGING_REQUIRED_ACTION_FLUSH_AND_STALL_ALL;

if (flush_frames) {

if (_buffer_make_mutable(buffer, p_buffer)) {

// The buffer must be mutable to be used as a copy source.

draw_graph.add_synchronization();

}

for (uint32_t i = 0; i < get_data_request.frame_local_count; i++) {

uint32_t local_index = get_data_request.frame_local_index + i;

draw_graph.add_buffer_get_data(buffer->driver_id, buffer->draw_tracker, frames[frame].download_buffer_staging_buffers[local_index], frames[frame].download_buffer_copy_regions[local_index]);

}

}