diff --git a/tinygrad/runtime/autogen/__init__.py b/tinygrad/runtime/autogen/__init__.py index 187dc7b685..29d29f9b47 100644 --- a/tinygrad/runtime/autogen/__init__.py +++ b/tinygrad/runtime/autogen/__init__.py @@ -40,7 +40,7 @@ def __getattr__(nm): case "cuda": return load("cuda", "'cuda'", ["/usr/include/cuda.h"], args=["-D__CUDA_API_VERSION_INTERNAL"], parse_macros=False) case "nvrtc": return load("nvrtc", "'nvrtc'", ["/usr/include/nvrtc.h"], paths=nv_lib_path, prolog=["import sysconfig"]) case "nvjitlink": load("nvjitlink", "'nvJitLink'", [root/"extra/nvJitLink.h"], paths=nv_lib_path, prolog=["import sysconfig"]) - case "kfd": return load("kfd", None, ["/usr/include/linux/kfd_ioctl.h"]) + case "kfd": return load("kfd", None, [root/"extra/hip_gpu_driver/kfd_ioctl.h"]) case "nv_570" | "nv_580": return load(nm, None, [ *[root/"extra/nv_gpu_driver"/s for s in ["clc9b0.h", "clc6c0qmd.h","clcec0qmd.h", "nvdec_drv.h"]], "{}/kernel-open/common/inc/nvmisc.h", diff --git a/tinygrad/runtime/autogen/kfd.py b/tinygrad/runtime/autogen/kfd.py index 66fbbccf18..c689aec53f 100644 --- a/tinygrad/runtime/autogen/kfd.py +++ b/tinygrad/runtime/autogen/kfd.py @@ -25,6 +25,8 @@ struct_kfd_ioctl_create_queue_args._fields_ = [ ('ctx_save_restore_address', ctypes.c_uint64), ('ctx_save_restore_size', ctypes.c_uint32), ('ctl_stack_size', ctypes.c_uint32), + ('sdma_engine_id', ctypes.c_uint32), + ('pad', ctypes.c_uint32), ] class struct_kfd_ioctl_destroy_queue_args(Struct): pass struct_kfd_ioctl_destroy_queue_args._fields_ = [ @@ -148,6 +150,17 @@ struct_kfd_ioctl_dbg_wave_control_args._fields_ = [ ('gpu_id', ctypes.c_uint32), ('buf_size_in_bytes', ctypes.c_uint32), ] +class struct_kfd_ioctl_dbg_trap_args_deprecated(Struct): pass +struct_kfd_ioctl_dbg_trap_args_deprecated._fields_ = [ + ('exception_mask', ctypes.c_uint64), + ('ptr', ctypes.c_uint64), + ('pid', ctypes.c_uint32), + ('op', ctypes.c_uint32), + ('data1', ctypes.c_uint32), + ('data2', ctypes.c_uint32), + ('data3', ctypes.c_uint32), + ('data4', ctypes.c_uint32), +] class struct_kfd_ioctl_create_event_args(Struct): pass struct_kfd_ioctl_create_event_args._fields_ = [ ('event_page_offset', ctypes.c_uint64), @@ -344,6 +357,21 @@ struct_kfd_ioctl_smi_events_args._fields_ = [ ('gpuid', ctypes.c_uint32), ('anon_fd', ctypes.c_uint32), ] +enum_kfd_ioctl_spm_op = CEnum(ctypes.c_uint32) +KFD_IOCTL_SPM_OP_ACQUIRE = enum_kfd_ioctl_spm_op.define('KFD_IOCTL_SPM_OP_ACQUIRE', 0) +KFD_IOCTL_SPM_OP_RELEASE = enum_kfd_ioctl_spm_op.define('KFD_IOCTL_SPM_OP_RELEASE', 1) +KFD_IOCTL_SPM_OP_SET_DEST_BUF = enum_kfd_ioctl_spm_op.define('KFD_IOCTL_SPM_OP_SET_DEST_BUF', 2) + +class struct_kfd_ioctl_spm_args(Struct): pass +struct_kfd_ioctl_spm_args._fields_ = [ + ('dest_buf', ctypes.c_uint64), + ('buf_size', ctypes.c_uint32), + ('op', ctypes.c_uint32), + ('timeout', ctypes.c_uint32), + ('gpu_id', ctypes.c_uint32), + ('bytes_copied', ctypes.c_uint32), + ('has_data_loss', ctypes.c_uint32), +] enum_kfd_criu_op = CEnum(ctypes.c_uint32) KFD_CRIU_OP_PROCESS_INFO = enum_kfd_criu_op.define('KFD_CRIU_OP_PROCESS_INFO', 0) KFD_CRIU_OP_CHECKPOINT = enum_kfd_criu_op.define('KFD_CRIU_OP_CHECKPOINT', 1) @@ -385,6 +413,32 @@ enum_kfd_mmio_remap = CEnum(ctypes.c_uint32) KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL = enum_kfd_mmio_remap.define('KFD_MMIO_REMAP_HDP_MEM_FLUSH_CNTL', 0) KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL = enum_kfd_mmio_remap.define('KFD_MMIO_REMAP_HDP_REG_FLUSH_CNTL', 4) +class struct_kfd_ioctl_ipc_export_handle_args(Struct): pass +struct_kfd_ioctl_ipc_export_handle_args._fields_ = [ + ('handle', ctypes.c_uint64), + ('share_handle', (ctypes.c_uint32 * 4)), + ('gpu_id', ctypes.c_uint32), + ('flags', ctypes.c_uint32), +] +class struct_kfd_ioctl_ipc_import_handle_args(Struct): pass +struct_kfd_ioctl_ipc_import_handle_args._fields_ = [ + ('handle', ctypes.c_uint64), + ('va_addr', ctypes.c_uint64), + ('mmap_offset', ctypes.c_uint64), + ('share_handle', (ctypes.c_uint32 * 4)), + ('gpu_id', ctypes.c_uint32), + ('flags', ctypes.c_uint32), +] +class struct_kfd_ioctl_cross_memory_copy_deprecated_args(Struct): pass +struct_kfd_ioctl_cross_memory_copy_deprecated_args._fields_ = [ + ('pid', ctypes.c_uint32), + ('flags', ctypes.c_uint32), + ('src_mem_range_array', ctypes.c_uint64), + ('src_mem_array_size', ctypes.c_uint64), + ('dst_mem_range_array', ctypes.c_uint64), + ('dst_mem_array_size', ctypes.c_uint64), + ('bytes_copied', ctypes.c_uint64), +] enum_kfd_ioctl_svm_op = CEnum(ctypes.c_uint32) KFD_IOCTL_SVM_OP_SET_ATTR = enum_kfd_ioctl_svm_op.define('KFD_IOCTL_SVM_OP_SET_ATTR', 0) KFD_IOCTL_SVM_OP_GET_ATTR = enum_kfd_ioctl_svm_op.define('KFD_IOCTL_SVM_OP_GET_ATTR', 1) @@ -451,6 +505,7 @@ KFD_DBG_TRAP_ADDRESS_WATCH_MODE_ALL = enum_kfd_dbg_trap_address_watch_mode.defin enum_kfd_dbg_trap_flags = CEnum(ctypes.c_uint32) KFD_DBG_TRAP_FLAG_SINGLE_MEM_OP = enum_kfd_dbg_trap_flags.define('KFD_DBG_TRAP_FLAG_SINGLE_MEM_OP', 1) +KFD_DBG_TRAP_FLAG_SINGLE_ALU_OP = enum_kfd_dbg_trap_flags.define('KFD_DBG_TRAP_FLAG_SINGLE_ALU_OP', 2) enum_kfd_dbg_trap_exception_code = CEnum(ctypes.c_uint32) EC_NONE = enum_kfd_dbg_trap_exception_code.define('EC_NONE', 0) @@ -656,12 +711,71 @@ struct_kfd_ioctl_dbg_trap_args._fields_ = [ ('op', ctypes.c_uint32), ('_0', struct_kfd_ioctl_dbg_trap_args_0), ] +enum_kfd_ioctl_pc_sample_op = CEnum(ctypes.c_uint32) +KFD_IOCTL_PCS_OP_QUERY_CAPABILITIES = enum_kfd_ioctl_pc_sample_op.define('KFD_IOCTL_PCS_OP_QUERY_CAPABILITIES', 0) +KFD_IOCTL_PCS_OP_CREATE = enum_kfd_ioctl_pc_sample_op.define('KFD_IOCTL_PCS_OP_CREATE', 1) +KFD_IOCTL_PCS_OP_DESTROY = enum_kfd_ioctl_pc_sample_op.define('KFD_IOCTL_PCS_OP_DESTROY', 2) +KFD_IOCTL_PCS_OP_START = enum_kfd_ioctl_pc_sample_op.define('KFD_IOCTL_PCS_OP_START', 3) +KFD_IOCTL_PCS_OP_STOP = enum_kfd_ioctl_pc_sample_op.define('KFD_IOCTL_PCS_OP_STOP', 4) + +enum_kfd_ioctl_pc_sample_method = CEnum(ctypes.c_uint32) +KFD_IOCTL_PCS_METHOD_HOSTTRAP = enum_kfd_ioctl_pc_sample_method.define('KFD_IOCTL_PCS_METHOD_HOSTTRAP', 1) +KFD_IOCTL_PCS_METHOD_STOCHASTIC = enum_kfd_ioctl_pc_sample_method.define('KFD_IOCTL_PCS_METHOD_STOCHASTIC', 2) + +enum_kfd_ioctl_pc_sample_type = CEnum(ctypes.c_uint32) +KFD_IOCTL_PCS_TYPE_TIME_US = enum_kfd_ioctl_pc_sample_type.define('KFD_IOCTL_PCS_TYPE_TIME_US', 0) +KFD_IOCTL_PCS_TYPE_CLOCK_CYCLES = enum_kfd_ioctl_pc_sample_type.define('KFD_IOCTL_PCS_TYPE_CLOCK_CYCLES', 1) +KFD_IOCTL_PCS_TYPE_INSTRUCTIONS = enum_kfd_ioctl_pc_sample_type.define('KFD_IOCTL_PCS_TYPE_INSTRUCTIONS', 2) + +class struct_kfd_pc_sample_info(Struct): pass +struct_kfd_pc_sample_info._fields_ = [ + ('interval', ctypes.c_uint64), + ('interval_min', ctypes.c_uint64), + ('interval_max', ctypes.c_uint64), + ('flags', ctypes.c_uint64), + ('method', ctypes.c_uint32), + ('type', ctypes.c_uint32), +] +class struct_kfd_ioctl_pc_sample_args(Struct): pass +struct_kfd_ioctl_pc_sample_args._fields_ = [ + ('sample_info_ptr', ctypes.c_uint64), + ('num_sample_info', ctypes.c_uint32), + ('op', ctypes.c_uint32), + ('gpu_id', ctypes.c_uint32), + ('trace_id', ctypes.c_uint32), + ('flags', ctypes.c_uint32), + ('version', ctypes.c_uint32), +] +enum_kfd_profiler_ops = CEnum(ctypes.c_uint32) +KFD_IOC_PROFILER_PMC = enum_kfd_profiler_ops.define('KFD_IOC_PROFILER_PMC', 0) +KFD_IOC_PROFILER_PC_SAMPLE = enum_kfd_profiler_ops.define('KFD_IOC_PROFILER_PC_SAMPLE', 1) +KFD_IOC_PROFILER_VERSION = enum_kfd_profiler_ops.define('KFD_IOC_PROFILER_VERSION', 2) + +class struct_kfd_ioctl_pmc_settings(Struct): pass +struct_kfd_ioctl_pmc_settings._fields_ = [ + ('gpu_id', ctypes.c_uint32), + ('lock', ctypes.c_uint32), + ('perfcount_enable', ctypes.c_uint32), +] +class struct_kfd_ioctl_profiler_args(Struct): pass +class struct_kfd_ioctl_profiler_args_0(ctypes.Union): pass +struct_kfd_ioctl_profiler_args_0._fields_ = [ + ('pc_sample', struct_kfd_ioctl_pc_sample_args), + ('pmc', struct_kfd_ioctl_pmc_settings), + ('version', ctypes.c_uint32), +] +struct_kfd_ioctl_profiler_args._anonymous_ = ['_0'] +struct_kfd_ioctl_profiler_args._fields_ = [ + ('op', ctypes.c_uint32), + ('_0', struct_kfd_ioctl_profiler_args_0), +] KFD_IOCTL_MAJOR_VERSION = 1 -KFD_IOCTL_MINOR_VERSION = 14 +KFD_IOCTL_MINOR_VERSION = 17 KFD_IOC_QUEUE_TYPE_COMPUTE = 0x0 KFD_IOC_QUEUE_TYPE_SDMA = 0x1 KFD_IOC_QUEUE_TYPE_COMPUTE_AQL = 0x2 KFD_IOC_QUEUE_TYPE_SDMA_XGMI = 0x3 +KFD_IOC_QUEUE_TYPE_SDMA_BY_ENG_ID = 0x4 KFD_MAX_QUEUE_PERCENTAGE = 100 KFD_MAX_QUEUE_PRIORITY = 15 KFD_IOC_CACHE_POLICY_COHERENT = 0 @@ -705,6 +819,7 @@ KFD_IOC_ALLOC_MEM_FLAGS_AQL_QUEUE_MEM = (1 << 27) KFD_IOC_ALLOC_MEM_FLAGS_COHERENT = (1 << 26) KFD_IOC_ALLOC_MEM_FLAGS_UNCACHED = (1 << 25) KFD_IOC_ALLOC_MEM_FLAGS_EXT_COHERENT = (1 << 24) +KFD_IOC_ALLOC_MEM_FLAGS_CONTIGUOUS = (1 << 23) KFD_SMI_EVENT_MASK_FROM_INDEX = lambda i: (1 << ((i) - 1)) KFD_SMI_EVENT_MSG_SIZE = 96 KFD_IOCTL_SVM_FLAG_HOST_ACCESS = 0x00000001 @@ -731,6 +846,9 @@ KFD_DBG_QUEUE_ERROR_BIT = 30 KFD_DBG_QUEUE_INVALID_BIT = 31 KFD_DBG_QUEUE_ERROR_MASK = (1 << KFD_DBG_QUEUE_ERROR_BIT) KFD_DBG_QUEUE_INVALID_MASK = (1 << KFD_DBG_QUEUE_INVALID_BIT) +KFD_IOCTL_PCS_FLAG_POWER_OF_2 = 0x00000001 +KFD_IOCTL_PCS_QUERY_TYPE_FULL = (1 << 0) +KFD_IOC_PROFILER_VERSION_NUM = 1 AMDKFD_IOCTL_BASE = 'K' AMDKFD_IO = lambda nr: _IO(AMDKFD_IOCTL_BASE, nr) AMDKFD_IOR = lambda nr,type: _IOR(AMDKFD_IOCTL_BASE, nr, type) @@ -775,4 +893,13 @@ AMDKFD_IOC_EXPORT_DMABUF = AMDKFD_IOWR(0x24, struct_kfd_ioctl_export_dmabuf_args AMDKFD_IOC_RUNTIME_ENABLE = AMDKFD_IOWR(0x25, struct_kfd_ioctl_runtime_enable_args) AMDKFD_IOC_DBG_TRAP = AMDKFD_IOWR(0x26, struct_kfd_ioctl_dbg_trap_args) AMDKFD_COMMAND_START = 0x01 -AMDKFD_COMMAND_END = 0x27 \ No newline at end of file +AMDKFD_COMMAND_END = 0x27 +AMDKFD_IOC_IPC_IMPORT_HANDLE = AMDKFD_IOWR(0x80, struct_kfd_ioctl_ipc_import_handle_args) +AMDKFD_IOC_IPC_EXPORT_HANDLE = AMDKFD_IOWR(0x81, struct_kfd_ioctl_ipc_export_handle_args) +AMDKFD_IOC_DBG_TRAP_DEPRECATED = AMDKFD_IOWR(0x82, struct_kfd_ioctl_dbg_trap_args_deprecated) +AMDKFD_IOC_CROSS_MEMORY_COPY_DEPRECATED = AMDKFD_IOWR(0x83, struct_kfd_ioctl_cross_memory_copy_deprecated_args) +AMDKFD_IOC_RLC_SPM = AMDKFD_IOWR(0x84, struct_kfd_ioctl_spm_args) +AMDKFD_IOC_PC_SAMPLE = AMDKFD_IOWR(0x85, struct_kfd_ioctl_pc_sample_args) +AMDKFD_IOC_PROFILER = AMDKFD_IOWR(0x86, struct_kfd_ioctl_profiler_args) +AMDKFD_COMMAND_START_2 = 0x80 +AMDKFD_COMMAND_END_2 = 0x87 \ No newline at end of file