nak works

This commit is contained in:
2025-09-02 09:29:32 -07:00
parent 897254ad6c
commit 629af6aefa
9 changed files with 16830 additions and 32 deletions
+55 -1
View File
@@ -442,6 +442,59 @@ generate_libusb() {
python3 -c "import tinygrad.runtime.autogen.libusb"
}
generate_nak() {
MESA_COMMIT_HASH=9e0991eff5aea2e064fc16d5c7fa0ee6cd52d894
MESA_SRC=/tmp/mesa-$MESA_COMMIT_HASH
if [ ! -d "$MESA_SRC" ]; then
git clone https://gitlab.freedesktop.org/mesa/mesa.git $MESA_SRC
pushd .
cd $MESA_SRC
git reset --hard $MESA_COMMIT_HASH
# clang 14 doesn't support packed enums
sed -i "s/enum \w\+ \(\w\+\);$/uint8_t \1;/" $MESA_SRC/src/nouveau/headers/nv_device_info.h
sed -i "s/enum \w\+ \(\w\+\);$/uint8_t \1;/" $MESA_SRC/src/nouveau/compiler/nak.h
sed -i "s/nir_instr_type \(\w\+\);/uint8_t \1;/" $MESA_SRC/src/compiler/nir/nir.h
mkdir -p gen/util/format
python3 src/util/format/u_format_table.py src/util/format/u_format.yaml --enums > gen/util/format/u_format_gen.h
python3 src/compiler/nir/nir_opcodes_h.py > gen/nir_opcodes.h
python3 src/compiler/nir/nir_intrinsics_h.py --outdir gen
python3 src/compiler/nir/nir_intrinsics_indices_h.py --outdir gen
python3 src/compiler/nir/nir_builder_opcodes_h.py > gen/nir_builder_opcodes.h
python3 src/compiler/nir/nir_intrinsics_h.py --outdir gen
popd
fi
clang2py -k cdefstu \
$MESA_SRC/src/nouveau/headers/nv_device_info.h \
$MESA_SRC/src/nouveau/compiler/nak.h \
--clang-args="-DHAVE_ENDIAN_H -I$MESA_SRC/src -I$MESA_SRC/include -I$MESA_SRC/src/compiler/nir" \
-o $BASE/nak.py
clang2py -v -k cdefstu \
$MESA_SRC/src/compiler/list.h \
$MESA_SRC/src/compiler/nir/nir.h \
$MESA_SRC/src/compiler/nir/nir_builder.h \
$MESA_SRC/src/compiler/nir/nir_shader_compiler_options.h \
$MESA_SRC/gen/nir_intrinsics.h \
--clang-args="-DHAVE_ENDIAN_H -DHAVE_STRUCT_TIMESPEC -DHAVE_PTHREAD -I$MESA_SRC/src -I$MESA_SRC/include -I$MESA_SRC/src/compiler/nir -I$MESA_SRC/gen" \
-o $BASE/nir.py
fixup $BASE/nak.py
fixup $BASE/nir.py
sed -i "s\FunctionFactoryStub()\ctypes.CDLL('/usr/lib/x86_64-linux-gnu/libvulkan_nouveau.so')\g" $BASE/nir.py
sed -i "s\FunctionFactoryStub()\ctypes.CDLL('/usr/lib/x86_64-linux-gnu/libvulkan_nouveau.so')\g" $BASE/nak.py
sed -i "s\import ctypes\import ctypes, os\g" $BASE/nak.py
sed -i "s\import ctypes\import ctypes, os\g" $BASE/nir.py
sed -i "s\'/usr/\os.getenv('MESA_PATH', '/usr/')+'/\g" $BASE/nak.py
sed -i "s\'/usr/\os.getenv('MESA_PATH', '/usr/')+'/\g" $BASE/nir.py
sed -i "s/ctypes.glsl_base_type/glsl_base_type/" $BASE/nir.py
# bitfield bug in clang2py
sed -i "s/('fp_fast_math', ctypes.c_bool, 9)/('fp_fast_math', ctypes.c_uint32, 9)/" $BASE/nir.py
sed -i "s/\([0-9]\+\)()/\1/" $BASE/nir.py
sed -i "s/\(struct_nir_builder._pack_\) = 1/\1 = 0/" $BASE/nir.py
python3 -c "import tinygrad.runtime.autogen.nak, tinygrad.runtime.autogen.nir"
}
if [ "$1" == "opencl" ]; then generate_opencl
elif [ "$1" == "hip" ]; then generate_hip
elif [ "$1" == "comgr" ]; then generate_comgr
@@ -465,6 +518,7 @@ elif [ "$1" == "pci" ]; then generate_pci
elif [ "$1" == "vfio" ]; then generate_vfio
elif [ "$1" == "webgpu" ]; then generate_webgpu
elif [ "$1" == "libusb" ]; then generate_libusb
elif [ "$1" == "all" ]; then generate_opencl; generate_hip; generate_comgr; generate_cuda; generate_nvrtc; generate_hsa; generate_kfd; generate_nv; generate_amd; generate_io_uring; generate_libc; generate_am; generate_webgpu
elif [ "$1" == "nak" ]; then generate_nak
elif [ "$1" == "all" ]; then generate_opencl; generate_hip; generate_comgr; generate_cuda; generate_nvrtc; generate_hsa; generate_kfd; generate_nv; generate_amd; generate_io_uring; generate_libc; generate_am; generate_webgpu; generate_nak
else echo "usage: $0 <type>"
fi
+18
View File
@@ -0,0 +1,18 @@
Make libnir visible:
sed -i '/gnu_symbol_visibility/d' src/compiler/nir/meson.build
Make libcompiler visible:
sed -i '/gnu_symbol_visibility/d' src/compiler/meson.build
Fix rpath:
patchelf --add-rpath $MESA_PREFIX/lib/x86_64-linux-gnu $MESA_PREFIX/lib/x86_64-linux-gnu/libvulkan_nouveau.so
LD_LIBRARY_PATH=$MESA_PREFIX/lib/x86_64-linux-gnu OCL_ICD_VENDORS="$MESA_PREFIX/etc/OpenCL/vendors/" \
RUSTICL_ENABLE=swrast RUSTICL_DEBUG=nir MESA_SHADER_CACHE_DISABLE=true DISABLE_COMPILER_CACHE=1 GPU=1 \
python3 -c "from tinygrad import Tensor; print((Tensor.eye(1,1) * 2).numpy())"
Generate NIR options for NAK:
cc nak_nir_options.c -o nak_nir_options \
-DHAVE_ENDIAN_H -DHAVE_PTHREAD -DHAVE_STRUCT_TIMESPEC \
-I$MESA_SRC/src -I$MESA_SRC/include -I$MESA_SRC/gen
./nak_nir_options | gzip | base64 -w 0
+177
View File
@@ -0,0 +1,177 @@
from __future__ import annotations
from tinygrad import Tensor, dtypes
import tinygrad.runtime.autogen.nak as nak
import tinygrad.runtime.autogen.nir as nir
import tinygrad.runtime.autogen.libc as libc
from tinygrad.runtime.ops_nv import NVDevice, NVProgram
import ctypes, os
stdout = ctypes.POINTER(nir.struct__IO_FILE).in_dll(libc._libraries['libc'], "stdout")
nir_intrinsic_infos = nir.nir_intrinsic_infos.in_dll(nir._libraries['FIXME_STUB'], "nir_intrinsic_infos")
glsl_type_builtin_uint64_t = nir.struct_glsl_type.in_dll(nir._libraries['FIXME_STUB'], "glsl_type_builtin_uint64_t")
def BITFIELD_BIT(b): return 1 << b
def BITFIELD_MASK(b): return 0xFFFFFFFF if b == 32 else BITFIELD_BIT(b & 31) - 1
"""
struct nak_shader_bin *
nak_compile_shader(nir_shader *nir, bool dump_asm,
const struct nak_compiler *nak,
nir_variable_mode robust2_modes,
const struct nak_fs_key *fs_key);
nir_builder nir_builder_init_simple_shader(
mesa_shader_stage stage, // MESA_SHADER_COMPUTE?
const nir_shader_compiler_options *options, // {}
const char *name, ...);
https://elixir.bootlin.com/mesa/mesa-25.2.0/source/src/nouveau/vulkan/nvk_shader.c#L479
"""
# https://elixir.bootlin.com/mesa/mesa-25.2.0/source/src/compiler/glsl_types.h#L172
def glsl_base_type_get_bit_size(base_type: nir.glsl_base_type) -> int:
return {
nir.GLSL_TYPE_BOOL : 1,
nir.GLSL_TYPE_INT : 32, nir.GLSL_TYPE_UINT : 32, nir.GLSL_TYPE_FLOAT : 32, nir.GLSL_TYPE_SUBROUTINE : 32, nir.GLSL_TYPE_COOPERATIVE_MATRIX : 32,
nir.GLSL_TYPE_FLOAT16 : 16, nir.GLSL_TYPE_BFLOAT16 : 16, nir.GLSL_TYPE_UINT16: 16, nir.GLSL_TYPE_INT16 : 16,
nir.GLSL_TYPE_UINT8 : 8, nir.GLSL_TYPE_INT8 : 8, nir.GLSL_TYPE_FLOAT_E4M3FN : 8, nir.GLSL_TYPE_FLOAT_E5M2 : 8,
nir.GLSL_TYPE_DOUBLE : 64, nir.GLSL_TYPE_INT64 : 64, nir.GLSL_TYPE_UINT64 : 64, nir.GLSL_TYPE_IMAGE : 64, nir.GLSL_TYPE_SAMPLER : 64, nir.GLSL_TYPE_TEXTURE : 64,
}[int(base_type)]
def show_layout(struct):
print(struct)
for k, _ in struct._fields_: print(f" {k}: 0x{getattr(struct, k).offset:X}")
def nir_src_for_ssa(d):
src = nir.nir_src()
src.ssa = d
print(d, src.ssa.contents)
return src
def nir_intrinsic_set(typ, instr, val):
info = nir_intrinsic_infos[instr.contents.intrinsic]
assert info.index_map[typ] > 0
instr.contents.const_index[info.index_map[typ] - 1] = val
def nir_intrinsic_set_access(instr, val): nir_intrinsic_set(nir.NIR_INTRINSIC_ACCESS, instr, val)
def nir_intrinsic_set_param_idx(instr, val): nir_intrinsic_set(nir.NIR_INTRINSIC_PARAM_IDX, instr, val)
def nir_intrinsic_set_write_mask(instr, val): nir_intrinsic_set(nir.NIR_INTRINSIC_WRITE_MASK, instr, val)
def nir_intrinsic_set_align_mul(instr, val): nir_intrinsic_set(nir.NIR_INTRINSIC_ALIGN_MUL, instr, val)
def nir_intrinsic_set_align_offset(instr, val): nir_intrinsic_set(nir.NIR_INTRINSIC_ALIGN_OFFSET, instr, val)
def nir_intrinsic_set_align(instr, mul, off):
assert off < mul
nir_intrinsic_set_align_mul(instr, mul)
nir_intrinsic_set_align_offset(instr, off)
def nir_intrinsic_set_range_base(instr, val): nir_intrinsic_set(nir.NIR_INTRINSIC_RANGE_BASE, instr, val)
def nir_intrinsic_set_range(instr, val): nir_intrinsic_set(nir.NIR_INTRINSIC_RANGE, instr, val)
def nir_build_load_param(b:nir.nir_builder, num:int, bit_sz:int, idx:int) -> ctypes._Pointer[nir.nir_def]:
intrin = nir.nir_intrinsic_instr_create(b.shader, nir.nir_intrinsic_load_param)
intrin.contents.num_components = num
nir.nir_def_init(intrin.contents.instr, getattr(intrin.contents, "def"), intrin.contents.num_components, bit_sz)
nir_intrinsic_set_param_idx(intrin, idx)
nir.nir_builder_instr_insert(b, intrin.contents.instr)
return ctypes.pointer(getattr(intrin.contents, "def"))
def nir_load_param(b:nir.nir_builder, idx:int) -> ctypes._Pointer[nir.nir_def]:
assert idx < b.impl.contents.function.contents.num_params
param = b.impl.contents.function.contents.params[idx]
return nir_build_load_param(b, param.num_components, param.bit_size, idx)
def nir_build_deref_var(b:nir.nir_builder, var:ctypes._Pointer[nir.nir_variable]) -> ctypes._Pointer[nir.nir_deref_instr]:
deref = nir.nir_deref_instr_create(b.shader, nir.nir_deref_type_var)
deref.contents.modes, deref.contents.type, deref.contents.var = var.contents.data.mode, var.contents.type, var
nir.nir_def_init(deref.contents.instr, getattr(deref.contents, "def"), 1, b.shader.contents.info.cs.ptr_size)
nir.nir_builder_instr_insert(b, deref.contents.instr)
return deref
def nir_build_load_deref(b:nir.nir_builder, num:int, bit_sz:int, src0:ctypes._Pointer[nir.nir_def], access:int) -> ctypes._Pointer[nir.nir_def]:
intrin = nir.nir_intrinsic_instr_create(b.shader, nir.nir_intrinsic_load_deref)
intrin.contents.num_components = num
nir.nir_def_init(intrin.contents.instr, getattr(intrin.contents, "def"), intrin.contents.num_components, bit_sz)
ctypes.cast(intrin.contents.src, ctypes.POINTER(nir.nir_src))[0] = nir_src_for_ssa(src0)
nir_intrinsic_set_access(intrin, access)
nir.nir_builder_instr_insert(b, intrin.contents.instr)
return ctypes.pointer(getattr(intrin.contents, "def"))
def nir_load_deref(b:nir.nir_builder, d:ctypes._Pointer[nir.nir_deref_instr]) -> ctypes._Pointer[nir.nir_def]:
return nir_build_load_deref(b, d.contents.type.contents.vector_elements, glsl_base_type_get_bit_size(d.contents.type.contents.base_type), ctypes.pointer(getattr(d.contents, "def")), 0)
def nir_load_ubo(b:nir.nir_builder, num:int, bit_sz:int, src0:ctypes._Pointer[nir.nir_def], src1:ctypes._Pointer[nir.nir_def],
access:int=0, align_mul:int=0, align_offset:int=0, range_base:int=0, range_:int=0) -> ctypes._Pointer[nir.nir_def]:
intrin = nir.nir_intrinsic_instr_create(b.shader, nir.nir_intrinsic_load_ubo)
intrin.contents.num_components = num
nir.nir_def_init(intrin.contents.instr, getattr(intrin.contents, "def"), intrin.contents.num_components, bit_sz)
arr = ctypes.cast(intrin.contents.src, ctypes.POINTER(nir.nir_src))
arr[0], arr[1] = nir_src_for_ssa(src0), nir_src_for_ssa(src1)
nir_intrinsic_set_access(intrin, access)
nir_intrinsic_set_align(intrin, align_mul if align_mul else getattr(intrin.contents, "def").bit_size // 8, align_offset)
nir_intrinsic_set_range_base(intrin, range_base)
nir_intrinsic_set_range(intrin, range_)
nir.nir_builder_instr_insert(b, intrin.contents.instr)
return ctypes.pointer(getattr(intrin.contents, "def"))
def nir_ldc_nv(b:nir.nir_builder, num:int, bit_sz:int, src0:ctypes._Pointer[nir.nir_def], src1:ctypes._Pointer[nir.nir_def],
access:int=0, align_mul:int=0, align_offset:int=0) -> ctypes._Pointer[nir.nir_def]:
intrin = nir.nir_intrinsic_instr_create(b.shader, nir.nir_intrinsic_ldc_nv)
intrin.contents.num_components = num
nir.nir_def_init(intrin.contents.instr, getattr(intrin.contents, "def"), intrin.contents.num_components, bit_sz)
arr = ctypes.cast(intrin.contents.src, ctypes.POINTER(nir.nir_src))
arr[0], arr[1] = nir_src_for_ssa(src0), nir_src_for_ssa(src1)
nir_intrinsic_set_access(intrin, access)
nir_intrinsic_set_align(intrin, align_mul if align_mul else getattr(intrin.contents, "def").bit_size // 8, align_offset)
nir.nir_builder_instr_insert(b, intrin.contents.instr)
return ctypes.pointer(getattr(intrin.contents, "def"))
def nir_iadd(b:nir.nir_builder, src0:nir.nir_def, src1:nir.nir_def) -> nir.nir_def: return nir.nir_build_alu2(b, nir.nir_op_iadd, src0, src1)
def nir_imm_int(b:nir.nir_builder, x:int) -> ctypes._Pointer[nir.nir_def]:
load_const = nir.nir_load_const_instr_create(b.shader, 1, 32)
ctypes.cast(load_const.contents.value, ctypes.POINTER(ctypes.c_int))[0] = x
nir.nir_builder_instr_insert(b, load_const.contents.instr)
return ctypes.pointer(getattr(load_const.contents, 'def'))
def nir_store_global(b:nir.nir_builder, addr:ctypes._Pointer[nir.nir_def], align:int, value:ctypes._Pointer[nir.nir_def], write_mask:int):
store = nir.nir_intrinsic_instr_create(b.shader, nir.nir_intrinsic_store_global)
store.contents.num_components = value.contents.num_components
arr = ctypes.cast(store.contents.src, ctypes.POINTER(nir.nir_src))
arr[0], arr[1] = nir_src_for_ssa(value), nir_src_for_ssa(addr)
nir_intrinsic_set_write_mask(store, write_mask & BITFIELD_MASK(value.contents.num_components))
nir_intrinsic_set_align(store, align, 0)
nir.nir_builder_instr_insert(b, store.contents.instr)
input(f"pid: {os.getpid()}. press enter to continue...")
dev = NVDevice()
b = nir.nir_builder_init_simple_shader(nir.MESA_SHADER_COMPUTE, nir.nir_shader_compiler_options(), None)
data0 = nir_ldc_nv(b, 1, 64, nir_imm_int(b, 0), nir_imm_int(b, 0x160))
nir_store_global(b, data0, 4, nir_imm_int(b, 1337), ~0)
"""
param = nir.nir_parameter(1, 64, False, type=ctypes.pointer(glsl_type_builtin_uint64_t), name=ctypes.create_string_buffer(b"hi"))
b.impl.contents.function.contents.num_params = 1
b.impl.contents.function.contents.params = ctypes.pointer(param)
data0 = nir_load_param(b, 0)
nir_store_global(b, data0, 4, nir_imm_int(b, 1), ~0)
"""
nir.nir_print_shader(b.shader, stdout)
cc = nak.nak_compiler_create(nak.struct_nv_device_info(sm=int(dev.arch[3:]), max_warps_per_mp=dev.max_warps_per_sm))
nak.nak_preprocess_nir(ctypes.cast(b.shader, ctypes.POINTER(nak.struct_nir_shader)), cc)
nir.nir_print_shader(b.shader, stdout)
out = nak.nak_compile_shader(ctypes.cast(b.shader, ctypes.POINTER(nak.struct_nir_shader)), True, cc, 0, None)
print(ctypes.string_at(out.contents.asm_str).decode())
if input("write to file? (y/n) ") == "y":
with open("out.cubin", "wb") as f: f.write(ctypes.string_at(out.contents.code, out.contents.code_size))
print(f"""info:
gprs: 0x{out.contents.info.num_gprs:X}""")
if b.shader.contents.constant_data_size > 0: print("constant data!")
prog = NVProgram(dev, "fxn", bytearray(ctypes.string_at(out.contents.code, out.contents.code_size)), raw=True, regs_usage=out.contents.info.num_gprs)
a = dev.allocator.alloc(4)
prog(a, wait=True)
a_out = bytearray(4)
dev.allocator._copyout(memoryview(a_out), a)
import struct
print(struct.unpack("I", a_out))
+185
View File
@@ -0,0 +1,185 @@
/*
* Copyright 2017 Red Hat Inc.
*
* Permission is hereby granted, free of charge, to any person obtaining a
* copy of this software and associated documentation files (the "Software"),
* to deal in the Software without restriction, including without limitation
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
* and/or sell copies of the Software, and to permit persons to whom the
* Software is furnished to do so, subject to the following conditions:
*
* The above copyright notice and this permission notice shall be included in
* all copies or substantial portions of the Software.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
* THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
* OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
* ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
* OTHER DEALINGS IN THE SOFTWARE.
*
* Authors: Karol Herbst <[email protected]>
*/
#define NVISA_G80_CHIPSET 0x50
#define NVISA_GF100_CHIPSET 0xc0
#define NVISA_GK104_CHIPSET 0xe0
#define NVISA_GK20A_CHIPSET 0xea
#define NVISA_GM107_CHIPSET 0x110
#define NVISA_GM200_CHIPSET 0x120
#define NVISA_GV100_CHIPSET 0x140
#include "compiler/nir/nir.h"
#include "compiler/nir/nir_builder.h"
/*
#include "util/u_debug.h"
#include "util/u_prim.h"
*/
/*
#include "nv50_ir.h"
#include "nv50_ir_lowering_helper.h"
#include "nv50_ir_target.h"
#include "nv50_ir_util.h"
#include "tgsi/tgsi_from_mesa.h"
*/
static nir_shader_compiler_options
nvir_nir_shader_compiler_options(int chipset, uint8_t shader_type)
{
nir_shader_compiler_options op = {};
op.lower_fdiv = (chipset >= NVISA_GV100_CHIPSET);
op.lower_ffma16 = false;
op.lower_ffma32 = false;
op.lower_ffma64 = false;
op.fuse_ffma16 = false; /* nir doesn't track mad vs fma */
op.fuse_ffma32 = false; /* nir doesn't track mad vs fma */
op.fuse_ffma64 = false; /* nir doesn't track mad vs fma */
op.lower_flrp16 = (chipset >= NVISA_GV100_CHIPSET);
op.lower_flrp32 = true;
op.lower_flrp64 = true;
op.lower_fpow = true;
op.lower_fsat = false;
op.lower_fsqrt = false; // TODO: only before gm200
op.lower_sincos = false;
op.lower_fmod = true;
op.lower_bitfield_extract = (chipset >= NVISA_GV100_CHIPSET || chipset < NVISA_GF100_CHIPSET);
op.lower_bitfield_insert = (chipset >= NVISA_GV100_CHIPSET || chipset < NVISA_GF100_CHIPSET);
op.lower_bitfield_reverse = (chipset < NVISA_GF100_CHIPSET);
op.lower_bit_count = (chipset < NVISA_GF100_CHIPSET);
op.lower_ifind_msb = (chipset < NVISA_GF100_CHIPSET);
op.lower_find_lsb = (chipset < NVISA_GF100_CHIPSET);
op.lower_uadd_carry = true; // TODO
op.lower_usub_borrow = true; // TODO
op.lower_mul_high = false;
op.lower_fneg = false;
op.lower_ineg = false;
op.lower_scmp = true; // TODO: not implemented yet
op.lower_vector_cmp = false;
op.lower_bitops = false;
op.lower_isign = (chipset >= NVISA_GV100_CHIPSET);
op.lower_fsign = (chipset >= NVISA_GV100_CHIPSET);
op.lower_fdph = false;
op.fdot_replicates = false; // TODO
op.lower_ffloor = false; // TODO
op.lower_ffract = true;
op.lower_fceil = false; // TODO
op.lower_ftrunc = false;
op.lower_ldexp = true;
op.lower_pack_half_2x16 = true;
op.lower_pack_unorm_2x16 = true;
op.lower_pack_snorm_2x16 = true;
op.lower_pack_unorm_4x8 = true;
op.lower_pack_snorm_4x8 = true;
op.lower_unpack_half_2x16 = true;
op.lower_unpack_unorm_2x16 = true;
op.lower_unpack_snorm_2x16 = true;
op.lower_unpack_unorm_4x8 = true;
op.lower_unpack_snorm_4x8 = true;
op.lower_pack_split = false;
op.lower_extract_byte = (chipset < NVISA_GM107_CHIPSET);
op.lower_extract_word = (chipset < NVISA_GM107_CHIPSET);
op.lower_insert_byte = true;
op.lower_insert_word = true;
op.vertex_id_zero_based = false;
op.lower_base_vertex = false;
op.lower_helper_invocation = false;
op.optimize_sample_mask_in = false;
op.lower_cs_local_index_to_id = true;
op.lower_cs_local_id_to_index = false;
op.lower_device_index_to_zero = true;
op.lower_wpos_pntc = false; // TODO
op.lower_hadd = true; // TODO
op.lower_uadd_sat = true; // TODO
op.lower_usub_sat = true; // TODO
op.lower_iadd_sat = true; // TODO
op.lower_to_scalar = false;
op.unify_interfaces = false;
op.lower_mul_2x32_64 = true; // TODO
op.has_rotate32 = (chipset >= NVISA_GV100_CHIPSET);
op.has_imul24 = false;
op.has_fmulz = (chipset > NVISA_G80_CHIPSET);
op.intel_vec4 = false;
op.lower_uniforms_to_ubo = true;
op.force_indirect_unrolling = (nir_variable_mode) (
((shader_type == MESA_SHADER_FRAGMENT) ? nir_var_shader_out : 0) |
/* HW doesn't support indirect addressing of fragment program inputs
* on Volta. The binary driver generates a function to handle every
* possible indirection, and indirectly calls the function to handle
* this instead.
*/
((chipset >= NVISA_GV100_CHIPSET && shader_type == MESA_SHADER_FRAGMENT) ? nir_var_shader_in : 0)
);
op.force_indirect_unrolling_sampler = (chipset < NVISA_GF100_CHIPSET);
op.max_unroll_iterations = 32;
op.lower_int64_options = (nir_lower_int64_options) (
((chipset >= NVISA_GV100_CHIPSET) ? nir_lower_imul64 : 0) |
((chipset >= NVISA_GV100_CHIPSET) ? nir_lower_isign64 : 0) |
nir_lower_divmod64 |
((chipset >= NVISA_GV100_CHIPSET) ? nir_lower_imul_high64 : 0) |
((chipset >= NVISA_GV100_CHIPSET) ? nir_lower_bcsel64 : 0) |
((chipset >= NVISA_GV100_CHIPSET) ? nir_lower_icmp64 : 0) |
((chipset >= NVISA_GV100_CHIPSET) ? nir_lower_iabs64 : 0) |
((chipset >= NVISA_GV100_CHIPSET) ? nir_lower_ineg64 : 0) |
((chipset >= NVISA_GV100_CHIPSET) ? nir_lower_logic64 : 0) |
((chipset >= NVISA_GV100_CHIPSET) ? nir_lower_minmax64 : 0) |
((chipset >= NVISA_GV100_CHIPSET) ? nir_lower_shift64 : 0) |
nir_lower_imul_2x32_64 |
((chipset >= NVISA_GM107_CHIPSET) ? nir_lower_extract64 : 0) |
nir_lower_ufind_msb64 |
((chipset >= NVISA_GV100_CHIPSET) ? nir_lower_conv64 : 0)
);
op.lower_doubles_options = (nir_lower_doubles_options) (
((chipset >= NVISA_GV100_CHIPSET) ? nir_lower_drcp : 0) |
((chipset >= NVISA_GV100_CHIPSET) ? nir_lower_dsqrt : 0) |
((chipset >= NVISA_GV100_CHIPSET) ? nir_lower_drsq : 0) |
((chipset >= NVISA_GV100_CHIPSET) ? nir_lower_dfract : 0) |
nir_lower_dmod |
((chipset >= NVISA_GV100_CHIPSET) ? nir_lower_dsub : 0) |
((chipset >= NVISA_GV100_CHIPSET) ? nir_lower_ddiv : 0)
);
op.discard_is_demote = true;
op.has_ddx_intrinsics = true;
op.scalarize_ddx = true;
op.support_indirect_inputs = (uint8_t)BITFIELD_MASK(MESA_SHADER_GEOMETRY + 1);
op.support_indirect_outputs = (uint8_t)BITFIELD_MASK(MESA_SHADER_GEOMETRY + 1);
/* HW doesn't support indirect addressing of fragment program inputs
* on Volta. The binary driver generates a function to handle every
* possible indirection, and indirectly calls the function to handle
* this instead.
*/
if (chipset < NVISA_GV100_CHIPSET)
op.support_indirect_outputs |= BITFIELD_BIT(MESA_SHADER_FRAGMENT);
return op;
}
int main(void) {
fprintf(stderr, "size: %ld\n", sizeof(nir_shader_compiler_options ));
nir_shader_compiler_options ops = nvir_nir_shader_compiler_options(NVISA_GV100_CHIPSET, MESA_SHADER_COMPUTE);
return write(1, &ops, sizeof(nir_shader_compiler_options));
}
+116
View File
@@ -0,0 +1,116 @@
from typing import cast
from tinygrad.dtype import DType, PtrDType, dtypes
from tinygrad.renderer import Renderer
from tinygrad.renderer.cstyle import CUDARenderer
from tinygrad.uop.ops import Ops, UOp, PatternMatcher, UPat
import tinygrad.runtime.autogen.nir as nir
import tinygrad.runtime.autogen.libc as libc
from tinygrad.runtime.support.nak import nak_nir_options
import ctypes, struct
# FIXME: this is because clang2py produces bad output for hidden symbols
nir_intrinsic_infos = nir.nir_intrinsic_infos.in_dll(nir._libraries['FIXME_STUB'], "nir_intrinsic_infos")
stdout = ctypes.POINTER(nir.struct__IO_FILE).in_dll(libc._libraries['libc'], "stdout")
def BITFIELD_BIT(b): return 1 << b
def BITFIELD_MASK(b): return 0xFFFFFFFF if b == 32 else BITFIELD_BIT(b & 31) - 1
# TODO: @functools.cache
def nir_imm(b:nir.nir_builder, x, dtype:DType) -> nir.nir_def:
assert dtype.fmt
instr = nir.nir_load_const_instr_create(b.shader, 1, dtype.itemsize * 8)
struct.pack_into(dtype.fmt, (ctypes.c_ubyte * dtype.itemsize).from_address(ctypes.addressof(instr.contents.value)), 0, x)
nir.nir_builder_instr_insert(b, instr.contents.instr)
return getattr(instr.contents, "def")
def nir_src_for_ssa(d:nir.nir_def) -> nir.nir_src: return nir.nir_src(ssa=ctypes.pointer(d))
def nir_intrinsic_set(typ, instr:nir.nir_intrinsic_instr, val:int):
info = nir_intrinsic_infos[instr.contents.intrinsic]
assert info.index_map[typ] > 0
instr.contents.const_index[info.index_map[typ] - 1] = val
def nir_build_alu(b:nir.nir_builder, op, *srcs:list[nir.nir_def]) -> nir.nir_def:
if len(srcs) == 1: return nir.nir_build_alu1(b, op, srcs[0]).contents
if len(srcs) == 2: return nir.nir_build_alu2(b, op, srcs[0], srcs[1]).contents
if len(srcs) == 3: return nir.nir_build_alu3(b, op, srcs[0], srcs[1], srcs[2]).contents
return nir.nir_build_alu4(b, op, srcs[0], srcs[1], srcs[2], srcs[3]).contents
def nir_store_global(b:nir.nir_builder, addr:nir.nir_def, value:nir.nir_def, write_mask:int):
store = nir.nir_intrinsic_instr_create(b.shader, nir.nir_intrinsic_store_global)
store.contents.num_components = value.num_components # is this right?
arr = ctypes.cast(store.contents.src, ctypes.POINTER(nir.nir_src))
arr[0], arr[1] = nir_src_for_ssa(value), nir_src_for_ssa(addr)
# TODO: think about what these should be set to
nir_intrinsic_set(nir.NIR_INTRINSIC_WRITE_MASK, store, write_mask & BITFIELD_MASK(value.num_components))
nir_intrinsic_set(nir.NIR_INTRINSIC_ALIGN_MUL, store, 4)
nir_intrinsic_set(nir.NIR_INTRINSIC_ALIGN_OFFSET, store, 0) # is setting to zero the default case?
nir.nir_builder_instr_insert(b, store.contents.instr)
return nir.nir_def() # FIXME!
class NIRRenderer(Renderer):
device = "NV"
suffix = "NAK"
global_max, local_max, shared_max = CUDARenderer.global_max, CUDARenderer.local_max, CUDARenderer.shared_max
extra_matcher = PatternMatcher([
(UPat(Ops.INDEX, src=(UPat.var("buf"), UPat(Ops.CONST, dtype=dtypes.int, name="idx"))),
lambda buf,idx: UOp(Ops.INDEX, src=(buf, UOp(Ops.CONST, dtype=dtypes.long, arg=idx.arg))))
])
def_rewrite = PatternMatcher([
(UPat(Ops.INDEX, src=(UPat.var("buf"), UPat.var('idx')), allow_any_len=True),
lambda ctx,buf,idx: nir_build_alu(ctx[0], nir.nir_op_iadd, ctx[1][buf], ctx[1][idx])),
# TODO: local store (a la ptx's mem_types)
(UPat(Ops.STORE, src=(UPat.var("addr"), UPat.var("val"))),
lambda ctx,addr,val: nir_store_global(ctx[0], ctx[1][addr], ctx[1][val], ~0))
])
def __init__(self, arch:str, device="NV"): self.device, self.arch = device, arch
def param(self, b:nir.nir_builder, dtype:DType, idx:int) -> nir.nir_def:
intrin = nir.nir_intrinsic_instr_create(b.shader, nir.nir_intrinsic_ldc_nv)
intrin.contents.num_components = 1
nir.nir_def_init(intrin.contents.instr, getattr(intrin.contents, "def"), 1, 64 if isinstance(dtype, PtrDType) else dtype.itemsize * 8)
arr = ctypes.cast(intrin.contents.src, ctypes.POINTER(nir.nir_src))
# is this the right offset?
arr[0], arr[1] = nir_src_for_ssa(nir_imm(b, 0, dtypes.int)), nir_src_for_ssa(nir_imm(b, 0x160 + idx * 8, dtypes.int))
# TODO: are these values correct?
nir_intrinsic_set(nir.NIR_INTRINSIC_ACCESS, intrin, 0)
nir_intrinsic_set(nir.NIR_INTRINSIC_ALIGN_MUL, intrin, getattr(intrin.contents, "def").bit_size // 8)
nir_intrinsic_set(nir.NIR_INTRINSIC_ALIGN_OFFSET, intrin, 0)
nir.nir_builder_instr_insert(b, intrin.contents.instr)
return getattr(intrin.contents, "def")
def render(self, uops:list[UOp]) -> str:
b = nir.nir_builder_init_simple_shader(nir.MESA_SHADER_COMPUTE, nak_nir_options, None)
r: dict[UOp, nir.nir_def] = {}
args: list[tuple[nir.nir_def, DType]] = []
for u in uops:
nir.nir_print_shader(b.shader, stdout)
if u.op is Ops.NOOP: continue
if u.op is Ops.SINK:
# TODO: https://elixir.bootlin.com/mesa/mesa-25.2.1/source/src/compiler/nir/nir_builder.c#L33
if u.arg is not None: pass
continue
if u.op in (Ops.DEFINE_GLOBAL, Ops.DEFINE_VAR):
assert u.op is Ops.DEFINE_GLOBAL
r[u] = self.param(b, u.dtype, u.arg)
args.append((r[u], u.dtype))
elif u.op in (Ops.DEFINE_LOCAL, Ops.DEFINE_REG): raise NotImplementedError("DEFINE_LOCAL/REG")
elif u.op is Ops.CONST: r[u] = nir_imm(b, u.arg, u.dtype)
else:
print(u)
if (d:=self.def_rewrite.rewrite(u, ctx=(b,r))) is None:
nir.nir_print_shader(b.shader, stdout)
raise RuntimeError(f"failed to render {u.op} with {u.dtype} srcs {[x.dtype for x in u.src]}")
r[u] = cast(nir.nir_def, d)
print(b.shader.contents)
import tinygrad.runtime.autogen.nak as nak
import os
input(f"pid: {os.getpid()}")
cc = nak.nak_compiler_create(nak.struct_nv_device_info(sm=86, max_warps_per_mp=48))
nak.nak_preprocess_nir(nak.struct_nir_shader.from_buffer(b.shader.contents), cc)
out = nak.nak_compile_shader(ctypes.cast(b.shader, ctypes.POINTER(nak.struct_nir_shader)), True, cc, 0, None)
print(ctypes.string_at(out.contents.asm_str).decode())
return b.shader.contents
+663
View File
@@ -0,0 +1,663 @@
# mypy: ignore-errors
# -*- coding: utf-8 -*-
#
# TARGET arch is: ['-DHAVE_ENDIAN_H', '-I/tmp/mesa-9e0991eff5aea2e064fc16d5c7fa0ee6cd52d894/src', '-I/tmp/mesa-9e0991eff5aea2e064fc16d5c7fa0ee6cd52d894/include', '-I/tmp/mesa-9e0991eff5aea2e064fc16d5c7fa0ee6cd52d894/src/compiler/nir']
# WORD_SIZE is: 8
# POINTER_SIZE is: 8
# LONGDOUBLE_SIZE is: 16
#
import ctypes, os
class AsDictMixin:
@classmethod
def as_dict(cls, self):
result = {}
if not isinstance(self, AsDictMixin):
# not a structure, assume it's already a python object
return self
if not hasattr(cls, "_fields_"):
return result
# sys.version_info >= (3, 5)
# for (field, *_) in cls._fields_: # noqa
for field_tuple in cls._fields_: # noqa
field = field_tuple[0]
if field.startswith('PADDING_'):
continue
value = getattr(self, field)
type_ = type(value)
if hasattr(value, "_length_") and hasattr(value, "_type_"):
# array
if not hasattr(type_, "as_dict"):
value = [v for v in value]
else:
type_ = type_._type_
value = [type_.as_dict(v) for v in value]
elif hasattr(value, "contents") and hasattr(value, "_type_"):
# pointer
try:
if not hasattr(type_, "as_dict"):
value = value.contents
else:
type_ = type_._type_
value = type_.as_dict(value.contents)
except ValueError:
# nullptr
value = None
elif isinstance(value, AsDictMixin):
# other structure
value = type_.as_dict(value)
result[field] = value
return result
class Structure(ctypes.Structure, AsDictMixin):
def __init__(self, *args, **kwds):
# We don't want to use positional arguments fill PADDING_* fields
args = dict(zip(self.__class__._field_names_(), args))
args.update(kwds)
super(Structure, self).__init__(**args)
@classmethod
def _field_names_(cls):
if hasattr(cls, '_fields_'):
return (f[0] for f in cls._fields_ if not f[0].startswith('PADDING'))
else:
return ()
@classmethod
def get_type(cls, field):
for f in cls._fields_:
if f[0] == field:
return f[1]
return None
@classmethod
def bind(cls, bound_fields):
fields = {}
for name, type_ in cls._fields_:
if hasattr(type_, "restype"):
if name in bound_fields:
if bound_fields[name] is None:
fields[name] = type_()
else:
# use a closure to capture the callback from the loop scope
fields[name] = (
type_((lambda callback: lambda *args: callback(*args))(
bound_fields[name]))
)
del bound_fields[name]
else:
# default callback implementation (does nothing)
try:
default_ = type_(0).restype().value
except TypeError:
default_ = None
fields[name] = type_((
lambda default_: lambda *args: default_)(default_))
else:
# not a callback function, use default initialization
if name in bound_fields:
fields[name] = bound_fields[name]
del bound_fields[name]
else:
fields[name] = type_()
if len(bound_fields) != 0:
raise ValueError(
"Cannot bind the following unknown callback(s) {}.{}".format(
cls.__name__, bound_fields.keys()
))
return cls(**fields)
class Union(ctypes.Union, AsDictMixin):
pass
c_int128 = ctypes.c_ubyte*16
c_uint128 = c_int128
void = None
if ctypes.sizeof(ctypes.c_longdouble) == 16:
c_long_double_t = ctypes.c_longdouble
else:
c_long_double_t = ctypes.c_ubyte*16
def string_cast(char_pointer, encoding='utf-8', errors='strict'):
value = ctypes.cast(char_pointer, ctypes.c_char_p).value
if value is not None and encoding is not None:
value = value.decode(encoding, errors=errors)
return value
def char_pointer_cast(string, encoding='utf-8'):
if encoding is not None:
try:
string = string.encode(encoding)
except AttributeError:
# In Python3, bytes has no encode attribute
pass
string = ctypes.c_char_p(string)
return ctypes.cast(string, ctypes.POINTER(ctypes.c_char))
class FunctionFactoryStub:
def __getattr__(self, _):
return ctypes.CFUNCTYPE(lambda y:y)
# libraries['FIXME_STUB'] explanation
# As you did not list (-l libraryname.so) a library that exports this function
# This is a non-working stub instead.
# You can either re-run clan2py with -l /path/to/library.so
# Or manually fix this by comment the ctypes.CDLL loading
_libraries = {}
_libraries['FIXME_STUB'] = ctypes.CDLL(os.getenv('MESA_PATH', '/usr/')+'/lib/x86_64-linux-gnu/libvulkan_nouveau.so') # ctypes.CDLL('FIXME_STUB')
# values for enumeration 'nv_device_type'
nv_device_type__enumvalues = {
0: 'NV_DEVICE_TYPE_IGP',
1: 'NV_DEVICE_TYPE_DIS',
2: 'NV_DEVICE_TYPE_SOC',
}
NV_DEVICE_TYPE_IGP = 0
NV_DEVICE_TYPE_DIS = 1
NV_DEVICE_TYPE_SOC = 2
nv_device_type = ctypes.c_uint32 # enum
class struct_nv_device_info(Structure):
pass
class struct_nv_device_info_pci(Structure):
pass
struct_nv_device_info_pci._pack_ = 1 # source:False
struct_nv_device_info_pci._fields_ = [
('domain', ctypes.c_uint16),
('bus', ctypes.c_ubyte),
('dev', ctypes.c_ubyte),
('func', ctypes.c_ubyte),
('revision_id', ctypes.c_ubyte),
]
struct_nv_device_info._pack_ = 1 # source:False
struct_nv_device_info._fields_ = [
('type', ctypes.c_ubyte),
('PADDING_0', ctypes.c_ubyte),
('device_id', ctypes.c_uint16),
('chipset', ctypes.c_uint16),
('device_name', ctypes.c_char * 64),
('chipset_name', ctypes.c_char * 16),
('pci', struct_nv_device_info_pci),
('sm', ctypes.c_ubyte),
('gpc_count', ctypes.c_ubyte),
('tpc_count', ctypes.c_uint16),
('mp_per_tpc', ctypes.c_ubyte),
('max_warps_per_mp', ctypes.c_ubyte),
('cls_copy', ctypes.c_uint16),
('cls_eng2d', ctypes.c_uint16),
('cls_eng3d', ctypes.c_uint16),
('cls_m2mf', ctypes.c_uint16),
('cls_compute', ctypes.c_uint16),
('PADDING_1', ctypes.c_ubyte * 4),
('vram_size_B', ctypes.c_uint64),
('bar_size_B', ctypes.c_uint64),
]
size_t = ctypes.c_uint64
try:
nv_device_uuid = _libraries['FIXME_STUB'].nv_device_uuid
nv_device_uuid.restype = None
nv_device_uuid.argtypes = [ctypes.POINTER(struct_nv_device_info), ctypes.POINTER(ctypes.c_ubyte), size_t, ctypes.c_bool]
except AttributeError:
pass
class struct_nak_compiler(Structure):
pass
try:
nak_compiler_create = _libraries['FIXME_STUB'].nak_compiler_create
nak_compiler_create.restype = ctypes.POINTER(struct_nak_compiler)
nak_compiler_create.argtypes = [ctypes.POINTER(struct_nv_device_info)]
except AttributeError:
pass
try:
nak_compiler_destroy = _libraries['FIXME_STUB'].nak_compiler_destroy
nak_compiler_destroy.restype = None
nak_compiler_destroy.argtypes = [ctypes.POINTER(struct_nak_compiler)]
except AttributeError:
pass
uint64_t = ctypes.c_uint64
try:
nak_debug_flags = _libraries['FIXME_STUB'].nak_debug_flags
nak_debug_flags.restype = uint64_t
nak_debug_flags.argtypes = [ctypes.POINTER(struct_nak_compiler)]
except AttributeError:
pass
class struct_nir_shader_compiler_options(Structure):
pass
try:
nak_nir_options = _libraries['FIXME_STUB'].nak_nir_options
nak_nir_options.restype = ctypes.POINTER(struct_nir_shader_compiler_options)
nak_nir_options.argtypes = [ctypes.POINTER(struct_nak_compiler)]
except AttributeError:
pass
class struct_nir_shader(Structure):
pass
try:
nak_preprocess_nir = _libraries['FIXME_STUB'].nak_preprocess_nir
nak_preprocess_nir.restype = None
nak_preprocess_nir.argtypes = [ctypes.POINTER(struct_nir_shader), ctypes.POINTER(struct_nak_compiler)]
except AttributeError:
pass
try:
nak_nir_lower_image_addrs = _libraries['FIXME_STUB'].nak_nir_lower_image_addrs
nak_nir_lower_image_addrs.restype = ctypes.c_bool
nak_nir_lower_image_addrs.argtypes = [ctypes.POINTER(struct_nir_shader), ctypes.POINTER(struct_nak_compiler)]
except AttributeError:
pass
class struct_nak_sample_location(Structure):
pass
struct_nak_sample_location._pack_ = 1 # source:False
struct_nak_sample_location._fields_ = [
('x_u4', ctypes.c_ubyte, 4),
('y_u4', ctypes.c_ubyte, 4),
]
class struct_nak_sample_mask(Structure):
pass
struct_nak_sample_mask._pack_ = 1 # source:False
struct_nak_sample_mask._fields_ = [
('sample_mask', ctypes.c_uint16),
]
class struct_nak_fs_key(Structure):
pass
struct_nak_fs_key._pack_ = 1 # source:False
struct_nak_fs_key._fields_ = [
('zs_self_dep', ctypes.c_bool),
('force_sample_shading', ctypes.c_bool),
('uses_underestimate', ctypes.c_bool),
('sample_info_cb', ctypes.c_ubyte),
('sample_locations_offset', ctypes.c_uint32),
('sample_masks_offset', ctypes.c_uint32),
]
# values for enumeration 'c__EA_nir_variable_mode'
c__EA_nir_variable_mode__enumvalues = {
1: 'nir_var_system_value',
2: 'nir_var_uniform',
4: 'nir_var_shader_in',
8: 'nir_var_shader_out',
16: 'nir_var_image',
32: 'nir_var_shader_call_data',
64: 'nir_var_ray_hit_attrib',
128: 'nir_var_mem_ubo',
256: 'nir_var_mem_push_const',
512: 'nir_var_mem_ssbo',
1024: 'nir_var_mem_constant',
2048: 'nir_var_mem_task_payload',
4096: 'nir_var_mem_node_payload',
8192: 'nir_var_mem_node_payload_in',
16384: 'nir_var_function_in',
32768: 'nir_var_function_out',
65536: 'nir_var_function_inout',
131072: 'nir_var_shader_temp',
262144: 'nir_var_function_temp',
524288: 'nir_var_mem_shared',
1048576: 'nir_var_mem_global',
1966080: 'nir_var_mem_generic',
1159: 'nir_var_read_only_modes',
1969033: 'nir_var_vec_indexable_modes',
21: 'nir_num_variable_modes',
2097151: 'nir_var_all',
}
nir_var_system_value = 1
nir_var_uniform = 2
nir_var_shader_in = 4
nir_var_shader_out = 8
nir_var_image = 16
nir_var_shader_call_data = 32
nir_var_ray_hit_attrib = 64
nir_var_mem_ubo = 128
nir_var_mem_push_const = 256
nir_var_mem_ssbo = 512
nir_var_mem_constant = 1024
nir_var_mem_task_payload = 2048
nir_var_mem_node_payload = 4096
nir_var_mem_node_payload_in = 8192
nir_var_function_in = 16384
nir_var_function_out = 32768
nir_var_function_inout = 65536
nir_var_shader_temp = 131072
nir_var_function_temp = 262144
nir_var_mem_shared = 524288
nir_var_mem_global = 1048576
nir_var_mem_generic = 1966080
nir_var_read_only_modes = 1159
nir_var_vec_indexable_modes = 1969033
nir_num_variable_modes = 21
nir_var_all = 2097151
c__EA_nir_variable_mode = ctypes.c_uint32 # enum
nir_variable_mode = c__EA_nir_variable_mode
nir_variable_mode__enumvalues = c__EA_nir_variable_mode__enumvalues
try:
nak_postprocess_nir = _libraries['FIXME_STUB'].nak_postprocess_nir
nak_postprocess_nir.restype = None
nak_postprocess_nir.argtypes = [ctypes.POINTER(struct_nir_shader), ctypes.POINTER(struct_nak_compiler), nir_variable_mode, ctypes.POINTER(struct_nak_fs_key)]
except AttributeError:
pass
# values for enumeration 'nak_ts_domain'
nak_ts_domain__enumvalues = {
0: 'NAK_TS_DOMAIN_ISOLINE',
1: 'NAK_TS_DOMAIN_TRIANGLE',
2: 'NAK_TS_DOMAIN_QUAD',
}
NAK_TS_DOMAIN_ISOLINE = 0
NAK_TS_DOMAIN_TRIANGLE = 1
NAK_TS_DOMAIN_QUAD = 2
nak_ts_domain = ctypes.c_uint32 # enum
# values for enumeration 'nak_ts_spacing'
nak_ts_spacing__enumvalues = {
0: 'NAK_TS_SPACING_INTEGER',
1: 'NAK_TS_SPACING_FRACT_ODD',
2: 'NAK_TS_SPACING_FRACT_EVEN',
}
NAK_TS_SPACING_INTEGER = 0
NAK_TS_SPACING_FRACT_ODD = 1
NAK_TS_SPACING_FRACT_EVEN = 2
nak_ts_spacing = ctypes.c_uint32 # enum
# values for enumeration 'nak_ts_prims'
nak_ts_prims__enumvalues = {
0: 'NAK_TS_PRIMS_POINTS',
1: 'NAK_TS_PRIMS_LINES',
2: 'NAK_TS_PRIMS_TRIANGLES_CW',
3: 'NAK_TS_PRIMS_TRIANGLES_CCW',
}
NAK_TS_PRIMS_POINTS = 0
NAK_TS_PRIMS_LINES = 1
NAK_TS_PRIMS_TRIANGLES_CW = 2
NAK_TS_PRIMS_TRIANGLES_CCW = 3
nak_ts_prims = ctypes.c_uint32 # enum
class struct_nak_xfb_info(Structure):
pass
struct_nak_xfb_info._pack_ = 1 # source:False
struct_nak_xfb_info._fields_ = [
('stride', ctypes.c_uint32 * 4),
('stream', ctypes.c_ubyte * 4),
('attr_count', ctypes.c_ubyte * 4),
('attr_index', ctypes.c_ubyte * 128 * 4),
]
class struct_nak_shader_info(Structure):
pass
# values for enumeration 'mesa_shader_stage'
mesa_shader_stage__enumvalues = {
-1: 'MESA_SHADER_NONE',
0: 'MESA_SHADER_VERTEX',
1: 'MESA_SHADER_TESS_CTRL',
2: 'MESA_SHADER_TESS_EVAL',
3: 'MESA_SHADER_GEOMETRY',
4: 'MESA_SHADER_FRAGMENT',
5: 'MESA_SHADER_COMPUTE',
6: 'MESA_SHADER_TASK',
7: 'MESA_SHADER_MESH',
8: 'MESA_SHADER_RAYGEN',
9: 'MESA_SHADER_ANY_HIT',
10: 'MESA_SHADER_CLOSEST_HIT',
11: 'MESA_SHADER_MISS',
12: 'MESA_SHADER_INTERSECTION',
13: 'MESA_SHADER_CALLABLE',
14: 'MESA_SHADER_KERNEL',
}
MESA_SHADER_NONE = -1
MESA_SHADER_VERTEX = 0
MESA_SHADER_TESS_CTRL = 1
MESA_SHADER_TESS_EVAL = 2
MESA_SHADER_GEOMETRY = 3
MESA_SHADER_FRAGMENT = 4
MESA_SHADER_COMPUTE = 5
MESA_SHADER_TASK = 6
MESA_SHADER_MESH = 7
MESA_SHADER_RAYGEN = 8
MESA_SHADER_ANY_HIT = 9
MESA_SHADER_CLOSEST_HIT = 10
MESA_SHADER_MISS = 11
MESA_SHADER_INTERSECTION = 12
MESA_SHADER_CALLABLE = 13
MESA_SHADER_KERNEL = 14
mesa_shader_stage = ctypes.c_int32 # enum
class union_nak_shader_info_0(Union):
pass
class struct_nak_shader_info_0_cs(Structure):
pass
struct_nak_shader_info_0_cs._pack_ = 1 # source:False
struct_nak_shader_info_0_cs._fields_ = [
('local_size', ctypes.c_uint16 * 3),
('smem_size', ctypes.c_uint16),
('_pad', ctypes.c_ubyte * 4),
]
class struct_nak_shader_info_0_fs(Structure):
pass
struct_nak_shader_info_0_fs._pack_ = 1 # source:False
struct_nak_shader_info_0_fs._fields_ = [
('writes_depth', ctypes.c_bool),
('reads_sample_mask', ctypes.c_bool),
('post_depth_coverage', ctypes.c_bool),
('uses_sample_shading', ctypes.c_bool),
('early_fragment_tests', ctypes.c_bool),
('_pad', ctypes.c_ubyte * 7),
]
class struct_nak_shader_info_0_ts(Structure):
pass
struct_nak_shader_info_0_ts._pack_ = 1 # source:False
struct_nak_shader_info_0_ts._fields_ = [
('domain', ctypes.c_ubyte),
('spacing', ctypes.c_ubyte),
('prims', ctypes.c_ubyte),
('_pad', ctypes.c_ubyte * 9),
]
union_nak_shader_info_0._pack_ = 1 # source:False
union_nak_shader_info_0._fields_ = [
('cs', struct_nak_shader_info_0_cs),
('fs', struct_nak_shader_info_0_fs),
('ts', struct_nak_shader_info_0_ts),
('_pad', ctypes.c_ubyte * 12),
]
class struct_nak_shader_info_vtg(Structure):
pass
struct_nak_shader_info_vtg._pack_ = 1 # source:False
struct_nak_shader_info_vtg._fields_ = [
('writes_layer', ctypes.c_bool),
('writes_point_size', ctypes.c_bool),
('writes_vprs_table_index', ctypes.c_bool),
('clip_enable', ctypes.c_ubyte),
('cull_enable', ctypes.c_ubyte),
('_pad', ctypes.c_ubyte * 3),
('xfb', struct_nak_xfb_info),
]
struct_nak_shader_info._pack_ = 1 # source:False
struct_nak_shader_info._anonymous_ = ('_0',)
struct_nak_shader_info._fields_ = [
('stage', mesa_shader_stage),
('sm', ctypes.c_ubyte),
('num_gprs', ctypes.c_ubyte),
('num_control_barriers', ctypes.c_ubyte),
('_pad0', ctypes.c_ubyte),
('max_warps_per_sm', ctypes.c_uint32),
('num_instrs', ctypes.c_uint32),
('num_static_cycles', ctypes.c_uint32),
('num_spills_to_mem', ctypes.c_uint32),
('num_fills_from_mem', ctypes.c_uint32),
('num_spills_to_reg', ctypes.c_uint32),
('num_fills_from_reg', ctypes.c_uint32),
('slm_size', ctypes.c_uint32),
('crs_size', ctypes.c_uint32),
('_0', union_nak_shader_info_0),
('vtg', struct_nak_shader_info_vtg),
('hdr', ctypes.c_uint32 * 32),
]
class struct_nak_shader_bin(Structure):
pass
struct_nak_shader_bin._pack_ = 1 # source:False
struct_nak_shader_bin._fields_ = [
('info', struct_nak_shader_info),
('code_size', ctypes.c_uint32),
('PADDING_0', ctypes.c_ubyte * 4),
('code', ctypes.POINTER(None)),
('asm_str', ctypes.POINTER(ctypes.c_char)),
]
try:
nak_shader_bin_destroy = _libraries['FIXME_STUB'].nak_shader_bin_destroy
nak_shader_bin_destroy.restype = None
nak_shader_bin_destroy.argtypes = [ctypes.POINTER(struct_nak_shader_bin)]
except AttributeError:
pass
try:
nak_compile_shader = _libraries['FIXME_STUB'].nak_compile_shader
nak_compile_shader.restype = ctypes.POINTER(struct_nak_shader_bin)
nak_compile_shader.argtypes = [ctypes.POINTER(struct_nir_shader), ctypes.c_bool, ctypes.POINTER(struct_nak_compiler), nir_variable_mode, ctypes.POINTER(struct_nak_fs_key)]
except AttributeError:
pass
class struct_nak_qmd_cbuf(Structure):
pass
struct_nak_qmd_cbuf._pack_ = 1 # source:False
struct_nak_qmd_cbuf._fields_ = [
('index', ctypes.c_uint32),
('size', ctypes.c_uint32),
('addr', ctypes.c_uint64),
]
class struct_nak_qmd_info(Structure):
pass
struct_nak_qmd_info._pack_ = 1 # source:False
struct_nak_qmd_info._fields_ = [
('addr', ctypes.c_uint64),
('smem_size', ctypes.c_uint16),
('smem_max', ctypes.c_uint16),
('global_size', ctypes.c_uint32 * 3),
('num_cbufs', ctypes.c_uint32),
('PADDING_0', ctypes.c_ubyte * 4),
('cbufs', struct_nak_qmd_cbuf * 8),
]
try:
nak_fill_qmd = _libraries['FIXME_STUB'].nak_fill_qmd
nak_fill_qmd.restype = None
nak_fill_qmd.argtypes = [ctypes.POINTER(struct_nv_device_info), ctypes.POINTER(struct_nak_shader_info), ctypes.POINTER(struct_nak_qmd_info), ctypes.POINTER(None), size_t]
except AttributeError:
pass
class struct_nak_qmd_dispatch_size_layout(Structure):
pass
struct_nak_qmd_dispatch_size_layout._pack_ = 1 # source:False
struct_nak_qmd_dispatch_size_layout._fields_ = [
('x_start', ctypes.c_uint16),
('x_end', ctypes.c_uint16),
('y_start', ctypes.c_uint16),
('y_end', ctypes.c_uint16),
('z_start', ctypes.c_uint16),
('z_end', ctypes.c_uint16),
]
try:
nak_get_qmd_dispatch_size_layout = _libraries['FIXME_STUB'].nak_get_qmd_dispatch_size_layout
nak_get_qmd_dispatch_size_layout.restype = struct_nak_qmd_dispatch_size_layout
nak_get_qmd_dispatch_size_layout.argtypes = [ctypes.POINTER(struct_nv_device_info)]
except AttributeError:
pass
class struct_nak_qmd_cbuf_desc_layout(Structure):
pass
struct_nak_qmd_cbuf_desc_layout._pack_ = 1 # source:False
struct_nak_qmd_cbuf_desc_layout._fields_ = [
('addr_shift', ctypes.c_uint16),
('addr_lo_start', ctypes.c_uint16),
('addr_lo_end', ctypes.c_uint16),
('addr_hi_start', ctypes.c_uint16),
('addr_hi_end', ctypes.c_uint16),
]
uint8_t = ctypes.c_uint8
try:
nak_get_qmd_cbuf_desc_layout = _libraries['FIXME_STUB'].nak_get_qmd_cbuf_desc_layout
nak_get_qmd_cbuf_desc_layout.restype = struct_nak_qmd_cbuf_desc_layout
nak_get_qmd_cbuf_desc_layout.argtypes = [ctypes.POINTER(struct_nv_device_info), uint8_t]
except AttributeError:
pass
__all__ = \
['MESA_SHADER_ANY_HIT', 'MESA_SHADER_CALLABLE',
'MESA_SHADER_CLOSEST_HIT', 'MESA_SHADER_COMPUTE',
'MESA_SHADER_FRAGMENT', 'MESA_SHADER_GEOMETRY',
'MESA_SHADER_INTERSECTION', 'MESA_SHADER_KERNEL',
'MESA_SHADER_MESH', 'MESA_SHADER_MISS', 'MESA_SHADER_NONE',
'MESA_SHADER_RAYGEN', 'MESA_SHADER_TASK', 'MESA_SHADER_TESS_CTRL',
'MESA_SHADER_TESS_EVAL', 'MESA_SHADER_VERTEX',
'NAK_TS_DOMAIN_ISOLINE', 'NAK_TS_DOMAIN_QUAD',
'NAK_TS_DOMAIN_TRIANGLE', 'NAK_TS_PRIMS_LINES',
'NAK_TS_PRIMS_POINTS', 'NAK_TS_PRIMS_TRIANGLES_CCW',
'NAK_TS_PRIMS_TRIANGLES_CW', 'NAK_TS_SPACING_FRACT_EVEN',
'NAK_TS_SPACING_FRACT_ODD', 'NAK_TS_SPACING_INTEGER',
'NV_DEVICE_TYPE_DIS', 'NV_DEVICE_TYPE_IGP', 'NV_DEVICE_TYPE_SOC',
'c__EA_nir_variable_mode', 'mesa_shader_stage',
'nak_compile_shader', 'nak_compiler_create',
'nak_compiler_destroy', 'nak_debug_flags', 'nak_fill_qmd',
'nak_get_qmd_cbuf_desc_layout',
'nak_get_qmd_dispatch_size_layout', 'nak_nir_lower_image_addrs',
'nak_nir_options', 'nak_postprocess_nir', 'nak_preprocess_nir',
'nak_shader_bin_destroy', 'nak_ts_domain', 'nak_ts_prims',
'nak_ts_spacing', 'nir_num_variable_modes', 'nir_var_all',
'nir_var_function_in', 'nir_var_function_inout',
'nir_var_function_out', 'nir_var_function_temp', 'nir_var_image',
'nir_var_mem_constant', 'nir_var_mem_generic',
'nir_var_mem_global', 'nir_var_mem_node_payload',
'nir_var_mem_node_payload_in', 'nir_var_mem_push_const',
'nir_var_mem_shared', 'nir_var_mem_ssbo',
'nir_var_mem_task_payload', 'nir_var_mem_ubo',
'nir_var_ray_hit_attrib', 'nir_var_read_only_modes',
'nir_var_shader_call_data', 'nir_var_shader_in',
'nir_var_shader_out', 'nir_var_shader_temp',
'nir_var_system_value', 'nir_var_uniform',
'nir_var_vec_indexable_modes', 'nir_variable_mode',
'nir_variable_mode__enumvalues', 'nv_device_type',
'nv_device_uuid', 'size_t', 'struct_nak_compiler',
'struct_nak_fs_key', 'struct_nak_qmd_cbuf',
'struct_nak_qmd_cbuf_desc_layout',
'struct_nak_qmd_dispatch_size_layout', 'struct_nak_qmd_info',
'struct_nak_sample_location', 'struct_nak_sample_mask',
'struct_nak_shader_bin', 'struct_nak_shader_info',
'struct_nak_shader_info_0_cs', 'struct_nak_shader_info_0_fs',
'struct_nak_shader_info_0_ts', 'struct_nak_shader_info_vtg',
'struct_nak_xfb_info', 'struct_nir_shader',
'struct_nir_shader_compiler_options', 'struct_nv_device_info',
'struct_nv_device_info_pci', 'uint64_t', 'uint8_t',
'union_nak_shader_info_0']
File diff suppressed because it is too large Load Diff
+42 -31
View File
@@ -11,12 +11,16 @@ from tinygrad.helpers import getenv, mv_address, round_up, data64, data64_le, pr
from tinygrad.renderer.ptx import PTXRenderer
from tinygrad.renderer.cstyle import NVRenderer
from tinygrad.runtime.support.compiler_cuda import CUDACompiler, PTXCompiler, PTX, NVPTXCompiler, NVCompiler
from tinygrad.runtime.support.nak import NAKCompiler, parse_nak_shader
from tinygrad.runtime.autogen import nv_gpu, pci
from tinygrad.runtime.support.elf import elf_loader
from tinygrad.runtime.support.nv.nvdev import NVDev, NVMemoryManager
from tinygrad.runtime.support.system import System, PCIIfaceBase, MAP_FIXED
if getenv("IOCTL"): import extra.nv_gpu_driver.nv_ioctl # noqa: F401 # pylint: disable=unused-import
NIR = getenv("NIR")
if NIR: from tinygrad.renderer.nir import NIRRenderer
def get_error_str(status): return f"{status}: {nv_gpu.nv_status_codes.get(status, 'Unknown error')}"
NV_PFAULT_FAULT_TYPE = {dt:name for name,dt in nv_gpu.__dict__.items() if name.startswith("NV_PFAULT_FAULT_TYPE_")}
@@ -191,39 +195,46 @@ class NVArgsState(CLikeArgsState):
class NVProgram(HCQProgram):
def __init__(self, dev:NVDevice, name:str, lib:bytes):
self.dev, self.name, self.lib = dev, name, lib
# For MOCKGPU, the lib is PTX code, so some values are emulated.
cbuf0_size = 0 if not MOCKGPU else 0x160
if MOCKGPU: image, sections, relocs = memoryview(bytearray(lib) + b'\x00' * (4 - len(lib)%4)).cast("I"), [], [] # type: ignore
else: image, sections, relocs = elf_loader(self.lib, force_section_align=128)
# NOTE: Ensure at least 4KB of space after the program to mitigate prefetch memory faults.
self.lib_gpu = self.dev.allocator.alloc(round_up(image.nbytes, 0x1000) + 0x1000, buf_spec:=BufferSpec(cpu_access=True))
self.prog_addr, self.prog_sz, self.regs_usage, self.shmem_usage, self.lcmem_usage = self.lib_gpu.va_addr, image.nbytes, 0, 0x400, 0
self.constbufs: dict[int, tuple[int, int]] = {0: (0, 0x160)} # dict[constbuf index, tuple[va_addr, size]]
for sh in sections:
if sh.name == f".nv.shared.{self.name}": self.shmem_usage = round_up(0x400 + sh.header.sh_size, 128)
if sh.name == f".text.{self.name}": self.prog_addr, self.prog_sz = self.lib_gpu.va_addr+sh.header.sh_addr, sh.header.sh_size
elif m:=re.match(r'\.nv\.constant(\d+)', sh.name): self.constbufs[int(m.group(1))] = (self.lib_gpu.va_addr+sh.header.sh_addr, sh.header.sh_size)
elif sh.name.startswith(".nv.info"):
for typ, param, data in self._parse_elf_info(sh):
if sh.name == f".nv.info.{name}" and param == 0xa: cbuf0_size = struct.unpack_from("IH", data)[1] # EIATTR_PARAM_CBANK
elif sh.name == ".nv.info" and param == 0x12: self.lcmem_usage = struct.unpack_from("II", data)[1] + 0x240 # EIATTR_MIN_STACK_SIZE
elif sh.name == ".nv.info" and param == 0x2f: self.regs_usage = struct.unpack_from("II", data)[1] # EIATTR_REGCOUNT
if NIR:
image, self.regs_usage, self.shmem_usage, self.lcmem_usage = parse_nak_shader(lib)
self.lib_gpu = self.dev.allocator.alloc(round_up(image.nbytes, 0x1000) + 0x1000, buf_spec:=BufferSpec(cpu_access=True))
cbuf0_size = 0x160 # ?
self.prog_addr, self.prog_sz = self.lib_gpu.va_addr, image.nbytes
else:
if MOCKGPU: image, sections, relocs = memoryview(bytearray(lib) + b'\x00' * (4 - len(lib)%4)).cast("I"), [], [] # type: ignore
else: image, sections, relocs = elf_loader(self.lib, force_section_align=128)
# For MOCKGPU, the lib is PTX code, so some values are emulated.
cbuf0_size = 0 if not MOCKGPU else 0x160
# NOTE: Ensure at least 4KB of space after the program to mitigate prefetch memory faults.
self.lib_gpu = self.dev.allocator.alloc(round_up(image.nbytes, 0x1000) + 0x1000, buf_spec:=BufferSpec(cpu_access=True))
self.prog_addr, self.prog_sz, self.regs_usage, self.shmem_usage, self.lcmem_usage = self.lib_gpu.va_addr, image.nbytes, 0, 0x400, 0x240
for sh in sections:
if sh.name == f".nv.shared.{self.name}": self.shmem_usage = round_up(0x400 + sh.header.sh_size, 128)
if sh.name == f".text.{self.name}": self.prog_addr, self.prog_sz = self.lib_gpu.va_addr+sh.header.sh_addr, sh.header.sh_size
elif m:=re.match(r'\.nv\.constant(\d+)', sh.name):
self.constbufs[int(m.group(1))] = (self.lib_gpu.va_addr+sh.header.sh_addr, sh.header.sh_size)
elif sh.name.startswith(".nv.info"):
for typ, param, data in self._parse_elf_info(sh):
if sh.name == f".nv.info.{name}" and param == 0xa: cbuf0_size = struct.unpack_from("IH", data)[1] # EIATTR_PARAM_CBANK
elif sh.name == ".nv.info" and param == 0x12: self.lcmem_usage = struct.unpack_from("II", data)[1] + 0x240 # EIATTR_MIN_STACK_SIZE
elif sh.name == ".nv.info" and param == 0x2f: self.regs_usage = struct.unpack_from("II", data)[1] # EIATTR_REGCOUNT
# Apply relocs
for apply_image_offset, rel_sym_offset, typ, _ in relocs:
# These types are CUDA-specific, applying them here
if typ == 2: image[apply_image_offset:apply_image_offset+8] = struct.pack('<Q', self.lib_gpu.va_addr + rel_sym_offset) # R_CUDA_64
elif typ == 0x38: image[apply_image_offset+4:apply_image_offset+8] = struct.pack('<I', (self.lib_gpu.va_addr + rel_sym_offset) & 0xffffffff)
elif typ == 0x39: image[apply_image_offset+4:apply_image_offset+8] = struct.pack('<I', (self.lib_gpu.va_addr + rel_sym_offset) >> 32)
else: raise RuntimeError(f"unknown NV reloc {typ}")
# Ensure device has enough local memory to run the program
self.dev._ensure_has_local_memory(self.lcmem_usage)
# Apply relocs
for apply_image_offset, rel_sym_offset, typ, _ in relocs:
# These types are CUDA-specific, applying them here
if typ == 2: image[apply_image_offset:apply_image_offset+8] = struct.pack('<Q', self.lib_gpu.va_addr + rel_sym_offset) # R_CUDA_64
elif typ == 0x38: image[apply_image_offset+4:apply_image_offset+8] = struct.pack('<I', (self.lib_gpu.va_addr + rel_sym_offset) & 0xffffffff)
elif typ == 0x39: image[apply_image_offset+4:apply_image_offset+8] = struct.pack('<I', (self.lib_gpu.va_addr + rel_sym_offset) >> 32)
else: raise RuntimeError(f"unknown NV reloc {typ}")
ctypes.memmove(self.lib_gpu.va_addr, mv_address(image), image.nbytes)
self.constbuffer_0 = [0] * (cbuf0_size // 4)
@@ -525,9 +536,9 @@ class NVDevice(HCQCompiled[HCQSignal]):
self.arch: str = "sm_120" if self.sm_version==0xa04 else f"sm_{(self.sm_version>>8)&0xff}{(val>>4) if (val:=self.sm_version&0xff) > 0xf else val}"
self.sass_version = ((self.sm_version & 0xf00) >> 4) | (self.sm_version & 0xf)
compiler_t = (PTXCompiler if PTX else CUDACompiler) if MOCKGPU else (NVPTXCompiler if PTX else NVCompiler)
super().__init__(device, NVAllocator(self), PTXRenderer(self.arch, device="NV") if PTX else NVRenderer(self.arch), compiler_t(self.arch),
functools.partial(NVProgram, self), HCQSignal, NVComputeQueue, NVCopyQueue)
cc = NAKCompiler(self) if NIR else (PTXCompiler if PTX else (CUDACompiler if MOCKGPU else (NVPTXCompiler if PTX else NVCompiler)))(self.arch)
rr = PTXRenderer(self.arch, device="NV") if PTX else (NIRRenderer(self.arch) if NIR else NVRenderer(self.arch))
super().__init__(device, NVAllocator(self), rr, cc, functools.partial(NVProgram, self), HCQSignal, NVComputeQueue, NVCopyQueue)
self._setup_gpfifos()
+30
View File
@@ -0,0 +1,30 @@
from typing import Tuple
from tinygrad.device import Compiler
from base64 import b64decode
from gzip import decompress
import tinygrad.runtime.autogen.nak as nak
import tinygrad.runtime.autogen.nir as nir
import ctypes
class NAKCompiler(Compiler):
def __init__(self, dev, cache_key="nak"):
print(dev.arch, dev.max_warps_per_sm)
self.cc = nak.nak_compiler_create(nak.struct_nv_device_info(sm=int(dev.arch[3:]), max_warps_per_mp=dev.max_warps_per_sm))
super().__init__(f"compile_{cache_key}_{dev.arch}")
def compile(self, src) -> bytes:
shader = nak.struct_nir_shader.from_buffer(src)
# TODO: only "True" if we want to print sass
nak.nak_preprocess_nir(shader, self.cc)
out = nak.nak_compile_shader(shader, True, self.cc, 0, None).contents
print(out)
return out
def parse_nak_shader(shader:bytes) -> Tuple[memoryview, int, int, int]:
sb = nak.struct_nak_shader_bin.from_buffer(shader)
return (memoryview(bytearray(ctypes.string_at(sb.code, sb.code_size))), sb.info.num_gprs, 0x400, 0x240)
# TODO: clean this up
nak_nir_options = nir.nir_shader_compiler_options.from_buffer_copy(
decompress(b64decode(b"H4sIAAAAAAAAA2NkAANGIABRIBZUAELBWWA2WB1cBSOqWohqEIQLoABGiBE4ABYpBRzS++sbGNzZIWx+fpgC7JYSAgBqun8JAAEAAA=="))
)