hosting/lib/nvml.py

463 lines
22 KiB
Python

from lib import config as config_module
from lib import logging as logging_lib
from lib import get_specs
config = config_module.config
log = logging_lib.log
import subprocess
import clore_pynvml as pynvml
import json
import math
HIVE_PATH="/hive/bin:/hive/sbin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:./"
GPU_MEM_ALLOWED_OC_RANGES = { # Known to be problematic GPUs
"NVIDIA P102-100": [-2000, 2000],
"NVIDIA P104-100": [-2000, 2000],
"NVIDIA P106-090": [-2000, 2000],
"NVIDIA P106-100": [-2000, 2000],
"NVIDIA GeForce GTX 1050 Ti": [-2000, 2000],
"NVIDIA GeForce GTX 1060 3GB": [-2000, 2000],
"NVIDIA GeForce GTX 1060 6GB": [-2000, 2000],
"NVIDIA GeForce GTX 1070": [-2000, 2000],
"NVIDIA GeForce GTX 1070 Ti": [-2000, 2000],
"NVIDIA GeForce GTX 1080": [-2000, 2000],
"NVIDIA GeForce GTX 1080 Ti": [-2000, 2000],
"NVIDIA CMP 30HX": [-2000, 6000],
"NVIDIA CMP 40HX": [-2000, 6000],
"NVIDIA CMP 50HX": [-2000, 6000],
"NVIDIA CMP 90HX": [-2000, 6000],
"NVIDIA GeForce GTX 1650": [-2000, 6000],
"NVIDIA GeForce GTX 1660 SUPER": [-2000, 6000],
"NVIDIA GeForce GTX 1660 Ti": [-2000, 6000],
"NVIDIA GeForce RTX 2060": [-2000, 6000],
"NVIDIA GeForce RTX 2060 SUPER": [-2000, 6000],
"NVIDIA GeForce RTX 2070": [-2000, 6000],
"NVIDIA GeForce RTX 2070 SUPER": [-2000, 6000],
"NVIDIA GeForce RTX 2080": [-2000, 6000],
"NVIDIA GeForce RTX 2080 Ti": [-2000, 6000]
}
GPU_CORE_ALLOWED_OC_RANGES = { # Known to be problematic GPUs
"NVIDIA P102-100": [-200, 1200],
"NVIDIA P104-100": [-200, 1200],
"NVIDIA P106-090": [-200, 1200],
"NVIDIA P106-100": [-200, 1200],
"NVIDIA GeForce GTX 1050 Ti": [-200, 1200],
"NVIDIA GeForce GTX 1060 3GB": [-200, 1200],
"NVIDIA GeForce GTX 1060 6GB": [-200, 1200],
"NVIDIA GeForce GTX 1070": [-200, 1200],
"NVIDIA GeForce GTX 1070 Ti": [-200, 1200],
"NVIDIA GeForce GTX 1080": [-200, 1200],
"NVIDIA GeForce GTX 1080 Ti": [-200, 1200],
"NVIDIA CMP 30HX": [-1000, 1000],
"NVIDIA CMP 40HX": [-1000, 1000],
"NVIDIA CMP 50HX": [-1000, 1000],
"NVIDIA CMP 90HX": [-1000, 1000],
"NVIDIA GeForce GTX 1650": [-1000, 1000],
"NVIDIA GeForce GTX 1660 SUPER": [-1000, 1000],
"NVIDIA GeForce GTX 1660 Ti": [-1000, 1000],
"NVIDIA GeForce RTX 2060": [-1000, 1000],
"NVIDIA GeForce RTX 2060 SUPER": [-1000, 1000],
"NVIDIA GeForce RTX 2070": [-1000, 1000],
"NVIDIA GeForce RTX 2070 SUPER": [-1000, 1000],
"NVIDIA GeForce RTX 2080": [-1000, 1000],
"NVIDIA GeForce RTX 2080 Ti": [-1000, 1000]
}
is_hive = False
all_gpus_data_list=[]
get_data_fail=False
def init(gpu_specs_file=None, allow_hive_binaries=True):
global is_hive, all_gpus_data_list, get_data_fail
log.info("Loading GPU OC specs [ working ]")
try:
pynvml.nvmlInit()
kernel = get_specs.get_kernel()
if "hive" in kernel and allow_hive_binaries:
is_hive=True
specs_file_loc = gpu_specs_file if gpu_specs_file else config.gpu_specs_file
regenerate_specs = False
parsed_specs={}
try:
with open(specs_file_loc, "r") as specs_file:
parsed_specs = json.loads(specs_file.read())
except Exception as specs_load_fail:
log.error(f"Failed loading gpu_specs_file ({specs_load_fail}) | regenerating...")
regenerate_specs=True
parsed_specs_keys = parsed_specs.keys()
gpu_count = pynvml.nvmlDeviceGetCount()
for i in range(0,gpu_count):
if regenerate_specs:
break
gpu_handle = pynvml.nvmlDeviceGetHandleByIndex(i)
gpu_uuid = pynvml.nvmlDeviceGetUUID(gpu_handle)
if not f"{i}-{gpu_uuid}" in parsed_specs_keys:
parsed_specs={}
regenerate_specs=True
break
elif not "locks" in parsed_specs[f"{i}-{gpu_uuid}"]:
parsed_specs={}
regenerate_specs=True
break
if regenerate_specs:
for i in range(0,gpu_count):
gpu_spec={}
mem_to_core_allowed_locks = get_gpu_locked_clocks(i)
gpu_handle = pynvml.nvmlDeviceGetHandleByIndex(i)
gpu_uuid = pynvml.nvmlDeviceGetUUID(gpu_handle)
power_limits = pynvml.nvmlDeviceGetPowerManagementLimitConstraints(gpu_handle)
min_power_limit = int(power_limits[0] / 1000.0)
max_power_limit = int(power_limits[1] / 1000.0)
gpu_spec["default_power_limit"] = int(pynvml.nvmlDeviceGetPowerManagementDefaultLimit(gpu_handle) / 1000.0)
gpu_spec["power_limits"] = [min_power_limit, max_power_limit]
gpu_spec["name"] = pynvml.nvmlDeviceGetName(gpu_handle)
gpu_spec["locks"] = mem_to_core_allowed_locks
pci_info = pynvml.nvmlDeviceGetPciInfo(gpu_handle)
pci_bus_id = pci_info.bus
pci_device_id = pci_info.device
pci_domain_id = pci_info.domain
gpu_spec["pci_core"] = f"{pci_domain_id}:{pci_bus_id:02d}:{pci_device_id:02d}.0"
mem_range = get_hive_clock_range(is_hive, i, "mem")
core_range = get_hive_clock_range(is_hive, i, "core")
try:
if type(mem_range) != list:
pynvml.nvmlDeviceSetMemoryLockedClocks(gpu_handle, 200, 300) # Force low clocks, so the GPU can't crash when testing if under load
failure_min, min_oc_solution = pinpoint_oc_limits_negative(gpu_handle)
failure_max, max_oc_solution = pinpoint_oc_limits_positive(gpu_handle)
if (not failure_min) and (not failure_max):
mem_range=[min_oc_solution, max_oc_solution]
pynvml.nvmlDeviceSetMemClkVfOffset(gpu_handle, 0)
pynvml.nvmlDeviceResetMemoryLockedClocks(gpu_handle)
if type(core_range) != list:
pynvml.nvmlDeviceSetGpuLockedClocks(gpu_handle, 300, 350) # Force low clocks, so the GPU can't crash when testing if under load
failure_min, min_oc_solution = pinpoint_oc_limits_negative(gpu_handle, True)
failure_max, max_oc_solution = pinpoint_oc_limits_positive(gpu_handle, True)
if (not failure_min) and (not failure_max):
core_range=[min_oc_solution, max_oc_solution]
pynvml.nvmlDeviceSetGpcClkVfOffset(gpu_handle, 0)
pynvml.nvmlDeviceResetGpuLockedClocks(gpu_handle)
except Exception as e_pinpointing:
if "not supported" in str(e_pinpointing).lower():
try:
min_core_offset, max_core_offset = pynvml.nvmlDeviceGetGpcClkMinMaxVfOffset(gpu_handle)
if min_core_offset>0:
min_core_offset = min_core_offset - math.floor((2**32)/1000)
if min_core_offset > -20000 and min_core_offset <= 0 and max_core_offset>=0 and min_core_offset < 20000:
core_range=[min_core_offset, max_core_offset]
else:
core_range=[0,0]
min_mem_offset, max_mem_offset = pynvml.nvmlDeviceGetMemClkMinMaxVfOffset(gpu_handle)
if min_mem_offset>0:
min_mem_offset = min_mem_offset - math.floor((2**32)/1000)
if min_mem_offset==0 and max_mem_offset==0:
if gpu_spec["name"] in GPU_MEM_ALLOWED_OC_RANGES:
mem_range = GPU_MEM_ALLOWED_OC_RANGES[gpu_spec["name"]]
else:
mem_range = [0,0]
elif min_mem_offset > -20000 and min_mem_offset <= 0 and max_mem_offset>=0 and max_mem_offset < 20000:
mem_range=[min_mem_offset, max_mem_offset]
else:
mem_range=[0,0]
except Exception as e2:
if "function not found" in str(e2).lower():
if gpu_spec["name"] in GPU_MEM_ALLOWED_OC_RANGES:
mem_range = GPU_MEM_ALLOWED_OC_RANGES[gpu_spec["name"]]
else:
mem_range = [0,0]
if gpu_spec["name"] in GPU_CORE_ALLOWED_OC_RANGES:
core_range = GPU_CORE_ALLOWED_OC_RANGES[gpu_spec["name"]]
else:
core_range = [0,0]
else:
get_data_fail=True
if type(mem_range) == list and type(core_range) == list and len(mem_range)==2 and len(core_range)==2:
gpu_spec["mem"]=mem_range
gpu_spec["core"]=core_range
else:
get_data_fail=True
parsed_specs[f"{i}-{gpu_uuid}"]=gpu_spec
with open(specs_file_loc, "w") as specs_file:
json.dump(parsed_specs, specs_file)
if not get_data_fail:
parsed_specs_keys=parsed_specs.keys()
for key in parsed_specs_keys:
all_gpus_data_list.append(parsed_specs[key])
except Exception as e:
get_data_fail=True
log.error("Loading GPU OC specs [ fail ]")
if not get_data_fail:
log.success("Loading GPU OC specs [ success ]")
print(all_gpus_data_list)
# Load GPU specs
def get_gpu_oc_specs():
global get_data_fail
if get_data_fail:
return False
else:
return all_gpus_data_list
def shutdown():
pynvml.nvmlShutdown()
def get_gpu_locked_clocks(gpu_index):
try:
handle = pynvml.nvmlDeviceGetHandleByIndex(gpu_index)
mem_clocks = pynvml.nvmlDeviceGetSupportedMemoryClocks(handle)
mem_to_core = {}
for idx, mem_clock in enumerate(mem_clocks):
if idx < 12 or idx == len(mem_clocks)-1:
graphics_clocks = pynvml.nvmlDeviceGetSupportedGraphicsClocks(handle, mem_clock)
mem_to_core[str(mem_clock)] = [min(graphics_clocks), max(graphics_clocks)]
return mem_to_core
except Exception as e:
return {}
def handle_nn(input_int):
if abs(4293967-input_int) < 10000:
return input_int-4293967
elif abs(8589934-input_int) < 10000:
return input_int-8589934
else:
return input_int
def pinpoint_find_dicts_negative(data):
false_success_items = [d for d in data if not d['success']]
true_success_items = [d for d in data if d['success']]
highest_false_success = max(false_success_items, key=lambda x: x['offset'], default=None)
lowest_true_success = min(true_success_items, key=lambda x: x['offset'], default=None)
return highest_false_success, lowest_true_success
def pinpoint_find_dicts_positive(data):
false_success_items = [d for d in data if not d['success']]
true_success_items = [d for d in data if d['success']]
lowest_false_success = min(false_success_items, key=lambda x: x['offset'], default=None)
highest_true_success = max(true_success_items, key=lambda x: x['offset'], default=None)
return highest_true_success, lowest_false_success
def pinpoint_oc_limits_negative(gpu_handle, core=False):
step_cnt = 0
found_solution = None
init_negative_max = -19855 # Probably
history_info = [{"offset": init_negative_max*2, "success":False}]
failure = False
max_step_cnt = 20
try:
while found_solution == None and step_cnt<max_step_cnt and not failure:
step_cnt+=1
#print("STEP", step_cnt)
#print(history_info)
highest_false_success, lowest_true_success = pinpoint_find_dicts_negative(history_info)
test_offset = None
if lowest_true_success == None:
test_offset = int(highest_false_success["offset"]/2)
elif highest_false_success != None:
test_offset = int((highest_false_success["offset"]+lowest_true_success["offset"])/2)
if not step_cnt<max_step_cnt:
found_solution=lowest_true_success["offset"]
test_offset=None
elif test_offset==lowest_true_success["offset"]:
found_solution=test_offset
test_offset=None
if test_offset != None:
any_exception = False
try:
if core:
pynvml.nvmlDeviceSetGpcClkVfOffset(gpu_handle, test_offset)
else:
pynvml.nvmlDeviceSetMemClkVfOffset(gpu_handle, test_offset)
except Exception as e:
any_exception=True
if not "Unknown Error" in str(e):
failure=True
history_info.append({"offset": test_offset, "success":not any_exception})
except Exception as e:
failure=True
return failure, found_solution
def pinpoint_oc_limits_positive(gpu_handle, core=False):
step_cnt = 0
found_solution = None
init_negative_max = 20000 # Probably
history_info = [{"offset": init_negative_max*2, "success":False}]
failure = False
max_step_cnt = 20
try:
while found_solution == None and step_cnt<max_step_cnt and not failure:
step_cnt+=1
#print("STEP", step_cnt)
#print(history_info)
highest_true_success, lowest_false_success = pinpoint_find_dicts_positive(history_info)
test_offset = None
if highest_true_success == None:
test_offset = int(lowest_false_success["offset"]/2)
elif lowest_false_success != None:
test_offset = int((highest_true_success["offset"]+lowest_false_success["offset"])/2)
if not step_cnt<max_step_cnt:
found_solution=highest_true_success["offset"]
test_offset=None
elif test_offset==highest_true_success["offset"]:
found_solution=test_offset
test_offset=None
if test_offset != None:
any_exception = False
try:
if core:
pynvml.nvmlDeviceSetGpcClkVfOffset(gpu_handle, test_offset)
else:
pynvml.nvmlDeviceSetMemClkVfOffset(gpu_handle, test_offset)
except Exception as e:
any_exception=True
if not "Unknown Error" in str(e):
failure=True
history_info.append({"offset": test_offset, "success":not any_exception})
except Exception as e:
failure=True
return failure, found_solution
def set_oc(settings):
global is_hive
try:
gpu_count = pynvml.nvmlDeviceGetCount()
settings_keys = settings.keys()
if len(settings_keys)==0: # Configure default clocks/pl
settings={}
for i in range(0,gpu_count):
settings[str(i)]={
"core":0,
"mem":0,
"pl": all_gpus_data_list[i]["default_power_limit"]
}
settings_keys = settings.keys()
log.debug(f"Rewriting settings with: {json.dumps(settings)}")
core_locks = []
mem_locks = []
any_lock_failure = False
for oc_gpu_index in settings_keys:
if oc_gpu_index.isdigit():
oc_gpu_index=int(oc_gpu_index)
if oc_gpu_index < gpu_count and type(settings[str(oc_gpu_index)])==dict:
gpu_oc_config = settings[str(oc_gpu_index)]
gpu_possible_ranges = all_gpus_data_list[oc_gpu_index]
gpu_handle = pynvml.nvmlDeviceGetHandleByIndex(oc_gpu_index)
if "core_lock" in gpu_oc_config:
core_lock = int(gpu_oc_config["core_lock"])
core_locks.append(str(core_lock))
try:
pynvml.nvmlDeviceSetGpuLockedClocks(gpu_handle, core_lock, core_lock)
except Exception as core_lock_exception:
any_lock_failure=True
else:
core_locks.append('0')
try:
pynvml.nvmlDeviceResetGpuLockedClocks(gpu_handle)
except Exception as core_lock_exception:
any_lock_failure=True
if "mem_lock" in gpu_oc_config:
mem_lock = int(gpu_oc_config["mem_lock"])
mem_locks.append(str(mem_lock))
try:
pynvml.nvmlDeviceSetMemoryLockedClocks(gpu_handle, mem_lock, mem_lock)
except Exception as mem_lock_exception:
any_lock_failure=True
else:
mem_locks.append('0')
try:
pynvml.nvmlDeviceResetMemoryLockedClocks(gpu_handle)
except Exception as mem_lock_exception:
any_lock_failure=True
if "core" in gpu_oc_config: # Core offset
wanted_core_clock = int(round(gpu_oc_config["core"]*2))
if gpu_possible_ranges["core"][0] <= wanted_core_clock and wanted_core_clock <= gpu_possible_ranges["core"][1]:
pynvml.nvmlDeviceSetGpcClkVfOffset(gpu_handle, wanted_core_clock)
else:
log.error(f"Requested OC for GPU:{oc_gpu_index} (CORE) out of bound | {wanted_core_clock} | [{gpu_possible_ranges["core"][0]}, {gpu_possible_ranges["core"][1]}]")
if "mem" in gpu_oc_config: # Memory offset
wanted_mem_clock = int(round(gpu_oc_config["mem"]*2))
if gpu_possible_ranges["mem"][0] <= wanted_mem_clock and wanted_mem_clock <= gpu_possible_ranges["mem"][1]:
pynvml.nvmlDeviceSetMemClkVfOffset(gpu_handle, wanted_mem_clock)
else:
log.error(f"Requested OC for GPU:{oc_gpu_index} (MEMORY) out of bound | {wanted_mem_clock} | [{gpu_possible_ranges["mem"][0]}, {gpu_possible_ranges["mem"][1]}]")
if "pl" in gpu_oc_config:
wanted_power_limit_milliwatts = gpu_oc_config["pl"]*1000 # convert W to mW
if gpu_possible_ranges["power_limits"][0] <= gpu_oc_config["pl"] and gpu_oc_config["pl"] <= gpu_possible_ranges["power_limits"][1]:
pynvml.nvmlDeviceSetPowerManagementLimit(gpu_handle, wanted_power_limit_milliwatts)
else:
log.error(f"Requested OC for GPU:{oc_gpu_index} (POWER LIMIT) out of bound | {gpu_oc_config["pl"]} | [{gpu_possible_ranges["power_limits"][0]}, {gpu_possible_ranges["power_limits"][1]}]")
if is_hive and any_lock_failure and len(mem_locks)==len(core_locks):
try:
nvtool_commands = []
for idx, mem_lock in enumerate(mem_locks):
core_lock = core_locks[idx]
nvtool_commands.append(f"nvtool -i {str(idx)} --setmem {mem_lock} --setcore {core_lock}")
cmd = ["bash",'-c',f"PATH={HIVE_PATH} && sudo {' && '.join(nvtool_commands)}"]
#print(cmd)
subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except Exception as hive_oc_settings:
pass
return True
except Exception as e:
log.error(f"set_oc | ERROR | {e}")
return False
def get_hive_clock_range(is_hive, gpu_index, part):
if is_hive:
try:
flag = "--setmemoffset" if part=="mem" else "--setcoreoffset"
cmd = ["bash",'-c',f"PATH={HIVE_PATH} && sudo nvtool -i {gpu_index} {flag} -100000"]
result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
lines = result.stdout.decode().splitlines()
stripped_lines = [line.strip() for line in lines]
non_empty_lines = [line for line in stripped_lines if line]
device_id = None
result=[]
for non_empty_line in non_empty_lines:
if non_empty_line[:8]=="DEVICE #":
device_id = int(non_empty_line[8:].replace(':',''))
elif " is not in range of " in non_empty_line and device_id!=None and device_id==gpu_index:
splited_line = non_empty_line.split(" is not in range of ",1)[1].split(' ',4)
min_val = int(splited_line[0])
max_val = int(splited_line[2])
result=[min_val, max_val]
if len(result)==0:
return False
else:
return result
except Exception as e:
return False
else:
return False
def get_vram_per_gpu():
vram_per_gpu = []
try:
gpu_count = pynvml.nvmlDeviceGetCount()
for i in range(0,gpu_count):
gpu_handle = pynvml.nvmlDeviceGetHandleByIndex(i)
mem_info = pynvml.nvmlDeviceGetMemoryInfo(gpu_handle)
vram_per_gpu.append(mem_info.total / 1024 ** 2)
except Exception as e:
log.error(f"Failed loading get_vram_per_gpu() | {e}")
pass
return vram_per_gpu