超算中心海光异构卡dcu bw 64G显卡报错 无法通过设置来解决，通过新增服务器跳过显卡

flash_attn: disabled HOSTQUEUE <0x1921b670>: >>>>>>>> DUMP KERNEL ARGS: size: 160 <<<<<<<<<
00 40 83 00 00 00 00 00 00 00 90 01 00 00 00 00
00 40 83 00 00 00 00 00 00 00 b4 4e e1 7f 00 00
00 00 b4 4e e1 7f 00 00 00 00 60 d0 d6 7f 00 00
00 80 6f 31 e2 7f 00 00 00 00 80 3f 00 00 00 00
00 14 00 00 01 00 00 00 00 14 00 00 01 00 00 00
00 14 00 00 01 00 00 00 00 14 00 00 01 00 00 00
00 14 00 00 90 06 00 00 01 00 00 00 00 14 00 00
00 00 00 00 14 00 00 00 07 00 00 00 00 00 00 00
07 00 00 00 93 24 49 12 00 00 00 00 00 00 00 00
00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00
HOSTQUEUE <0x1921b670>: >>>>>>>> DUMP KERNEL ARGS PTR INFO <<<<<<<<<
HOSTQUEUE <0x1921b670>: ptr arg index: 3, ptr: 0x7fe14eb40000
HOSTQUEUE <0x1921b670>: origin ptr: 0x7fe14d200000, size byte: 113246208
HOSTQUEUE <0x1921b670>: ptr arg index: 4, ptr: 0x7fe14eb40000
HOSTQUEUE <0x1921b670>: origin ptr: 0x7fe14d200000, size byte: 113246208
HOSTQUEUE <0x1921b670>: ptr arg index: 5, ptr: 0x7fd6d0600000
HOSTQUEUE <0x1921b670>: origin ptr: 0x7fd6d0600000, size byte: 52428800
HOSTQUEUE <0x1921b670>: ptr arg index: 6, ptr: 0x7fe2316f8000
HOSTQUEUE <0x1921b670>: origin ptr: 0x7fe22ec00000, size byte: 88080384
HOSTQUEUE <0x1921b670>: >>>>>>>> DUMP AQL 0x7fe93e825000 PACKET WITH INDEX 217545 <<<<<<<<<
HOSTQUEUE <0x1921b670>: >>>>>>>> MATCH KERNEL COMMAND <<<<<<<<<
HOSTQUEUE <0x1921b670>: header: 0xb02
HOSTQUEUE <0x1921b670>: setup: 3
HOSTQUEUE <0x1921b670>: workgroup: x:256, y:1, z:1
HOSTQUEUE <0x1921b670>: grid: x:2150400, y:1, z:1
HOSTQUEUE <0x1921b670>: group_segment_size: 6144
HOSTQUEUE <0x1921b670>: private_segment_size: 0
HOSTQUEUE <0x1921b670>: completion_signal: 0x0
HOSTQUEUE <0x1921b670>: kernel_object: 0x7fe4a82424c0
HOSTQUEUE <0x1921b670>: device id: 0
Invalid address access: 0x7fde7654c000, Error code: 3.

KERNEL VMFault !!! <<<<<<
PID: 6011, SIGNAL: 0 !!! <<<<<<
还是一样

正确的环境变量

# =======曙光BW 2卡DCU 100小时稳定训练【无报错版】 =======
# =====  超算中心  老是报错 可以 ，换2个显卡 ====
# NCCL 内部通信错误
# export NCCL_IGNORE_DISABLED_P2P=1
# export NCCL_P2P_DISABLE=1

# 1. 禁用可能存在 Bug 的加速算子库
export ROC_BLAS_USE_HIPBLASLT=0
# 2. 强制同步执行，防止异步内存访问越界（虽然慢，但能保命）
export HIP_LAUNCH_BLOCKING=1
# 3. 限制 MIOpen 算子，防止在卷积或矩阵运算时出现段错误
export MIOPEN_DEBUG_DISABLE_CUDNN_COMPAT=1

# 不必删除 checkpoint-2500的global_step，否则我启动不了

# gfx936 不一定需要
export HSA_OVERRIDE_GFX_VERSION="9.3.6"


# unset HSA_OVERRIDE_GFX_VERSION


# 设置NCCL环境变量
# 4. 彻底禁用 P2P 通信，防止单卡模式下误触通信 Bug
export NCCL_P2P_DISABLE=1          # 海光DCU必须开！不开直接卡
export NCCL_BLOCKING_WAIT=1         # 防止卡死
export NCCL_ASYNC_ERROR_HANDLING=1  # 防止通信崩溃
export RCCL_P2P_DISABLE=1

# 超算BW禁用IB远程通信，避免分布式训练网络报错
export NCCL_IB_DISABLE=1

# 指定NCCL使用本地回环网卡lo，双卡本机通信最稳定
# 本地双卡 不能用网卡
export NCCL_SOCKET_IFNAME=lo

# 开启NCCL异步错误处理，防止通信卡死导致节点失败
export NCCL_ASYNC_ERROR_HANDLING=1

# NCCL超长超时(100小时)，避免慢节点中断
export NCCL_TIMEOUT=3600000





# 修复 ROCm 显存碎片
# 我直接给你深算 3 号 Z100 (gfx928) 专用、100% 能跑的最终版
# 海光DCU不支持expandable_segments，只保留max_split_size_mb防止显存碎片
# 但需要注意，海光环境有时对 expandable_segments 支持有 Bug。如果显存端也出现无故 OOM，可以尝试将其简化为 export PYTORCH_HIP_ALLOC_CONF="garbage_collection_threshold:0.8"
# export PYTORCH_HIP_ALLOC_CONF="expandable_segments:True,garbage_collection_threshold:0.8"
# export PYTORCH_HIP_ALLOC_CONF="garbage_collection_threshold:0.8"

# 建议修改这一行
export PYTORCH_HIP_ALLOC_CONF="max_split_size_mb:128,garbage_collection_threshold:0.8"






# 关闭MIOPEN确定性计算，减少内存占用、提速
export MIOPEN_DEBUG_CONVOLUTION_DETERMINISTIC=0

# 关闭MIOPEN冗余日志，避免缓冲区占满内存
export MIOPEN_LOG_LEVEL=1

# 指定使用2张DCU显卡
export HIP_VISIBLE_DEVICES=0,1

export ROCM_VISIBLE_DEVICES=0,1

unset CUDA_VISIBLE_DEVICES

# DCU兼容性必须设置，解决算子崩溃/显存泄漏
# export HSA_OVERRIDE_GFX_VERSION=10.3.0 pytorch不支持

# 关闭CuDNN，防止DCU环境不兼容
export Torch_USE_CUDNN=0