From 04c271ac414c42a0bbfe58fdcaa4ae0a5bc797a2 Mon Sep 17 00:00:00 2001 From: qazal <77887910+Qazalin@users.noreply.github.com> Date: Wed, 12 Aug 2026 15:00:04 +0800 Subject: [PATCH] simplify digitalocean_mi350x (#17502) * simplify digitalocean_mi350x * no hardcoded rocm path --- .../tinybox_8xMI350X/dev_run.sh | 4 ++-- extra/runbook_digitalocean_mi350x.md | 24 ++++--------------- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh index bb53cc0ded..6ec564f12d 100755 --- a/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh +++ b/examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash export PYTHONPATH="." -export PATH="/opt/rocm-7.1.1/bin:$PATH" -export ROCM_PATH="/opt/rocm-7.1.1" +export ROCM_PATH=${ROCM_PATH:-/opt/rocm-7.1.1} +export PATH="$ROCM_PATH/bin:$PATH" export DEV=${DEV:-AMD} export CHECK_OOB=0 export REWRITE_STACK_LIMIT=5000000 HCQDEV_WAIT_TIMEOUT_MS=240000 diff --git a/extra/runbook_digitalocean_mi350x.md b/extra/runbook_digitalocean_mi350x.md index b76e9c77de..edb09e6f2f 100644 --- a/extra/runbook_digitalocean_mi350x.md +++ b/extra/runbook_digitalocean_mi350x.md @@ -74,8 +74,8 @@ rclone config create mlc-training s3 provider=Cloudflare \ secret_access_key=a53625c4d45e3ca8ac0df8a353ea3a41ffc3292aa25259addd8b7dc5a6ce2936 \ endpoint=c2686074cb2caf5cbaf6d134bdba8b47.r2.cloudflarestorage.com -mkdir -p /root/datasets/c4-8b -rclone copy mlc-training:mlcommons-training-wg-public/llama3_1/datasets/c4/llama3_1_8b/ /root/datasets/c4-8b/ -P +mkdir -p /raid/datasets/c4-8b +rclone copy mlc-training:mlcommons-training-wg-public/llama3_1/datasets/c4/llama3_1_8b/ /raid/datasets/c4-8b/ -P ``` Files downloaded (~85GB total, ~6 minutes): @@ -85,13 +85,6 @@ Files downloaded (~85GB total, ~6 minutes): - `c4-validation-91205-samples.en_text_document.idx` (1.8 MB) - `LICENSE.txt`, `NOTICE.txt` -### Symlink for the submission script -The `dev_run.sh` script hardcodes `BASEDIR="/raid/datasets/c4-8b/"`. Symlink: -```bash -mkdir -p /raid/datasets -ln -s /root/datasets/c4-8b /raid/datasets/c4-8b -``` - ## Phase 4: wandb Login ```bash wandb login @@ -100,7 +93,7 @@ Enter API key from https://wandb.ai/authorize ## Phase 5: Run Training -### 5.1 Smoke test (beam search, 2 layers, fake data) +### 5.1 Smoke test (beam search, 2 layers, real data) Always run beam first to validate the pipeline: ```bash cd /root/tinygrad @@ -108,7 +101,7 @@ COMGR_PATH=/opt/rocm/lib/libamd_comgr.so \ COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so \ CC=/opt/rocm/core-7.14/lib/llvm/bin/clang \ DEV=AMD:HIP \ -ROCM_PATH=/opt/rocm BASEDIR=/root/datasets/c4-8b/ \ +ROCM_PATH=/opt/rocm \ bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_beam.sh ``` @@ -119,7 +112,7 @@ COMGR_PATH=/opt/rocm/lib/libamd_comgr.so \ COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so \ CC=/opt/rocm/core-7.14/lib/llvm/bin/clang \ DEV=AMD:HIP \ -ROCM_PATH=/opt/rocm BASEDIR=/root/datasets/c4-8b/ \ +ROCM_PATH=/opt/rocm \ WANDB=1 \ bash examples/mlperf/training_submission_v6.0/tinycorp/benchmarks/llama31_8b/implementations/tinybox_8xMI350X/dev_run.sh ``` @@ -133,7 +126,6 @@ WANDB=1 \ | `CC` | `/opt/rocm/core-7.14/lib/llvm/bin/clang` | System clang doesn't know gfx950; must use ROCm's bundled clang | | `DEV` | `AMD:HIP` | Force HIPRenderer (comgr-based) over HIPCCRenderer (hipcc subprocess) | | `ROCM_PATH` | `/opt/rocm` | Script defaults to `/opt/rocm-7.1.1` which doesn't exist | -| `BASEDIR` | `/root/datasets/c4-8b/` | Where C4 dataset was downloaded (script hardcodes `/raid/datasets/c4-8b/`) | | `WANDB` | `1` | Enable wandb logging (off by default) | ## Architecture @@ -173,12 +165,6 @@ ldconfig ### `comgr not available: try setting COMGR_3_PATH?` comgr 3.x uses a separate module. Set `COMGR_3_PATH=/opt/rocm/lib/libamd_comgr.so` too. -### `FileNotFoundError: '/raid/datasets/c4-8b/...'` -Script hardcodes `BASEDIR`. Either symlink or edit the script: -```bash -mkdir -p /raid/datasets && ln -s /root/datasets/c4-8b /raid/datasets/c4-8b -``` - ### `No such file or directory: 'clang'` Install clang: `apt-get install -y clang` (for CPU compilation). For gfx950 HIP compilation, comgr (not clang) is used — ensure the ROCm 7.14 comgr 3.3 is properly loaded via `COMGR_PATH` and `COMGR_3_PATH`.