diff --git a/README.md b/README.md index 1f8e9c086..d892e6271 100644 --- a/README.md +++ b/README.md @@ -45,18 +45,18 @@ To ensure that everything is working you can run tests for the package with pkg> test AMDGPU ``` -Or specifying a subset of tests to run: +Or specifying a subset of tests to run on 4 workers in parallel: ```julia julia> using Pkg -julia> Pkg.test("AMDGPU"; test_args=["core", "kernelabstractions"]) +julia> Pkg.test("AMDGPU"; test_args=`--jobs=4 core kernelabstractions`) ``` -Full list of tests to run can be obtained with `--list` argument: +Full list of tests to run can be obtained with `--list` argument (refer to [ParallelTestRunner.jl's doc](https://juliatesting.github.io/ParallelTestRunner.jl/dev/#Command-Line-Options) for the full list of available command line arguments): ```julia -julia> Pkg.test("AMDGPU"; test_args=["--list"]) +julia> Pkg.test("AMDGPU"; test_args=`--list`]) ``` ## Questions and Contributions diff --git a/docs/make.jl b/docs/make.jl index 744f028f9..886bdb456 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -35,6 +35,7 @@ function main() "Performance Tips" => "tutorials/perf.md", "Profiling" => "tutorials/profiling.md", "Installation Tips" => "install_tips.md", + "Testing" => "testing.md", ], "API" => [ "Devices" => "api/devices.md", diff --git a/docs/src/index.md b/docs/src/index.md index 7b4ec5e87..bf34563c4 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -28,6 +28,7 @@ Simply add the AMDGPU.jl package to your Julia environment: ```julia using Pkg + Pkg.add("AMDGPU") ``` @@ -62,6 +63,7 @@ To ensure that everything works, you can run the test suite: ```julia using AMDGPU using Pkg + Pkg.test("AMDGPU") ``` diff --git a/docs/src/testing.md b/docs/src/testing.md new file mode 100644 index 000000000..795ccea17 --- /dev/null +++ b/docs/src/testing.md @@ -0,0 +1,42 @@ +# Testing + +To ensure that everything is working you can run tests for the package with: + +```julia +pkg> test AMDGPU +``` + +## Advanced testing options + +AMDGPU tests use [ParallelTestRunner.jl]((https://juliatesting.github.io/ParallelTestRunner.jl) which allow for [running tests with various (command line) options](https://juliatesting.github.io/ParallelTestRunner.jl/dev/#Running-Tests). + +To, e.g., launch a subset of tests `core` and `kernelabstractions` on 4 runners in parallel: + +```julia +julia> using Pkg + +julia> Pkg.test("AMDGPU"; test_args=`--jobs=4 core kernelabstractions`) +``` + +The full list of tests to run can be obtained with `--list` argument: + +```julia +julia> Pkg.test("AMDGPU"; test_args=`--list`]) +``` + +## Testing categories + +Although tests can be run in a custom fashion upon exploring the output of listing (using the `--list` test argument), tests are organised such that following grouping by categories is possible: + +``` +core device hip external gpuarrays kernelabstractions wmma enzyme +``` + +which allows to, e.g., run all `gpuarrays` related tests as + +``` +julia> Pkg.test("AMDGPU"; test_args=`gpuarrays`) +``` + +!!! warning "Large memory tests" + Some tests HIP and GPUArrays tests may use > 20GB of host RAM. It is recommended to use fewer workers (<= 4) on machines that have < 32Gb of host RAM in case running tests would result in out of memory errors. diff --git a/test/codegen/codegen.jl b/test/core/codegen.jl similarity index 100% rename from test/codegen/codegen.jl rename to test/core/codegen.jl diff --git a/test/core_tests.jl b/test/core/core_tests.jl similarity index 100% rename from test/core_tests.jl rename to test/core/core_tests.jl diff --git a/test/rocarray/base.jl b/test/core/rocarray_base.jl similarity index 100% rename from test/rocarray/base.jl rename to test/core/rocarray_base.jl diff --git a/test/rocarray/broadcast.jl b/test/core/rocarray_broadcast.jl similarity index 91% rename from test/rocarray/broadcast.jl rename to test/core/rocarray_broadcast.jl index 8fbeb3725..13c1ce3d8 100644 --- a/test/rocarray/broadcast.jl +++ b/test/core/rocarray_broadcast.jl @@ -2,10 +2,6 @@ using Test using AMDGPU using AMDGPU: ROCArray -import GPUArrays -include(joinpath(pkgdir(GPUArrays), "test", "testsuite.jl")) -testf(f, xs...; kwargs...) = TestSuite.compare(f, AMDGPU.ROCArray, xs...; kwargs...) - @testset "broadcast" begin @test testf((x) -> fill!(x, 1), rand(3,3)) @test testf((x, y) -> map(+, x, y), rand(2, 3), rand(2, 3)) diff --git a/test/tls.jl b/test/core/tls.jl similarity index 100% rename from test/tls.jl rename to test/core/tls.jl diff --git a/test/gpuarrays_base.jl b/test/gpuarrays_base.jl deleted file mode 100644 index 8c3526fb6..000000000 --- a/test/gpuarrays_base.jl +++ /dev/null @@ -1,17 +0,0 @@ -using Test -using AMDGPU - -import GPUArrays -include(joinpath(pkgdir(GPUArrays), "test", "testsuite.jl")) - -const BASE_TESTS = ["base", "constructors", "vectors", "random", "broadcasting", "statistics", "alloc cache"] - -@testset "GPUArrays - Base" begin - for (name, test) in TestSuite.tests - if name in BASE_TESTS - @testset "$name" begin - test(AMDGPU.ROCArray) - end - end - end -end diff --git a/test/gpuarrays_indexing.jl b/test/gpuarrays_indexing.jl deleted file mode 100644 index cd8822ccd..000000000 --- a/test/gpuarrays_indexing.jl +++ /dev/null @@ -1,15 +0,0 @@ -using Test -using AMDGPU - -import GPUArrays -include(joinpath(pkgdir(GPUArrays), "test", "testsuite.jl")) - -@testset "GPUArrays - Indexing" begin - for (name, test) in TestSuite.tests - if startswith(name, "indexing") - @testset "$name" begin - test(AMDGPU.ROCArray) - end - end - end -end diff --git a/test/gpuarrays_linalg.jl b/test/gpuarrays_linalg.jl deleted file mode 100644 index 9ebe6ae60..000000000 --- a/test/gpuarrays_linalg.jl +++ /dev/null @@ -1,15 +0,0 @@ -using Test -using AMDGPU - -import GPUArrays -include(joinpath(pkgdir(GPUArrays), "test", "testsuite.jl")) - -@testset "GPUArrays - Linear Algebra" begin - for (name, test) in TestSuite.tests - if startswith(name, "linalg") - @testset "$name" begin - test(AMDGPU.ROCArray) - end - end - end -end diff --git a/test/gpuarrays_math.jl b/test/gpuarrays_math.jl deleted file mode 100644 index 6f3b5151a..000000000 --- a/test/gpuarrays_math.jl +++ /dev/null @@ -1,15 +0,0 @@ -using Test -using AMDGPU - -import GPUArrays -include(joinpath(pkgdir(GPUArrays), "test", "testsuite.jl")) - -@testset "GPUArrays - Math" begin - for (name, test) in TestSuite.tests - if startswith(name, "math") - @testset "$name" begin - test(AMDGPU.ROCArray) - end - end - end -end diff --git a/test/gpuarrays_misc.jl b/test/gpuarrays_misc.jl deleted file mode 100644 index c0995db0e..000000000 --- a/test/gpuarrays_misc.jl +++ /dev/null @@ -1,21 +0,0 @@ -using Test -using AMDGPU - -import GPUArrays -include(joinpath(pkgdir(GPUArrays), "test", "testsuite.jl")) - -# Tests that don't fit in other categories -const KNOWN_CATEGORIES = ["indexing", "reductions", "linalg", "math"] -const BASE_TESTS = ["base", "constructors", "vectors", "random", "broadcasting", "statistics", "alloc cache"] - -@testset "GPUArrays - Misc" begin - for (name, test) in TestSuite.tests - # Skip tests covered by other files - any(startswith(name, cat) for cat in KNOWN_CATEGORIES) && continue - name in BASE_TESTS && continue - - @testset "$name" begin - test(AMDGPU.ROCArray) - end - end -end diff --git a/test/gpuarrays_reductions.jl b/test/gpuarrays_reductions.jl deleted file mode 100644 index 40ddd9ccb..000000000 --- a/test/gpuarrays_reductions.jl +++ /dev/null @@ -1,15 +0,0 @@ -using Test -using AMDGPU - -import GPUArrays -include(joinpath(pkgdir(GPUArrays), "test", "testsuite.jl")) - -@testset "GPUArrays - Reductions" begin - for (name, test) in TestSuite.tests - if startswith(name, "reductions") - @testset "$name" begin - test(AMDGPU.ROCArray) - end - end - end -end diff --git a/test/dnn/activations.jl b/test/hip_dnn/activations.jl similarity index 100% rename from test/dnn/activations.jl rename to test/hip_dnn/activations.jl diff --git a/test/dnn/batchnorm.jl b/test/hip_dnn/batchnorm.jl similarity index 100% rename from test/dnn/batchnorm.jl rename to test/hip_dnn/batchnorm.jl diff --git a/test/dnn/conv.jl b/test/hip_dnn/conv.jl similarity index 100% rename from test/dnn/conv.jl rename to test/hip_dnn/conv.jl diff --git a/test/dnn/descriptors.jl b/test/hip_dnn/descriptors.jl similarity index 100% rename from test/dnn/descriptors.jl rename to test/hip_dnn/descriptors.jl diff --git a/test/dnn/pool.jl b/test/hip_dnn/pool.jl similarity index 100% rename from test/dnn/pool.jl rename to test/hip_dnn/pool.jl diff --git a/test/rocarray/blas.jl b/test/hip_rocarray/blas.jl similarity index 99% rename from test/rocarray/blas.jl rename to test/hip_rocarray/blas.jl index 0c6276d08..8f22642e3 100644 --- a/test/rocarray/blas.jl +++ b/test/hip_rocarray/blas.jl @@ -3,10 +3,6 @@ using AMDGPU using AMDGPU: ROCArray, ROCVector, ROCMatrix using LinearAlgebra -import GPUArrays -include(joinpath(pkgdir(GPUArrays), "test", "testsuite.jl")) -testf(f, xs...; kwargs...) = TestSuite.compare(f, AMDGPU.ROCArray, xs...; kwargs...) - @assert AMDGPU.functional(:rocblas) @testset "BLAS" begin @@ -132,7 +128,7 @@ end @test testf( (y, a, b) -> mul!(y, Hermitian(a), b), rand(T, 5), rand(T, 5, 5), rand(T, 5)) - + A_ = rand(T, m, m) A = A_ + A_' x = rand(T, m) @@ -423,7 +419,7 @@ end dC = rocBLAS.trsm('L', 'U', 'N', 'N', one(T), dA, dB) @test collect(dC) ≈ triu(view(A,1:m,1:m)) \ view(B,1:m,1:m) end - + @testset "trsm_batched" begin batch_count = 3 A = [rand(T, m, m) for ix in 1:batch_count] @@ -661,7 +657,7 @@ end @test C ≈ h_C end @testset "herk T=$T" for T in (ComplexF32, ComplexF64) - T1 = T + T1 = T T2 = real(T) # generate parameters α = rand(T2) @@ -689,7 +685,7 @@ end @test C ≈ h_C end @testset "her2k T=$T" for T in (ComplexF32, ComplexF64) - T1 = T + T1 = T T2 = real(T) # generate parameters α = rand(T1) diff --git a/test/rocarray/fft.jl b/test/hip_rocarray/fft.jl similarity index 100% rename from test/rocarray/fft.jl rename to test/hip_rocarray/fft.jl diff --git a/test/rocarray/indexing.jl b/test/hip_rocarray/indexing.jl similarity index 100% rename from test/rocarray/indexing.jl rename to test/hip_rocarray/indexing.jl diff --git a/test/rocarray/multi_gpu.jl b/test/hip_rocarray/multi_gpu.jl similarity index 100% rename from test/rocarray/multi_gpu.jl rename to test/hip_rocarray/multi_gpu.jl diff --git a/test/rocarray/random.jl b/test/hip_rocarray/random.jl similarity index 100% rename from test/rocarray/random.jl rename to test/hip_rocarray/random.jl diff --git a/test/rocarray/reverse.jl b/test/hip_rocarray/reverse.jl similarity index 100% rename from test/rocarray/reverse.jl rename to test/hip_rocarray/reverse.jl diff --git a/test/rocarray/solver.jl b/test/hip_rocarray/solver.jl similarity index 100% rename from test/rocarray/solver.jl rename to test/hip_rocarray/solver.jl diff --git a/test/rocarray/sorting.jl b/test/hip_rocarray/sorting.jl similarity index 100% rename from test/rocarray/sorting.jl rename to test/hip_rocarray/sorting.jl diff --git a/test/rocsparse/broadcast.jl b/test/hip_rocsparse/broadcast.jl similarity index 100% rename from test/rocsparse/broadcast.jl rename to test/hip_rocsparse/broadcast.jl diff --git a/test/rocsparse/conversions.jl b/test/hip_rocsparse/conversions.jl similarity index 100% rename from test/rocsparse/conversions.jl rename to test/hip_rocsparse/conversions.jl diff --git a/test/rocsparse/device.jl b/test/hip_rocsparse/device.jl similarity index 100% rename from test/rocsparse/device.jl rename to test/hip_rocsparse/device.jl diff --git a/test/rocsparse/generic.jl b/test/hip_rocsparse/generic.jl similarity index 100% rename from test/rocsparse/generic.jl rename to test/hip_rocsparse/generic.jl diff --git a/test/rocsparse/interfaces.jl b/test/hip_rocsparse/interfaces.jl similarity index 100% rename from test/rocsparse/interfaces.jl rename to test/hip_rocsparse/interfaces.jl diff --git a/test/rocsparse/linalg.jl b/test/hip_rocsparse/linalg.jl similarity index 100% rename from test/rocsparse/linalg.jl rename to test/hip_rocsparse/linalg.jl diff --git a/test/rocsparse/preconditioners.jl b/test/hip_rocsparse/preconditioners.jl similarity index 100% rename from test/rocsparse/preconditioners.jl rename to test/hip_rocsparse/preconditioners.jl diff --git a/test/ka_tests.jl b/test/kernelabstractions_tests.jl similarity index 100% rename from test/ka_tests.jl rename to test/kernelabstractions_tests.jl diff --git a/test/runtests.jl b/test/runtests.jl index 7cc666ebd..a1829c758 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -2,86 +2,92 @@ using AMDGPU using AMDGPU: Device, Runtime, @allowscalar import AMDGPU.Device: HostCallHolder, hostcall! -import Pkg -import PrettyTables -import InteractiveUtils - using ParallelTestRunner using Test -AMDGPU.allowscalar(false) - -# Test names for filtering (excluding enzyme by default) -const DEFAULT_TESTS = ["core", "device", "hip", "external", "gpuarrays", "kernelabstractions", "wmma"] - -function parse_flags!(args, flag; default = nothing, typ = typeof(default)) - for f in args - startswith(f, flag) || continue +import Pkg +import InteractiveUtils - if f != flag - val = split(f, '=')[2] - if !(typ === nothing || typ <: AbstractString) - val = parse(typ, val) - end - else - val = default - end +AMDGPU.allowscalar(false) - filter!(x -> x != f, args) - return true, val +# Force 4 workers if running on buildkite +if parse(Bool, get(ENV, "BUILDKITE", "false")) + if !any(startswith("--jobs"), ARGS) + push!(ARGS, "--jobs=4") end - return false, default end -do_help, _ = parse_flags!(ARGS, "--help") -if do_help - println(""" - Usage: runtests.jl [--help] [--list] [--jobs=N] [TESTS...] +# Default to 4 workers if running on machine with <= 32GB RAM and not on CI +if !any(startswith("--jobs"), ARGS) && (Sys.total_memory() <= 32*2^30) + push!(ARGS, "--jobs=4") +end + +@info "System information:\n" +InteractiveUtils.versioninfo() - --help Show this text. - --jobs=N Launch `N` processes to perform tests (defaults to dynamically inspecting nthreads and RAM availability of the CPU) - --list List all available tests. +AMDGPU.versioninfo() - Remaining arguments filter the tests that will be executed.""") - exit(0) +# Autodiscovered tests +testsuite = find_tests(@__DIR__) + +## GPUArrays test suite +import GPUArrays +gpuarrays = pathof(GPUArrays) +gpuarrays_root = dirname(dirname(gpuarrays)) +gpuarrays_testsuite = joinpath(gpuarrays_root, "test", "testsuite.jl") +include(gpuarrays_testsuite) +for name in keys(TestSuite.tests) + testsuite["gpuarrays/$name"] = :(TestSuite.tests[$name](AMDGPU.ROCArray)) end -do_list, _ = parse_flags!(ARGS, "--list") -if do_list - println("Available tests to run: $DEFAULT_TESTS, enzyme.") - exit(0) -end +args = parse_args(ARGS) -# No options should remain (except test names). -optlike_args = filter(startswith("-"), ARGS) -if !isempty(optlike_args) - error(""" - Unknown test options: `$(join(optlike_args, " "))`. - Try `--help` for usage instructions. - """) +# Don't run Enzyme tests by default +if filter_tests!(testsuite, args) + delete!(testsuite, "enzyme_tests") end -# Determine which tests to run -const TARGET_TESTS = isempty(ARGS) ? DEFAULT_TESTS : ARGS - -if "enzyme" in TARGET_TESTS +if any(name -> startswith(name, "enzyme"), keys(testsuite)) + @info "Running Enzyme tests\n" Pkg.add(["EnzymeCore", "Enzyme"]) end -InteractiveUtils.versioninfo() -AMDGPU.versioninfo() - -@info "Test suite info" -data = String["$(AMDGPU.device())" join(TARGET_TESTS, ", ");] -PrettyTables.pretty_table(data; column_labels=["Device", "Tests"], - fit_table_in_display_vertically=false, - fit_table_in_display_horizontally=false) +# Hostcall tests must run on main thread (not in parallel workers). To be addressed by https://github.com/JuliaTesting/ParallelTestRunner.jl/issues/77 +delete!(testsuite, "device/hostcall") +delete!(testsuite, "device/output") + +# Code to run in each test's sandbox module before running the test +init_code = quote + import GPUArrays + using AMDGPU + include($gpuarrays_testsuite) + testf(f, xs...; kwargs...) = TestSuite.compare(f, AMDGPU.ROCArray, xs...; kwargs...) + + macro grab_output(ex, io=stdout) + quote + mktemp() do fname, fout + ret = nothing + open(fname, "w") do fout + if $io == stdout + redirect_stdout(fout) do + ret = $(esc(ex)) + end + elseif $io == stderr + redirect_stderr(fout) do + ret = $(esc(ex)) + end + end + end + ret, read(fname, String) + end + end + end +end -# Run tests with ParallelTestRunner -runtests(AMDGPU, ARGS) +runtests(AMDGPU, args; testsuite, init_code) -# Hostcall tests must run on main thread (not in parallel workers) -if "core" in TARGET_TESTS && Sys.islinux() +# Hostcall tests must run on main thread (not in parallel workers). To be addressed by https://github.com/JuliaTesting/ParallelTestRunner.jl/issues/77 +if any(name -> startswith(name, "core"), keys(testsuite)) && Sys.islinux() @info "Testing `Hostcalls` on the main thread." @testset "Hostcalls" begin include("device/hostcall.jl")