refactor: introducing tasks for map operations #10051
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: test | |
| on: | |
| push: | |
| branches: | |
| - "main" | |
| - "release-*" | |
| pull_request: | |
| branches: [main] | |
| jobs: | |
| ui: | |
| name: UI | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 6 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Setup Node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20" | |
| - name: Setup Node-Cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ui/node_modules | |
| key: ${{ runner.os }}-node-dep-v1-${{ hashFiles('**/yarn.lock') }} | |
| - name: Yarn Build/Test | |
| run: make ui-test | |
| - name: Ensure nothing changed | |
| run: git diff --exit-code | |
| codegen: | |
| name: Codegen | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 5 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Restore go build cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/go-build | |
| key: ${{ runner.os }}-go-build-v1-${{ hashFiles('**/go.mod') }} | |
| - name: Setup Golang | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: "1.24" | |
| - name: Add bins to PATH | |
| run: | | |
| echo /home/runner/go/bin >> $GITHUB_PATH | |
| echo /usr/local/bin >> $GITHUB_PATH | |
| - name: Get dependencies | |
| run: go mod download | |
| - name: Make codegen | |
| run: | | |
| echo 'GOPATH=/home/runner/go' >> $GITHUB_ENV | |
| make -B codegen | |
| - name: Ensure nothing changed | |
| run: git diff --exit-code | |
| unit-tests: | |
| name: Unit Tests | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 15 | |
| steps: | |
| - name: Start JetStream | |
| run: docker run -d -p 4222:4222 nats:latest -js | |
| - name: Start Pulsar standalone container | |
| run: docker run -d -p 6650:6650 -p 8080:8080 apachepulsar/pulsar:4.0.0 bin/pulsar standalone | |
| - name: Start Kafka container | |
| run: docker run -d -p 9092:9092 --name kafka-broker apache/kafka:4.0.0 | |
| - name: Start Redis standalone container | |
| run: docker run -d -p 6379:6379 --name redis-server redis:8-alpine | |
| - name: Set up Go 1.x | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: "1.24" | |
| id: go | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Restore Go build cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/go-build | |
| key: ${{ runner.os }}-go-build-v1-${{ github.run_id }} | |
| - name: Get dependencies | |
| run: go mod download -x | |
| - name: Test Go | |
| run: make test-coverage | |
| - name: Install Rust | |
| uses: actions-rust-lang/setup-rust-toolchain@v1 | |
| with: | |
| cache-workspaces: rust -> target | |
| - name: Install llvm-tools-preview | |
| working-directory: ./rust | |
| run: rustup component add llvm-tools-preview | |
| - name: Install grcov | |
| uses: taiki-e/install-action@v2 | |
| with: | |
| tool: grcov | |
| - name: Install Protoc | |
| uses: arduino/setup-protoc@v3 | |
| - name: Test Rust | |
| working-directory: ./rust | |
| run: | | |
| CARGO_INCREMENTAL=0 RUSTFLAGS='-Cinstrument-coverage' LLVM_PROFILE_FILE='./target/debug/coverage/cargo-test-%p-%m.profraw' make all-tests | |
| grcov . -s ./target/debug/coverage/ --binary-path ./target/debug/ -t lcov --branch --ignore-not-existing --ignore "**/build.rs" -o ./target/debug/coverage/lcov.info | |
| - name: Check Rust formatting | |
| working-directory: ./rust | |
| run: | | |
| cargo fmt -- --check | |
| - name: Check Clippy | |
| working-directory: ./rust | |
| run: | | |
| cargo clippy-ci | |
| - name: Upload coverage reports to Codecov | |
| uses: codecov/codecov-action@v4 | |
| with: | |
| files: ./test/profile.cov,./rust/target/debug/coverage/lcov.info | |
| env: | |
| CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} | |
| lint: | |
| name: Lint | |
| runs-on: ubuntu-24.04 | |
| timeout-minutes: 10 | |
| env: | |
| GOPATH: /home/runner/go | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Setup Golang | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: "1.24" | |
| - name: Restore Go build cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/go-build | |
| key: ${{ runner.os }}-go-build-v1-${{ github.run_id }} | |
| - run: make lint | |
| - run: git diff --exit-code | |
| build-rust-amd64: | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| # TODO: Use actions/cache@v4 to cache target/ directory | |
| - name: Build binary | |
| run: make build-rust-docker-ghactions | |
| - name: Rename binaries | |
| run: | | |
| cp -pv rust/target/x86_64-unknown-linux-gnu/release/numaflow ./numaflow-rs-linux-amd64 | |
| cp -pv rust/target/x86_64-unknown-linux-gnu/release/entrypoint ./entrypoint-linux-amd64 | |
| - name: Verify numaflow binary is Statically Linked | |
| run: | | |
| file ./numaflow-rs-linux-amd64 | |
| file ./numaflow-rs-linux-amd64 | grep -q 'static-pie linked' | |
| - name: Verify entrypoint binary is Statically Linked | |
| run: | | |
| file ./entrypoint-linux-amd64 | |
| file ./entrypoint-linux-amd64 | grep -q 'static-pie linked' | |
| - name: Upload numaflow binary | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: numaflow-rs-linux-amd64 | |
| path: numaflow-rs-linux-amd64 | |
| if-no-files-found: error | |
| - name: Upload entrypoint binary | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: entrypoint-linux-amd64 | |
| path: entrypoint-linux-amd64 | |
| if-no-files-found: error | |
| e2e-tests: | |
| name: E2E Tests | |
| runs-on: ubuntu-24.04 | |
| needs: [build-rust-amd64] | |
| timeout-minutes: 20 | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 13 | |
| matrix: | |
| driver: [jetstream] | |
| case: | |
| [ | |
| e2e, | |
| diamond-e2e, | |
| transformer-e2e, | |
| kafka-e2e, | |
| map-e2e, | |
| reduce-one-e2e, | |
| reduce-two-e2e, | |
| udsource-e2e, | |
| api-e2e, | |
| sideinputs-e2e, | |
| idle-source-e2e, | |
| monovertex-e2e, | |
| builtin-source-e2e, | |
| serving-e2e, | |
| ] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Restore go build cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/go-build | |
| key: ${{ runner.os }}-go-build-v2-${{ hashFiles('**/go.mod') }} | |
| - name: Setup Golang | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: "1.24" | |
| - name: Add bins to PATH | |
| run: | | |
| echo /home/runner/go/bin >> $GITHUB_PATH | |
| echo /usr/local/bin >> $GITHUB_PATH | |
| - name: Setup Node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20" | |
| - name: Setup Node-Cache | |
| uses: actions/cache@v4 | |
| with: | |
| path: ui/node_modules | |
| key: ${{ runner.os }}-node-dep-v1-${{ hashFiles('**/yarn.lock') }} | |
| - name: Download Rust amd64 binaries | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: numaflow-rs-linux-amd64 | |
| - name: Download Entrypoint amd64 binaries | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: entrypoint-linux-amd64 | |
| - name: Install k3d | |
| run: curl -sfL https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash & | |
| - name: Create a cluster | |
| run: | | |
| k3d cluster create e2e | |
| k3d kubeconfig get e2e > ~/.kube/numaflow-e2e-config | |
| - name: Install Numaflow | |
| env: | |
| GOPATH: /home/runner/go | |
| run: | | |
| KUBECONFIG=~/.kube/numaflow-e2e-config VERSION=${{ github.sha }} make start | |
| - name: Run tests | |
| env: | |
| GOPATH: /home/runner/go | |
| run: KUBECONFIG=~/.kube/numaflow-e2e-config VERSION=${{ github.sha }} ISBSVC=${{matrix.driver}} SKIP_IMAGE_BUILD=true make test-${{matrix.case}} |