99 lines
3.9 KiB
YAML
99 lines
3.9 KiB
YAML
# Run all tests, including flaky test.
|
|
#
|
|
# The default CI workflow ignores flaky tests. This workflow will run
|
|
# all tests, including ignored ones.
|
|
#
|
|
# To use this workflow you can either:
|
|
#
|
|
# - Label a PR with "flaky-test", the normal CI workflow will not run
|
|
# any jobs but the jobs here will be run. Note that to merge the PR
|
|
# you'll need to remove the label eventually because the normal CI
|
|
# jobs are required by branch protection.
|
|
#
|
|
# - Manually trigger the workflow, you may choose a branch for this to
|
|
# run on.
|
|
#
|
|
# Additionally this jobs runs once a day on a schedule.
|
|
#
|
|
# Currently doctests are not run by this workflow.
|
|
|
|
name: Flaky CI
|
|
|
|
on:
|
|
pull_request:
|
|
types: [ 'labeled', 'unlabeled', 'opened', 'synchronize', 'reopened' ]
|
|
schedule:
|
|
# 06:30 UTC every day
|
|
- cron: '30 6 * * *'
|
|
workflow_dispatch:
|
|
inputs:
|
|
branch:
|
|
description: 'Branch to run on, defaults to main'
|
|
required: true
|
|
default: 'main'
|
|
type: string
|
|
|
|
concurrency:
|
|
group: flaky-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
env:
|
|
IROH_FORCE_STAGING_RELAYS: "1"
|
|
|
|
jobs:
|
|
tests:
|
|
if: "contains(github.event.pull_request.labels.*.name, 'flaky-test') || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'"
|
|
uses: './.github/workflows/tests.yaml'
|
|
with:
|
|
flaky: true
|
|
git-ref: ${{ inputs.branch }}
|
|
notify:
|
|
needs: tests
|
|
if: ${{ always() }}
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Extract test results
|
|
run: |
|
|
printf '${{ toJSON(needs) }}\n'
|
|
result=$(echo '${{ toJSON(needs) }}' | jq -r .tests.result)
|
|
echo TESTS_RESULT=$result
|
|
echo "TESTS_RESULT=$result" >>"$GITHUB_ENV"
|
|
- name: download nextest reports
|
|
uses: actions/download-artifact@v5
|
|
with:
|
|
pattern: libtest_run_${{ github.run_number }}-${{ github.run_attempt }}-*
|
|
merge-multiple: true
|
|
path: nextest-results
|
|
- name: create summary report
|
|
id: make_summary
|
|
run: |
|
|
# prevent the glob expression in the loop to match on itself when the dir is empty
|
|
shopt -s nullglob
|
|
# to deal with multiline outputs it's recommended to use a random EOF, the syntax is based on
|
|
# https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings
|
|
EOF=aP51VriWCxNJ1JjvmO9i
|
|
echo "summary<<$EOF" >> $GITHUB_OUTPUT
|
|
echo "Flaky tests failure:" >> $GITHUB_OUTPUT
|
|
echo " " >> $GITHUB_OUTPUT
|
|
for report in nextest-results/*.json; do
|
|
# remove the name prefix and extension, and split the parts
|
|
name=$(echo ${report:16:-5} | tr _ ' ')
|
|
echo $name
|
|
echo "- **$name**" >> $GITHUB_OUTPUT
|
|
# select the failed tests
|
|
# the tests have this format "crate::module$test_name", the sed expressions remove the quotes and replace $ for ::
|
|
failure=$(jq --slurp '.[] | select(.["type"] == "test" and .["event"] == "failed" ) | .["name"]' $report | sed -e 's/^"//g' -e 's/\$/::/' -e 's/"//')
|
|
echo "$failure"
|
|
echo "$failure" >> $GITHUB_OUTPUT
|
|
done
|
|
echo "" >> $GITHUB_OUTPUT
|
|
echo "See https://github.com/${{ github.repository }}/actions/workflows/flaky.yaml" >> $GITHUB_OUTPUT
|
|
echo "$EOF" >> $GITHUB_OUTPUT
|
|
- name: Notify discord on failure
|
|
uses: n0-computer/discord-webhook-notify@v1
|
|
if: ${{ env.TESTS_RESULT == 'failure' || env.TESTS_RESULT == 'success' }}
|
|
with:
|
|
text: "Flaky tests in **${{ github.repository }}**:"
|
|
severity: ${{ env.TESTS_RESULT == 'failure' && 'warn' || 'info' }}
|
|
details: ${{ env.TESTS_RESULT == 'failure' && steps.make_summary.outputs.summary || 'No flaky failures!' }}
|
|
webhookUrl: ${{ secrets.DISCORD_N0_GITHUB_CHANNEL_WEBHOOK_URL }}
|