diff --git a/.github/workflows/generate-reference-results-manual.yml b/.github/workflows/generate-reference-results-manual.yml index 413223f66..8a9933397 100644 --- a/.github/workflows/generate-reference-results-manual.yml +++ b/.github/workflows/generate-reference-results-manual.yml @@ -15,6 +15,17 @@ on: description: 'Commit message' default: "Add reference results" type: string + tutorials_ref_source: + description: 'Use tutorials from' + default: 'reference_versions.yaml' + type: choice + options: + - 'reference_versions.yaml' + - 'workflow branch' + build_args: + description: 'Optional overrides for reference_versions.yaml (e.g. TUTORIALS_REF:my-branch)' + required: false + type: string clean_docker: description: 'Run docker-system-prune before running tests' default: 'FALSE' @@ -34,8 +45,10 @@ jobs: generate_reference_results_manual: uses: ./.github/workflows/generate-reference-results.yml with: + suites: ${{ inputs.suites }} from_ref: ${{ inputs.from_ref }} commit_msg: ${{ inputs.commit_msg }} - suites: ${{ inputs.suites }} + tutorials_ref_source: ${{ inputs.tutorials_ref_source }} + build_args: ${{ inputs.build_args }} clean_docker: ${{ inputs.clean_docker }} log_level: ${{ inputs.log_level }} \ No newline at end of file diff --git a/.github/workflows/generate-reference-results.yml b/.github/workflows/generate-reference-results.yml index fd9493fa5..6d94b8b6a 100644 --- a/.github/workflows/generate-reference-results.yml +++ b/.github/workflows/generate-reference-results.yml @@ -14,6 +14,14 @@ on: description: 'Commit message' default: "Add reference results" type: string + tutorials_ref_source: + description: 'Where to take TUTORIALS_REF from when build_args is empty' + default: 'reference_versions.yaml' + type: string + build_args: + description: 'Comma-separated overrides for reference_versions.yaml (e.g. TUTORIALS_REF:my-branch)' + default: '' + type: string clean_docker: description: 'Run docker-system-prune before running tests' default: 'FALSE' @@ -33,9 +41,16 @@ jobs: echo "- Test suites: \`${{ inputs.suites || 'all (no filter)' }}\`" echo "- Git branch to commit to: \`${{ inputs.from_ref }}\`" echo "- Commit message: \`${{ inputs.commit_msg }}\`" + if [ -n "${{ inputs.build_args }}" ]; then + echo "- Build args override: \`${{ inputs.build_args }}\`" + elif [ "${{ inputs.tutorials_ref_source }}" = "workflow branch" ]; then + echo "- Tutorials source: workflow branch (\`TUTORIALS_REF:${{ inputs.from_ref }}\`)" + else + echo "- Tutorials source: \`reference_versions.yaml\`" + fi echo "- Run \`docker-system-prune\` before running tests: \`${{ inputs.clean_docker }}\`" echo "- Log level: \`${{ inputs.log_level }}\`" - echo "- Running \`generate_reference_results.py --suite ${{inputs.suites}} --log_level ${{inputs.log_level}}\`" + echo "- Running \`generate_reference_results.py --suites ${{ inputs.suites }} --log_level ${{ inputs.log_level }}\`" } >> "$GITHUB_STEP_SUMMARY" - name: Move LFS URL to local LFS server run: | @@ -70,7 +85,15 @@ jobs: if [ -n "${{ inputs.suites }}" ]; then SUITES_ARGS+=(--suites "${{ inputs.suites }}") fi - python generate_reference_results.py --log_level="${{inputs.log_level}}" "${SUITES_ARGS[@]}" + BUILD_ARGS_STR="${{ inputs.build_args }}" + if [ -z "$BUILD_ARGS_STR" ] && [ "${{ inputs.tutorials_ref_source }}" = "workflow branch" ]; then + BUILD_ARGS_STR="TUTORIALS_REF:${{ inputs.from_ref }}" + fi + BUILD_ARGS=() + if [ -n "$BUILD_ARGS_STR" ]; then + BUILD_ARGS=(--build_args "$BUILD_ARGS_STR") + fi + python generate_reference_results.py --log_level="${{ inputs.log_level }}" "${BUILD_ARGS[@]}" "${SUITES_ARGS[@]}" cd ../../ - name: Create commit if: success() diff --git a/changelog-entries/844.md b/changelog-entries/844.md new file mode 100644 index 000000000..d1ecd82ed --- /dev/null +++ b/changelog-entries/844.md @@ -0,0 +1 @@ +- `generate_reference_results.py` accepts `--build_args` to override values from `reference_versions.yaml` (e.g. `TUTORIALS_REF` on a feature branch). The generate-reference-results workflow uses `reference_versions.yaml` by default and can optionally use the workflow branch ([#869](https://github.com/precice/tutorials/pull/869)). diff --git a/tools/tests/README.md b/tools/tests/README.md index 6f48f7461..f9b430a9a 100644 --- a/tools/tests/README.md +++ b/tools/tests/README.md @@ -166,7 +166,11 @@ run-before: ./set-case.sh 1d3d You will need to define a reference results file. The reference results can and should be generated on GitHub using the [Generate reference results (manual)](https://github.com/precice/tutorials/actions/workflows/generate-reference-results-manual.yml) workflow for the respective test suite. You might want to temporarily set the `selected` test suite for requesting results only for a subset of test cases. -Note that you will need to define the `TUTORIALS_REF` in the file [`reference_versions.yaml`](https://github.com/precice/tutorials/actions/workflows/generate-reference-results-manual.yml) to match the respective branch. Restore that to `develop` after that. See a [related issue](https://github.com/precice/tutorials/issues/844). +By default, the [Generate reference results (manual)](https://github.com/precice/tutorials/actions/workflows/generate-reference-results-manual.yml) workflow uses `TUTORIALS_REF` from [`reference_versions.yaml`](https://github.com/precice/tutorials/blob/develop/tools/tests/reference_versions.yaml). For a feature branch, set Use tutorials from to `workflow branch`, or pass `TUTORIALS_REF` via `--build_args` (locally) or the optional `build_args` workflow input. + +{% note %} +The two options cannot be combined: defining any overrides to `reference_versions.yaml` will ignore the option to use the tutorials from the workflow branch. +{% endnote %} The results will be added to a Git LFS, but you will need special push access: just use the aforementioned GitHub Actions workflow, instead. diff --git a/tools/tests/generate_reference_results.py b/tools/tests/generate_reference_results.py index b52486b75..0787cc6e4 100644 --- a/tools/tests/generate_reference_results.py +++ b/tools/tests/generate_reference_results.py @@ -118,6 +118,11 @@ def main(): 'If not specified, all suites are used.') parser.add_argument('--log_level', choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], default='INFO', help='Set the logging level') + parser.add_argument( + '--build_args', + type=str, + help='Comma-separated overrides for reference_versions.yaml ' + '(e.g., "TUTORIALS_REF:my-feature-branch")') args = parser.parse_args() @@ -162,8 +167,13 @@ def main(): test_suites = all_test_suites logging.info("No --suites filter specified, generating reference results for all suites.") - # Read in parameters + # Read in parameters from reference_versions.yaml, applying any CLI overrides. build_args = SystemtestArguments.from_yaml(PRECICE_TESTS_DIR / "reference_versions.yaml") + overrides = SystemtestArguments.from_args(args.build_args) + if overrides.arguments: + build_args = SystemtestArguments( + {**build_args.arguments, **overrides.arguments}) + logging.info(f"Applied build argument overrides: {overrides}") systemtests_to_run = set() for test_suite in test_suites: