Bug 1971771 [wpt PR 53030] - Move Safari PR jobs to GH Actions, a=testonly

Automatic update from web-platform-tests
Migrate PR jobs using Safari to GitHub Actions

This adds new jobs using Safari to GitHub Actions, using the same
`safari-wptrunner.yml` as the full runs to try and make these as
similar as possible.

And then, this removes the corresponding jobs from Azure Pipelines and
removed related, now unused, actions.

Additionally, this also removes the Azure Pipelines actions
`install_fonts.yml` and `pip_install.yml`; these have been unused
since bf5d16e1d7 (2019!) and a11a6e0e7a (2023) respectively.

--

wpt-commits: c86c9fd558a3691364981c5633a924b9599d7141
wpt-pr: 53030

testing/web-platform/tests/.azure-pipelines.yml

@@ -27,44 +27,6 @@ variables:
   safaridriver_diagnose: false
 
 jobs:
-# The affected tests jobs are unconditional for speed, as most PRs have one or
-# more affected tests: https://github.com/web-platform-tests/wpt/issues/13936.
-- job: affected_safari_preview
-  displayName: 'affected tests: Safari Technology Preview'
-  condition: eq(variables['Build.Reason'], 'PullRequest')
-  pool:
-    vmImage: 'macOS-14'
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.12'
-  - template: tools/ci/azure/affected_tests.yml
-    parameters:
-      artifactName: 'safari-preview-affected-tests'
-- template: tools/ci/azure/fyi_hook.yml
-  parameters:
-    dependsOn: affected_safari_preview
-    artifactName: safari-preview-affected-tests
-
-- job: affected_without_changes_safari_preview
-  displayName: 'affected tests without changes: Safari Technology Preview'
-  condition: eq(variables['Build.Reason'], 'PullRequest')
-  pool:
-    vmImage: 'macOS-14'
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.13'
-  - template: tools/ci/azure/affected_tests.yml
-    parameters:
-      checkoutCommit: 'HEAD^1'
-      affectedRange: 'HEAD@{1}'
-      artifactName: 'safari-preview-affected-tests-without-changes'
-- template: tools/ci/azure/fyi_hook.yml
-  parameters:
-    dependsOn: affected_without_changes_safari_preview
-    artifactName: safari-preview-affected-tests-without-changes
-
 # The decision jobs runs `./wpt test-jobs` to determine which jobs to run,
 # and all following jobs wait for it to finish and depend on its output.
 - job: decision
@@ -87,37 +49,6 @@ jobs:
     name: test_jobs
     displayName: 'Run ./wpt test-jobs'
 
-- job: infrastructure_mac
-  displayName: 'infrastructure/ tests: macOS'
-  dependsOn: decision
-  condition: dependencies.decision.outputs['test_jobs.wptrunner_infrastructure']
-  pool:
-    vmImage: 'macOS-14'
-  steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.13'
-  - template: tools/ci/azure/checkout.yml
-  - template: tools/ci/azure/install_fonts.yml
-  - template: tools/ci/azure/install_certs.yml
-  - template: tools/ci/azure/color_profile.yml
-  - template: tools/ci/azure/install_safari.yml
-  - template: tools/ci/azure/update_hosts.yml
-  - template: tools/ci/azure/update_manifest.yml
-  - script: |
-      set -eux -o pipefail
-      export SYSTEM_VERSION_COMPAT=0
-      ./wpt run --yes --no-manifest-update --manifest MANIFEST.json --metadata infrastructure/metadata/ --log-mach - --log-mach-level info  --log-wptreport $(Build.ArtifactStagingDirectory)/wpt_report_macos_safari.json --channel preview safari infrastructure/
-    condition: succeededOrFailed()
-    displayName: 'Run tests (Safari Technology Preview)'
-  - task: PublishBuildArtifacts@1
-    condition: succeededOrFailed()
-    displayName: 'Publish results'
-    inputs:
-      artifactName: 'infrastructure-results'
-  - template: tools/ci/azure/publish_logs.yml
-  - template: tools/ci/azure/sysdiagnose.yml
-
 - job: tools_unittest_mac_py38
   displayName: 'tools/ unittests: macOS + Python 3.8'
   dependsOn: decision

testing/web-platform/tests/.github/workflows/pull_request_test_jobs.yml

@@ -0,0 +1,61 @@
+name: test-jobs
+on:
+  pull_request:
+
+jobs:
+  decision:
+    name: ./wpt test-jobs
+    runs-on: ubuntu-24.04
+    outputs:
+      test_jobs: ${{ steps.test_jobs.outputs.test_jobs }}
+    steps:
+      - name: checkout
+        uses: actions/checkout@v4.1.0
+        with:
+          fetch-depth: 2
+      - name: ./wpt test-jobs
+        id: test_jobs
+        run: |
+          set -eux
+          echo "test_jobs=$( ./wpt test-jobs --json HEAD^1..HEAD )" >> $GITHUB_OUTPUT
+
+  affected_safari_preview:
+    name: "affected tests: Safari Technology Preview"
+    needs: decision
+    if: contains(fromJSON(needs.decision.outputs.test_jobs), 'affected_tests')
+    uses: ./.github/workflows/safari-wptrunner.yml
+    with:
+      artifact-name: safari-preview-affected-tests-with-changes
+      merged-artifact-name: safari-preview-affected-tests
+      safari-technology-preview: true
+      safaridriver-diagnose: false
+      fetch-depth: 2
+      extra-options: --affected ${{ github.sha }}^1
+
+  affected_without_changes_safari_preview:
+    name: "affected tests without changes: Safari Technology Preview"
+    needs: decision
+    if: contains(fromJSON(needs.decision.outputs.test_jobs), 'affected_tests')
+    uses: ./.github/workflows/safari-wptrunner.yml
+    with:
+      artifact-name: safari-preview-affected-tests-without-changes
+      merged-artifact-name: safari-preview-affected-tests-without-changes
+      safari-technology-preview: true
+      safaridriver-diagnose: false
+      fetch-depth: 2
+      test-rev: HEAD^1
+      extra-options: --affected ${{ github.sha }}
+
+  infrastructure_mac:
+    name: "infrastructure/ tests: macOS"
+    needs: decision
+    if: contains(fromJSON(needs.decision.outputs.test_jobs), 'wptrunner_infrastructure')
+    uses: ./.github/workflows/safari-wptrunner.yml
+    with:
+      artifact-name: safari-infrastructure-results
+      safari-technology-preview: true
+      safaridriver-diagnose: false
+      extra-options: >-
+        --manifest MANIFEST.json
+        --metadata infrastructure/metadata/
+        --include infrastructure/

testing/web-platform/tests/.github/workflows/safari-wptrunner.yml

@@ -4,38 +4,70 @@ on:
   workflow_call:
     inputs:
       artifact-name:
-        description: "Prefix for the artifact uploaded"
+        description: Prefix for the artifact uploaded
         required: true
         type: string
+      merged-artifact-name:
+        description: Merge the uploaded artifacts into a singular artifact
+        required: false
+        type: string
       safari-technology-preview:
-        description: "Run Safari Technology Preview rather than the system Safari"
+        description: Run Safari Technology Preview rather than the system Safari
         required: true
         type: boolean
       safaridriver-diagnose:
-        description: "Run safaridriver capturing diagnostics"
+        description: Run safaridriver capturing diagnostics
         required: true
         type: boolean
+      fetch-ref:
+        description: The ref to fetch and initially checkout
+        required: false
+        type: string
+      fetch-depth:
+        description: The fetch-depth to checkout
+        required: false
+        type: number
+      test-rev:
+        description: The rev to checkout before running the tests
+        required: false
+        type: string
+      matrix:
+        description: Test matrix, to override test-type/current-chunk/total-chunks
+        required: false
+        type: string
+      extra-options:
+        description: Extra options to pass to wpt run
+        required: false
+        type: string
 
 # We never interact with the GitHub API, thus we can simply disable all
 # permissions the GitHub token would have.
 permissions: {}
 
 jobs:
-  safari-results:
-    name: ${{ matrix.current-chunk }} (of ${{ matrix.total-chunks }})
+  results:
+    name: ${{ matrix.current-chunk || 1 }} (of ${{ matrix.total-chunks || 1 }})
+    env:
+      CURRENT_CHUNK: ${{ matrix.current-chunk || 1 }}
+      TOTAL_CHUNKS: ${{ matrix.total-chunks || 1 }}
     runs-on:
       - self-hosted
       - webkit-ews
     timeout-minutes: 180
     strategy:
-      matrix:
-        current-chunk: [1, 2, 3, 4, 5, 6, 7, 8]
-        total-chunks: [8]
+      matrix: "${{ fromJSON(inputs.matrix || '{\"pointless-matrix-item\": [0]}') }}"
     steps:
       - name: checkout
         uses: actions/checkout@v4.1.0
         with:
-          fetch-depth: 1
+          fetch-depth: ${{ inputs.fetch-depth || 1 }}
+          ref: ${{ inputs.fetch-ref }}
+      - name: test-checkout
+        if: inputs.test-rev
+        env:
+          REV: ${{ inputs.test-rev }}
+        run: |-
+          git switch --force --progress -d "$REV"
       - name: Set display color profile
         run: |-
           ./wpt macos-color-profile
@@ -71,30 +103,33 @@ jobs:
             --no-manifest-update \
             --no-restart-on-unexpected \
             --no-fail-on-unexpected \
-            --this-chunk=${{ matrix.current-chunk }} \
-            --total-chunks=${{ matrix.total-chunks }} \
+            --no-pause \
+            --this-chunk "$CURRENT_CHUNK" \
+            --total-chunks "$TOTAL_CHUNKS" \
             --chunk-type hash \
-            --log-wptreport ${{ runner.temp }}/wpt_report_${{ matrix.current-chunk }}.json \
-            --log-wptscreenshot ${{ runner.temp }}/wpt_screenshot_${{ matrix.current-chunk }}.txt \
+            --log-wptreport ${{ runner.temp }}/wpt_report_"$CURRENT_CHUNK".json \
+            --log-wptscreenshot ${{ runner.temp }}/wpt_screenshot_"$CURRENT_CHUNK".txt \
             --log-mach - \
             --log-mach-level info \
             --channel ${{ inputs.safari-technology-preview && 'preview' || 'stable' }} \
             --kill-safari \
             --max-restarts 100 \
+            ${{ inputs.extra-options }} \
+            -- \
             safari
       - name: Publish results
         uses: actions/upload-artifact@v4.1.0
         with:
-          name: ${{ inputs.artifact-name }}-${{ matrix.current-chunk }}
+          name: ${{ inputs.artifact-name }}-${{ env.CURRENT_CHUNK }}
           path: |
             ${{ runner.temp }}/wpt_report_*.json
             ${{ runner.temp }}/wpt_screenshot_*.txt
-          if-no-files-found: "error"
+          if-no-files-found: error
       - name: Publish safaridriver logs
         if: inputs.safaridriver-diagnose
         uses: actions/upload-artifact@v4.1.0
         with:
-          name: ${{ inputs.artifact-name }}-safaridriver-logs-${{ matrix.current-chunk }}
+          name: ${{ inputs.artifact-name }}-safaridriver-logs-${{ env.CURRENT_CHUNK }}
           path: ~/Library/Logs/com.apple.WebDriver/
           if-no-files-found: warn
       - name: Disable safaridriver diagnostics
@@ -105,11 +140,33 @@ jobs:
       - name: Cleanup
         if: always()
         run: |-
-            set -ux
-            sudo sed -i '' '/^# Start web-platform-tests hosts$/,/^# End web-platform-tests hosts$/d' /etc/hosts
+          set -ux
+          sudo sed -i '' '/^# Start web-platform-tests hosts$/,/^# End web-platform-tests hosts$/d' /etc/hosts
 
-  safari-notify:
-    needs: safari-results
+  notify-unmerged:
+    name: Notify (unmerged results)
+    needs: results
+    if: "! inputs.merged-artifact-name"
     uses: ./.github/workflows/wpt_fyi_notify.yml
     with:
-      artifact-name: "${{ inputs.artifact-name }}-*"
+      artifact-name: ${{ inputs.artifact-name }}-*
+
+  merge-results:
+    name: Merge results artifacts
+    runs-on: ubuntu-24.04
+    needs: results
+    if: inputs.merged-artifact-name
+    steps:
+      - name: Merge Artifacts
+        uses: actions/upload-artifact/merge@v4
+        with:
+          name: ${{ inputs.merged-artifact-name }}
+          pattern: ${{ inputs.artifact-name }}-*
+
+  notify-merged:
+    name: Notify (merged results)
+    needs: merge-results
+    if: inputs.merged-artifact-name
+    uses: ./.github/workflows/wpt_fyi_notify.yml
+    with:
+      artifact-name: ${{ inputs.merged-artifact-name }}

testing/web-platform/tests/.github/workflows/safari_stable.yml

@@ -7,7 +7,8 @@ permissions: {}
 on:
   workflow_dispatch:
   workflow_run:
-    workflows: [epochs]
+    workflows:
+      - epochs
     types:
       - completed
   push:
@@ -17,7 +18,7 @@ on:
 
 jobs:
   check-workflow-run:
-    name: "Check for appropriate epochs"
+    name: Check for appropriate epochs
     uses: ./.github/workflows/check-workflow-run.yml
     with:
       check-refs: '["refs/heads/epochs/daily"]'
@@ -31,6 +32,7 @@ jobs:
       github.event_name != 'workflow_run' || fromJSON(needs.check-workflow-run.outputs.updated-refs)[0] != null
     uses: ./.github/workflows/safari-wptrunner.yml
     with:
-      artifact-name: "safari-results"
+      artifact-name: safari-results
       safari-technology-preview: false
       safaridriver-diagnose: false
+      matrix: '{"current-chunk": [1, 2, 3, 4, 5, 6, 7, 8], "total-chunks": [8]}'

testing/web-platform/tests/.github/workflows/safari_technology_preview.yml

@@ -7,7 +7,8 @@ permissions: {}
 on:
   workflow_dispatch:
   workflow_run:
-    workflows: [epochs]
+    workflows:
+      - epochs
     types:
       - completed
   push:
@@ -17,7 +18,7 @@ on:
 
 jobs:
   check-workflow-run:
-    name: "Check for appropriate epochs"
+    name: Check for appropriate epochs
     uses: ./.github/workflows/check-workflow-run.yml
     with:
       check-refs: '["refs/heads/epochs/three_hourly"]'
@@ -31,6 +32,7 @@ jobs:
       github.event_name != 'workflow_run' || fromJSON(needs.check-workflow-run.outputs.updated-refs)[0] != null
     uses: ./.github/workflows/safari-wptrunner.yml
     with:
-      artifact-name: "safari-technology-preview-results"
+      artifact-name: safari-technology-preview-results
       safari-technology-preview: true
       safaridriver-diagnose: false
+      matrix: '{"current-chunk": [1, 2, 3, 4, 5, 6, 7, 8], "total-chunks": [8]}'

testing/web-platform/tests/tools/ci/azure/affected_tests.yml

@@ -1,27 +0,0 @@
-parameters:
-  checkoutCommit: ''
-  affectedRange: 'HEAD^1'
-  artifactName: ''
-
-steps:
-- template: checkout.yml
-- ${{ if ne(parameters.checkoutCommit, '') }}:
-  - script: |
-      set -eux -o pipefail
-      git checkout ${{ parameters.checkoutCommit }}
-    displayName: 'Checkout ${{ parameters.checkoutCommit }}'
-- template: install_certs.yml
-- template: color_profile.yml
-- template: install_safari.yml
-- template: update_hosts.yml
-- template: update_manifest.yml
-- script: |
-    set -eux -o pipefail
-      export SYSTEM_VERSION_COMPAT=0
-    ./wpt run --yes --no-pause --no-fail-on-unexpected --no-restart-on-unexpected --affected ${{ parameters.affectedRange }} --log-wptreport $(Build.ArtifactStagingDirectory)/wpt_report.json --log-wptscreenshot $(Build.ArtifactStagingDirectory)/wpt_screenshot.txt --channel preview --kill-safari safari
-  displayName: 'Run tests'
-- task: PublishBuildArtifacts@1
-  displayName: 'Publish results'
-  inputs:
-    artifactName: '${{ parameters.artifactName }}'
-  condition: succeededOrFailed()

testing/web-platform/tests/tools/ci/azure/com.apple.SafariTechnologyPreview.plist

@@ -1,8 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
-<plist version="1.0">
-<dict>
-	<key>AllowRemoteAutomation</key>
-	<true/>
-</dict>
-</plist>

testing/web-platform/tests/tools/ci/azure/install_fonts.yml

@@ -1,7 +0,0 @@
-steps:
-# Installig Ahem in /Library/Fonts instead of using --install-fonts is a
-# workaround for https://github.com/web-platform-tests/wpt/issues/13803.
-- script: |
-    set -eux -o pipefail
-    sudo cp fonts/Ahem.ttf /Library/Fonts
-  displayName: 'Install Ahem font'

testing/web-platform/tests/tools/ci/azure/install_safari.yml

@@ -1,29 +0,0 @@
-parameters:
-  channel: preview
-
-# Should match https://web-platform-tests.org/running-tests/safari.html
-steps:
-- script: defaults write com.apple.WebDriver DiagnosticsEnabled 1
-  displayName: 'Enable safaridriver diagnostics'
-  condition: eq(variables['safaridriver_diagnose'], true)
-- ${{ if eq(parameters.channel, 'preview') }}:
-  - script: |
-      set -eux -o pipefail
-      export SYSTEM_VERSION_COMPAT=0
-      ./wpt install --channel preview --download-only -d . --rename STP safari browser
-      sudo installer -pkg STP.pkg -target LocalSystem
-      # Workaround for `sudo safardriver --enable` not working on Catalina:
-      # https://github.com/web-platform-tests/wpt/issues/21751
-      mkdir -p ~/Library/WebDriver/
-      cp tools/ci/azure/com.apple.SafariTechnologyPreview.plist ~/Library/WebDriver/
-      defaults write com.apple.SafariTechnologyPreview WebKitJavaScriptCanOpenWindowsAutomatically 1
-      defaults write com.apple.SafariTechnologyPreview ExperimentalServerTimingEnabled 1
-    displayName: 'Install Safari Technology Preview'
-- ${{ if eq(parameters.channel, 'stable') }}:
-  - script: |
-      set -eux -o pipefail
-      export SYSTEM_VERSION_COMPAT=0
-      sudo softwareupdate --install $( softwareupdate -l | grep -o '\* Label: \(Safari.*\)' | sed -e 's/* Label: //' )
-      sudo safaridriver --enable
-      defaults write com.apple.Safari WebKitJavaScriptCanOpenWindowsAutomatically 1
-    displayName: 'Configure Safari'

testing/web-platform/tests/tools/ci/azure/pip_install.yml

@@ -1,6 +0,0 @@
-parameters:
-  packages: ''
-
-steps:
-- script: pip --disable-pip-version-check install --upgrade ${{ parameters.packages }}
-  displayName: 'Install Python packages'