shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[CI] Move CI over to new project computation script

Browse Source 
This patch migrates the CI over to the new compute_projects.py script
for calculating what projects need to be tested based on a change to
LLVM.

Reviewers: lnihlen, ldionne, tstellar, Endilll, joker-eph, Keenuts

Reviewed By: Keenuts, tstellar

Pull Request: https://github.com/llvm/llvm-project/pull/132642
This commit is contained in:
Aiden Grossman 2025-03-28 22:25:52 -07:00 committed by GitHub
parent 40c1d50024
commit 34d858635f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 27 additions and 279 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

194
.ci/compute-projects.sh
View File

@ -1,194 +0,0 @@
#!/usr/bin/env bash
#===----------------------------------------------------------------------===##
#
# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#
#===----------------------------------------------------------------------===##
#
# This file contains functions to compute which projects should be built by CI
# systems and is intended to provide common functionality applicable across
# multiple systems during a transition period.
#
function compute-projects-to-test() {
isForWindows=$1
shift
projects=${@}
for project in ${projects}; do
echo "${project}"
case ${project} in
lld)
for p in bolt cross-project-tests; do
echo $p
done
;;
llvm)
for p in bolt clang clang-tools-extra lld lldb mlir polly; do
echo $p
done
# Flang is not stable in Windows CI at the moment
if [[ $isForWindows == 0 ]]; then
echo flang
fi
;;
clang)
# lldb is temporarily removed to alleviate Linux pre-commit CI waiting times
for p in clang-tools-extra compiler-rt cross-project-tests; do
echo $p
done
;;
clang-tools-extra)
echo libc
;;
mlir)
# Flang is not stable in Windows CI at the moment
if [[ $isForWindows == 0 ]]; then
echo flang
fi
;;
*)
# Nothing to do
;;
esac
done
}
function compute-runtimes-to-test() {
projects=${@}
for project in ${projects}; do
case ${project} in
clang)
for p in libcxx libcxxabi libunwind; do
echo $p
done
;;
*)
# Nothing to do
;;
esac
done
}
function add-dependencies() {
projects=${@}
for project in ${projects}; do
echo "${project}"
case ${project} in
bolt)
for p in clang lld llvm; do
echo $p
done
;;
cross-project-tests)
for p in lld clang; do
echo $p
done
;;
clang-tools-extra)
for p in llvm clang; do
echo $p
done
;;
compiler-rt|libc|openmp)
echo clang lld
;;
flang|lldb|libclc)
for p in llvm clang; do
echo $p
done
;;
lld|mlir|polly)
echo llvm
;;
*)
# Nothing to do
;;
esac
done
}
function exclude-linux() {
projects=${@}
for project in ${projects}; do
case ${project} in
cross-project-tests) ;; # tests failing
openmp) ;; # https://github.com/google/llvm-premerge-checks/issues/410
*)
echo "${project}"
;;
esac
done
}
function exclude-windows() {
projects=${@}
for project in ${projects}; do
case ${project} in
cross-project-tests) ;; # tests failing
compiler-rt) ;; # tests taking too long
openmp) ;; # TODO: having trouble with the Perl installation
libc) ;; # no Windows support
lldb) ;; # custom environment requirements (https://github.com/llvm/llvm-project/pull/94208#issuecomment-2146256857)
bolt) ;; # tests are not supported yet
*)
echo "${project}"
;;
esac
done
}
# Prints only projects that are both present in $modified_dirs and the passed
# list.
function keep-modified-projects() {
projects=${@}
for project in ${projects}; do
if echo "$modified_dirs" | grep -q -E "^${project}$"; then
echo "${project}"
fi
done
}
function check-targets() {
# Do not use "check-all" here because if there is "check-all" plus a
# project specific target like "check-clang", that project's tests
# will be run twice.
projects=${@}
for project in ${projects}; do
case ${project} in
clang-tools-extra)
echo "check-clang-tools"
;;
compiler-rt)
echo "check-compiler-rt"
;;
cross-project-tests)
echo "check-cross-project"
;;
libcxx)
echo "check-cxx"
;;
libcxxabi)
echo "check-cxxabi"
;;
libunwind)
echo "check-unwind"
;;
lldb)
echo "check-lldb"
;;
pstl)
# Currently we do not run pstl tests in CI.
;;
libclc)
# Currently there is no testing for libclc.
;;
*)
echo "check-${project}"
;;
esac
done
}

22
.ci/generate-buildkite-pipeline-premerge
View File

@ -52,8 +52,6 @@ modified_dirs=$(echo "$MODIFIED_FILES" | cut -d'/' -f1 | sort -u)
echo "Directories modified:" >&2 echo "Directories modified:" >&2
echo "$modified_dirs" >&2 echo "$modified_dirs" >&2
. ./.ci/compute-projects.sh
# Project specific pipelines. # Project specific pipelines.
# If libc++ or one of the runtimes directories changed. # If libc++ or one of the runtimes directories changed.
@ -73,20 +71,16 @@ fi
# needs while letting them run on the infrastructure provided by LLVM. # needs while letting them run on the infrastructure provided by LLVM.
# Figure out which projects need to be built on each platform # Figure out which projects need to be built on each platform
all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl" source <(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD | python3 .ci/compute_projects.py Linux)
modified_projects="$(keep-modified-projects ${all_projects})" linux_projects=${projects_to_build}
linux_check_targets=${project_check_targets}
linux_runtimes=${runtimes_to_build}
linux_runtime_check_targets=${runtimes_check_targets}
linux_projects_to_test=$(exclude-linux $(compute-projects-to-test 0 ${modified_projects})) source <(git diff --name-only origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}...HEAD | python3 .ci/compute_projects.py Windows)
linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq) windows_projects=${projects_to_build}
linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq) windows_check_targets=${project_check_targets}
linux_runtimes_to_test=$(compute-runtimes-to-test ${linux_projects_to_test})
linux_runtime_check_targets=$(check-targets ${linux_runtimes_to_test} | sort | uniq)
linux_runtimes=$(echo ${linux_runtimes_to_test} | sort | uniq)
windows_projects_to_test=$(exclude-windows $(compute-projects-to-test 1 ${modified_projects}))
windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq)
windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq)
# Generate the appropriate pipeline # Generate the appropriate pipeline
if [[ "${linux_projects}" != "" ]]; then if [[ "${linux_projects}" != "" ]]; then

90
.github/workflows/premerge.yaml vendored
View File

@ -49,39 +49,22 @@ jobs:
run: | run: |
git config --global --add safe.directory '*' git config --global --add safe.directory '*'
modified_files=$(git diff --name-only HEAD~1...HEAD) source <(git diff --name-only HEAD~1...HEAD | python3 .ci/compute_projects.py)
modified_dirs=$(echo "$modified_files" | cut -d'/' -f1 | sort -u)
echo $modified_files if [[ "${projects_to_build}" == "" ]]; then
echo $modified_dirs
. ./.ci/compute-projects.sh
all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
modified_projects="$(keep-modified-projects ${all_projects})"
linux_projects_to_test=$(exclude-linux $(compute-projects-to-test 0 ${modified_projects}))
linux_check_targets=$(check-targets ${linux_projects_to_test} | sort | uniq)
linux_projects=$(add-dependencies ${linux_projects_to_test} | sort | uniq)
linux_runtimes_to_test=$(compute-runtimes-to-test ${linux_projects_to_test})
linux_runtime_check_targets=$(check-targets ${linux_runtimes_to_test} | sort | uniq)
linux_runtimes=$(echo ${linux_runtimes_to_test} | sort | uniq)
if [[ "${linux_projects}" == "" ]]; then
echo "No projects to build" echo "No projects to build"
exit 0 exit 0
fi fi
echo "Building projects: ${linux_projects}" echo "Building projects: ${projects_to_build}"
echo "Running project checks targets: ${linux_check_targets}" echo "Running project checks targets: ${project_check_targets}"
echo "Building runtimes: ${linux_runtimes}" echo "Building runtimes: ${runtimes_to_build}"
echo "Running runtimes checks targets: ${linux_runtime_check_targets}" echo "Running runtimes checks targets: ${runtimes_check_targets}"
export CC=/opt/llvm/bin/clang export CC=/opt/llvm/bin/clang
export CXX=/opt/llvm/bin/clang++ export CXX=/opt/llvm/bin/clang++
./.ci/monolithic-linux.sh "$(echo ${linux_projects} | tr ' ' ';')" "$(echo ${linux_check_targets})" "$(echo ${linux_runtimes} | tr ' ' ';')" "$(echo ${linux_runtime_check_targets})" ./.ci/monolithic-linux.sh "${projects_to_build}" "${project_check_targets}" "${runtimes_to_build}" "${runtimes_check_targets}"
premerge-checks-windows: premerge-checks-windows:
name: Windows Premerge Checks (Test Only - Please Ignore Results) name: Windows Premerge Checks (Test Only - Please Ignore Results)
@ -105,30 +88,17 @@ jobs:
- name: Compute Projects - name: Compute Projects
id: vars id: vars
run: | run: |
modified_files=$(git diff --name-only HEAD~1...HEAD) source <(git diff --name-only HEAD~1...HEAD | python .ci/compute_projects.py)
modified_dirs=$(echo "$modified_files" | cut -d'/' -f1 | sort | uniq)
echo $modified_files if [[ "${projects_to_build}" == "" ]]; then
echo $modified_dirs
. ./.ci/compute-projects.sh
all_projects="bolt clang clang-tools-extra compiler-rt cross-project-tests flang libc libclc lld lldb llvm mlir openmp polly pstl"
modified_projects="$(keep-modified-projects ${all_projects})"
windows_projects_to_test=$(exclude-windows $(compute-projects-to-test 1 ${modified_projects}))
windows_check_targets=$(check-targets ${windows_projects_to_test} | sort | uniq | tr -d '\r' | tr '\n' ' ')
windows_projects=$(add-dependencies ${windows_projects_to_test} | sort | uniq | tr -d '\r' | tr '\n' ';')
if [[ "${windows_projects}" == "" ]]; then
echo "No projects to build" echo "No projects to build"
fi fi
echo "Building projects: ${windows_projects}" echo "Building projects: ${projects_to_build}"
echo "Running project checks targets: ${windows_check_targets}" echo "Running project checks targets: ${project_check_targets}"
echo "windows-projects=${windows_projects}" >> $GITHUB_OUTPUT echo "windows-projects=${projects_to_build}" >> $GITHUB_OUTPUT
echo "windows-check-targets=${windows_check_targets}" >> $GITHUB_OUTPUT echo "windows-check-targets=${project_check_targets}" >> $GITHUB_OUTPUT
- name: Build and Test - name: Build and Test
# Mark the job as a success even if the step fails so that people do # Mark the job as a success even if the step fails so that people do
# not get notified while the new premerge pipeline is in an # not get notified while the new premerge pipeline is in an
@ -165,37 +135,15 @@ jobs:
uses: llvm/actions/install-ninja@main uses: llvm/actions/install-ninja@main
- name: Build and Test - name: Build and Test
run: | run: |
modified_files=$(git diff --name-only HEAD~1...HEAD) source <(git diff --name-only HEAD~2..HEAD | python3 .ci/compute_projects.py)
modified_dirs=$(echo "$modified_files" | cut -d'/' -f1 | sort -u)
echo $modified_files if [[ "${projects_to_build}" == "" ]]; then
echo $modified_dirs
. ./.ci/compute-projects.sh
all_projects="clang clang-tools-extra lld lldb llvm mlir"
modified_projects="$(keep-modified-projects ${all_projects})"
# We have to disable the runtimes builds due to https://github.com/llvm/llvm-project/issues/90568
# and the lldb tests depend on libcxx, so we need to skip them.
mac_check_targets=$(check-targets ${modified_projects} | sort | uniq | tr '\n' ' ' | sed -e 's/check-lldb //g')
mac_projects=$(add-dependencies ${modified_projects} | sort | uniq | tr '\n' ' ')
mac_runtimes_to_test=$(compute-runtimes-to-test ${modified_projects})
mac_runtime_check_targets=$(check-targets ${mac_runtimes_to_test} | sort | uniq | tr '\n' ' ')
mac_runtimes=$(echo ${mac_runtimes_to_test} | tr ' ' '\n' | sort | uniq | tr '\n' ' ')
if [[ "${mac_projects}" == "" ]]; then
echo "No projects to build" echo "No projects to build"
exit 0 exit 0
fi fi
echo "Projects to test: ${modified_projects}" echo "Building projects: ${projects_to_build}"
echo "Runtimes to test: ${mac_runtimes_to_test}" echo "Running project checks targets: ${project_check_targets}"
echo "Building projects: ${mac_projects}"
echo "Running project checks targets: ${mac_check_targets}"
echo "Building runtimes: ${mac_runtimes}"
echo "Running runtimes checks targets: ${mac_runtime_check_targets}"
# -DLLVM_DISABLE_ASSEMBLY_FILES=ON is for # -DLLVM_DISABLE_ASSEMBLY_FILES=ON is for
# https://github.com/llvm/llvm-project/issues/81967 # https://github.com/llvm/llvm-project/issues/81967
@ -203,7 +151,7 @@ jobs:
cmake -G Ninja \ cmake -G Ninja \
-B build \ -B build \
-S llvm \ -S llvm \
-DLLVM_ENABLE_PROJECTS="$(echo ${mac_projects} | tr ' ' ';')" \ -DLLVM_ENABLE_PROJECTS="${projects_to_build}" \
-DLLVM_DISABLE_ASSEMBLY_FILES=ON \ -DLLVM_DISABLE_ASSEMBLY_FILES=ON \
-DCMAKE_BUILD_TYPE=Release \ -DCMAKE_BUILD_TYPE=Release \
-DLLDB_INCLUDE_TESTS=OFF \ -DLLDB_INCLUDE_TESTS=OFF \
@ -212,4 +160,4 @@ jobs:
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
# The libcxx tests fail, so we are skipping the runtime targets. # The libcxx tests fail, so we are skipping the runtime targets.
ninja -C build $mac_check_targets ninja -C build ${project_check_targets}