(not at the head of any monitored branch or PR)
2025-12-30 15:25.26: New job: build llvm.21-static, using opam dev
                              from https://github.com/ocaml/opam-repository.git#refs/pull/29110/head (a16fbbbc4ad239dba151649395b1e98d3d337d30)
                              on centos-9-ocaml-4.14/amd64

To reproduce locally:

cd $(mktemp -d)
git clone --recursive "https://github.com/ocaml/opam-repository.git" && cd "opam-repository" && git fetch origin "refs/pull/29110/head" && git reset --hard a16fbbbc
git fetch origin master
git merge --no-edit f2d102f951a5dc2af6db218c6aef53c80186295a
cat > ../Dockerfile <<'END-OF-DOCKERFILE'
FROM ocaml/opam:centos-9-ocaml-4.14@sha256:b6d95f3bc3bb5c66afde03363bea6e2d4d1c2b3f9be57ca9ed04454c0f58f5e5
USER 1000:1000
WORKDIR /home/opam
RUN sudo ln -f /usr/bin/opam-dev /usr/bin/opam
RUN opam init --reinit -ni
RUN opam option solver=builtin-0install && opam config report
ENV OPAMDOWNLOADJOBS="1"
ENV OPAMERRLOGLEN="0"
ENV OPAMPRECISETRACKING="1"
ENV CI="true"
ENV OPAM_REPO_CI="true"
RUN rm -rf opam-repository/
COPY --chown=1000:1000 . opam-repository/
RUN opam repository set-url --strict default opam-repository/
RUN opam update --depexts || true
RUN opam pin add -k version -yn llvm.21-static 21-static
RUN opam reinstall llvm.21-static; \
    res=$?; \
    test "$res" != 31 && exit "$res"; \
    export OPAMCLI=2.0; \
    build_dir=$(opam var prefix)/.opam-switch/build; \
    failed=$(ls "$build_dir"); \
    partial_fails=""; \
    for pkg in $failed; do \
    if opam show -f x-ci-accept-failures: "$pkg" | grep -qF "\"centos-9\""; then \
    echo "A package failed and has been disabled for CI using the 'x-ci-accept-failures' field."; \
    fi; \
    test "$pkg" != 'llvm.21-static' && partial_fails="$partial_fails $pkg"; \
    done; \
    test "${partial_fails}" != "" && echo "opam-repo-ci detected dependencies failing: ${partial_fails}"; \
    exit 1

END-OF-DOCKERFILE
docker build -f ../Dockerfile .

2025-12-30 15:25.26: Using cache hint "ocaml/opam:centos-9-ocaml-4.14@sha256:b6d95f3bc3bb5c66afde03363bea6e2d4d1c2b3f9be57ca9ed04454c0f58f5e5-llvm.21-static-a16fbbbc4ad239dba151649395b1e98d3d337d30"
2025-12-30 15:25.26: Using OBuilder spec:
((from ocaml/opam:centos-9-ocaml-4.14@sha256:b6d95f3bc3bb5c66afde03363bea6e2d4d1c2b3f9be57ca9ed04454c0f58f5e5)
 (user (uid 1000) (gid 1000))
 (workdir /home/opam)
 (run (shell "sudo ln -f /usr/bin/opam-dev /usr/bin/opam"))
 (run (network host)
      (shell "opam init --reinit --config .opamrc-sandbox -ni"))
 (run (shell "opam option solver=builtin-0install && opam config report"))
 (env OPAMDOWNLOADJOBS 1)
 (env OPAMERRLOGLEN 0)
 (env OPAMPRECISETRACKING 1)
 (env CI true)
 (env OPAM_REPO_CI true)
 (run (shell "rm -rf opam-repository/"))
 (copy (src .) (dst opam-repository/))
 (run (shell "opam repository set-url --strict default opam-repository/"))
 (run (network host)
      (shell "opam update --depexts || true"))
 (run (shell "opam pin add -k version -yn llvm.21-static 21-static"))
 (run (cache (opam-archives (target /home/opam/.opam/download-cache)))
      (network host)
      (shell  "opam reinstall llvm.21-static;\
             \n        res=$?;\
             \n        test \"$res\" != 31 && exit \"$res\";\
             \n        export OPAMCLI=2.0;\
             \n        build_dir=$(opam var prefix)/.opam-switch/build;\
             \n        failed=$(ls \"$build_dir\");\
             \n        partial_fails=\"\";\
             \n        for pkg in $failed; do\
             \n          if opam show -f x-ci-accept-failures: \"$pkg\" | grep -qF \"\\\"centos-9\\\"\"; then\
             \n            echo \"A package failed and has been disabled for CI using the 'x-ci-accept-failures' field.\";\
             \n          fi;\
             \n          test \"$pkg\" != 'llvm.21-static' && partial_fails=\"$partial_fails $pkg\";\
             \n        done;\
             \n        test \"${partial_fails}\" != \"\" && echo \"opam-repo-ci detected dependencies failing: ${partial_fails}\";\
             \n        exit 1"))
)

2025-12-30 15:25.26: Waiting for resource in pool OCluster
2025-12-30 15:25.26: Waiting for worker…
2025-12-30 15:28.36: Got resource from pool OCluster
Building on eumache
All commits already cached
HEAD is now at f2d102f951 Merge pull request #29150 from raphael-proust/llvm-20-21-only-cosmetics
Merge made by the 'ort' strategy.
 packages/conf-llvm-shared/conf-llvm-shared.20/opam | 34 +++++++++++
 packages/conf-llvm-shared/conf-llvm-shared.21/opam | 34 +++++++++++
 packages/conf-llvm-static/conf-llvm-static.20/opam | 34 +++++++++++
 packages/conf-llvm-static/conf-llvm-static.21/opam | 34 +++++++++++
 packages/llvm/llvm.20-shared/opam                  | 67 ++++++++++++++++++++++
 packages/llvm/llvm.20-static/opam                  | 67 ++++++++++++++++++++++
 packages/llvm/llvm.21-shared/opam                  | 67 ++++++++++++++++++++++
 packages/llvm/llvm.21-static/opam                  | 67 ++++++++++++++++++++++
 8 files changed, 404 insertions(+)
 create mode 100644 packages/conf-llvm-shared/conf-llvm-shared.20/opam
 create mode 100644 packages/conf-llvm-shared/conf-llvm-shared.21/opam
 create mode 100644 packages/conf-llvm-static/conf-llvm-static.20/opam
 create mode 100644 packages/conf-llvm-static/conf-llvm-static.21/opam
 create mode 100644 packages/llvm/llvm.20-shared/opam
 create mode 100644 packages/llvm/llvm.20-static/opam
 create mode 100644 packages/llvm/llvm.21-shared/opam
 create mode 100644 packages/llvm/llvm.21-static/opam

(from ocaml/opam:centos-9-ocaml-4.14@sha256:b6d95f3bc3bb5c66afde03363bea6e2d4d1c2b3f9be57ca9ed04454c0f58f5e5)
Unable to find image 'ocaml/opam:centos-9-ocaml-4.14@sha256:b6d95f3bc3bb5c66afde03363bea6e2d4d1c2b3f9be57ca9ed04454c0f58f5e5' locally
docker.io/ocaml/opam@sha256:b6d95f3bc3bb5c66afde03363bea6e2d4d1c2b3f9be57ca9ed04454c0f58f5e5: Pulling from ocaml/opam
f8cd8c42bb80: Already exists
322e94cfde58: Already exists
12ea695e8f14: Already exists
a0619735e51a: Already exists
320d94d31091: Already exists
44197641e273: Already exists
2d0a3dad40fa: Already exists
6cda2abc8884: Already exists
c59382bce11f: Already exists
a0a7d888b56b: Already exists
29c93ca5ecc2: Already exists
4eb5bcb684d9: Already exists
7953ecba1643: Already exists
812b50b59e90: Already exists
c22f854befed: Already exists
d9334dcacbf9: Already exists
c5d32c8f3628: Already exists
586f185021dc: Already exists
586f185021dc: Already exists
80c15c541dc3: Already exists
456e2364418d: Already exists
d60b7ce96989: Already exists
37d1b6b41f24: Already exists
4f4fb700ef54: Already exists
253d43aa8999: Already exists
571faceb7b57: Already exists
7dc7f33c3fdb: Already exists
3f67bb5a83dd: Already exists
ad3cc0f698b1: Already exists
b4ed53aef5c7: Already exists
5324a3b75bc0: Already exists
a80faa10099d: Already exists
c252346bd2ee: Already exists
7e4192004608: Already exists
487c544f740e: Already exists
a2e884390d84: Already exists
395513f9504f: Already exists
3118900a0b6c: Already exists
a101d96ef179: Already exists
8188c7857b74: Already exists
3cdc8c100551: Already exists
0dc549f4ecc1: Already exists
94decd476a4f: Pulling fs layer
fc1ab310907b: Pulling fs layer
ffd45684b7bb: Pulling fs layer
1a622f7819ac: Pulling fs layer
ffd45684b7bb: Waiting
1a622f7819ac: Waiting
94decd476a4f: Waiting
fc1ab310907b: Waiting
94decd476a4f: Verifying Checksum
94decd476a4f: Download complete
fc1ab310907b: Verifying Checksum
fc1ab310907b: Download complete
ffd45684b7bb: Download complete
1a622f7819ac: Download complete
94decd476a4f: Pull complete
fc1ab310907b: Pull complete
ffd45684b7bb: Pull complete
1a622f7819ac: Pull complete
Digest: sha256:b6d95f3bc3bb5c66afde03363bea6e2d4d1c2b3f9be57ca9ed04454c0f58f5e5
Status: Downloaded newer image for ocaml/opam@sha256:b6d95f3bc3bb5c66afde03363bea6e2d4d1c2b3f9be57ca9ed04454c0f58f5e5
2025-12-30 15:28.38 ---> using "01e427e87d372981cd6075a0335380e047c2aa27af9d9f7e46e3bc9b566520a3" from cache

/: (user (uid 1000) (gid 1000))

/: (workdir /home/opam)

/home/opam: (run (shell "sudo ln -f /usr/bin/opam-dev /usr/bin/opam"))
2025-12-30 15:28.38 ---> using "e92f5d6f720187f8311d4bf7bfcff0dc57720dc2859b007a16f8641b3b835fe8" from cache

/home/opam: (run (network host)
                 (shell "opam init --reinit --config .opamrc-sandbox -ni"))
Configuring from /home/opam/.opamrc-sandbox, then /home/opam/.opamrc, and finally from built-in defaults.
Checking for available remotes: rsync and local, git.
  - you won't be able to use mercurial repositories unless you install the hg command on your system.
  - you won't be able to use darcs repositories unless you install the darcs command on your system.

This version of opam requires an update to the layout of /home/opam/.opam from version 2.0 to version 2.2, which can't be reverted.
You may want to back it up before going further.

Continue? [Y/n] y
Format upgrade done.

<><> Updating repositories ><><><><><><><><><><><><><><><><><><><><><><><><><><>
[default] Initialised
2025-12-30 15:28.38 ---> using "bdd07b88a827d9e51c4796bfcb3618f97fdade8dc32faa986a5dbe65563a8b30" from cache

/home/opam: (run (shell "opam option solver=builtin-0install && opam config report"))
Set to 'builtin-0install' the field solver in global configuration
# opam config report
# opam-version         2.5.0
# self-upgrade         no
# system               arch=x86_64 os=linux os-distribution=centos os-version=9
# solver               builtin-0install
# install-criteria     -changed,-count[avoid-version,solution]
# upgrade-criteria     -count[avoid-version,solution]
# jobs                 71
# repositories         1 (version-controlled)
# pinned               1 (version)
# current-switch       4.14
# invariant            ["ocaml-base-compiler" {= "4.14.2"}]
# compiler-packages    ocaml-base-compiler.4.14.2, ocaml-options-vanilla.1
# ocaml:native         true
# ocaml:native-tools   true
# ocaml:native-dynlink true
# ocaml:stubsdir       /home/opam/.opam/4.14/lib/ocaml/stublibs:/home/opam/.opam/4.14/lib/ocaml
# ocaml:preinstalled   false
# ocaml:compiler       4.14.2
2025-12-30 15:28.38 ---> using "869abdaefda2d8448989784297997a275e6ccc1762b36085b0d6b4f9f3a81355" from cache

/home/opam: (env OPAMDOWNLOADJOBS 1)

/home/opam: (env OPAMERRLOGLEN 0)

/home/opam: (env OPAMPRECISETRACKING 1)

/home/opam: (env CI true)

/home/opam: (env OPAM_REPO_CI true)

/home/opam: (run (shell "rm -rf opam-repository/"))
2025-12-30 15:28.38 ---> using "1fa63af69304be4d51ba7323e7972e36adb1d16f7023ceab2f8b05dcf9128117" from cache

/home/opam: (copy (src .) (dst opam-repository/))
2025-12-30 15:28.48 ---> saved as "4e773d4f9f59fa1789c39d08be41091305bf2a3cc64f7beaa2c5dc02d99942d4"

/home/opam: (run (shell "opam repository set-url --strict default opam-repository/"))
[default] Initialised
2025-12-30 15:29.10 ---> saved as "a0a005067e0abfd1917e6b127fa19ce132394fce81e57c089d82227e1ea8f271"

/home/opam: (run (network host)
                 (shell "opam update --depexts || true"))
+ /usr/bin/sudo "yum" "makecache"
- CentOS Stream 9 - BaseOS                         66 kB/s |  15 kB     00:00    
- CentOS Stream 9 - AppStream                      40 kB/s |  15 kB     00:00    
- CentOS Stream 9 - CRB                            43 kB/s |  14 kB     00:00    
- CentOS Stream 9 - Extras packages                79 kB/s |  16 kB     00:00    
- Metadata cache created.
2025-12-30 15:29.13 ---> saved as "ca5ef8b35374c8591a237c63c09e7a585fe3f0e0526f881f157d7478c327f96c"

/home/opam: (run (shell "opam pin add -k version -yn llvm.21-static 21-static"))
llvm is now pinned to version 21-static
2025-12-30 15:29.13 ---> saved as "408dec9fb1d1e9f475d805afc86ebb1f3b20d8e842e7cbeb53a88cd990b841a6"

/home/opam: (run (cache (opam-archives (target /home/opam/.opam/download-cache)))
                 (network host)
                 (shell  "opam reinstall llvm.21-static;\
                        \n        res=$?;\
                        \n        test \"$res\" != 31 && exit \"$res\";\
                        \n        export OPAMCLI=2.0;\
                        \n        build_dir=$(opam var prefix)/.opam-switch/build;\
                        \n        failed=$(ls \"$build_dir\");\
                        \n        partial_fails=\"\";\
                        \n        for pkg in $failed; do\
                        \n          if opam show -f x-ci-accept-failures: \"$pkg\" | grep -qF \"\\\"centos-9\\\"\"; then\
                        \n            echo \"A package failed and has been disabled for CI using the 'x-ci-accept-failures' field.\";\
                        \n          fi;\
                        \n          test \"$pkg\" != 'llvm.21-static' && partial_fails=\"$partial_fails $pkg\";\
                        \n        done;\
                        \n        test \"${partial_fails}\" != \"\" && echo \"opam-repo-ci detected dependencies failing: ${partial_fails}\";\
                        \n        exit 1"))
llvm.21-static is not installed. Install it? [Y/n] y
The following actions will be performed:
=== install 11 packages
  - install conf-bash         1                  [required by conf-llvm-static]
  - install conf-cmake        1                  [required by llvm]
  - install conf-llvm-static  21                 [required by llvm]
  - install csexp             1.5.2              [required by dune-configurator]
  - install ctypes            0.24.0             [required by llvm]
  - install dune              3.20.2             [required by ctypes]
  - install dune-configurator 3.20.2             [required by ctypes]
  - install integers          0.7.0              [required by ctypes]
  - install llvm              21-static (pinned)
  - install ocamlfind         1.9.8              [required by llvm]
  - install stdlib-shims      0.3.0              [required by integers]

The following system packages will first need to be installed:
    cmake3 epel-release llvm21-devel

<><> Handling external dependencies <><><><><><><><><><><><><><><><><><><><><><>

opam believes some required external dependencies are missing. opam can:
> 1. Run yum to install them (may need root/sudo access)
  2. Display the recommended yum command and wait while you run it manually (e.g. in another terminal)
  3. Continue anyway, and, upon success, permanently register that this external dependency is present, but not detectable
  4. Abort the installation

[1/2/3/4] 1

+ /usr/bin/sudo "yum" "install" "-y" "epel-release"
- Last metadata expiration check: 0:00:04 ago on Tue Dec 30 15:29:13 2025.
- Dependencies resolved.
- ================================================================================
-  Package                Arch        Version            Repository          Size
- ================================================================================
- Installing:
-  epel-release           noarch      9-7.el9            extras-common       19 k
- Installing weak dependencies:
-  dnf-plugins-core       noarch      4.3.0-25.el9       baseos              36 k
-  epel-next-release      noarch      9-7.el9            extras-common      8.1 k
- 
- Transaction Summary
- ================================================================================
- Install  3 Packages
- 
- Total download size: 62 k
- Installed size: 51 k
- Downloading Packages:
- (1/3): dnf-plugins-core-4.3.0-25.el9.noarch.rpm 731 kB/s |  36 kB     00:00    
- (2/3): epel-release-9-7.el9.noarch.rpm          311 kB/s |  19 kB     00:00    
- (3/3): epel-next-release-9-7.el9.noarch.rpm     1.5 kB/s | 8.1 kB     00:05    
- --------------------------------------------------------------------------------
- Total                                            10 kB/s |  62 kB     00:06     
- CentOS Stream 9 - Extras packages               2.1 MB/s | 2.1 kB     00:00    
- Importing GPG key 0x1D997668:
-  Userid     : "CentOS Extras SIG (https://wiki.centos.org/SpecialInterestGroup) <security@centos.org>"
-  Fingerprint: 363F C097 2F64 B699 AED3 968E 1FF6 A217 1D99 7668
-  From       : /etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-SIG-Extras-SHA512
- Key imported successfully
- Running transaction check
- Transaction check succeeded.
- Running transaction test
- Transaction test succeeded.
- Running transaction
-   Preparing        :                                                        1/1 
-   Installing       : dnf-plugins-core-4.3.0-25.el9.noarch                   1/3 
-   Installing       : epel-next-release-9-7.el9.noarch                       2/3 
-   Installing       : epel-release-9-7.el9.noarch                            3/3 
-   Running scriptlet: epel-release-9-7.el9.noarch                            3/3 
- Many EPEL packages require the CodeReady Builder (CRB) repository.
- It is recommended that you run /usr/bin/crb enable to enable the CRB repository.
- 
-   Verifying        : dnf-plugins-core-4.3.0-25.el9.noarch                   1/3 
-   Verifying        : epel-next-release-9-7.el9.noarch                       2/3 
-   Verifying        : epel-release-9-7.el9.noarch                            3/3 
- 
- Installed:
-   dnf-plugins-core-4.3.0-25.el9.noarch     epel-next-release-9-7.el9.noarch    
-   epel-release-9-7.el9.noarch             
- 
- Complete!
+ /usr/bin/sudo "yum" "install" "-y" "cmake3" "llvm21-devel"
- Extra Packages for Enterprise Linux 9 - x86_64  2.3 MB/s |  20 MB     00:08    
- Extra Packages for Enterprise Linux 9 openh264  1.9 kB/s | 2.5 kB     00:01    
- Extra Packages for Enterprise Linux 9 - Next -  168 kB/s | 259 kB     00:01    
- No match for argument: llvm21-devel
- Error: Unable to find a match: llvm21-devel
[ERROR] System package install failed with exit code 1 at command:
            sudo yum install -y cmake3 llvm21-devel
[ERROR] These packages are still missing: cmake3 epel-release llvm21-devel

[NOTE] You can retry with '--assume-depexts' to skip this check, or run 'opam option depext=false' to permanently disable handling of system packages.

"/usr/bin/env" "bash" "-c" "opam reinstall llvm.21-static;
        res=$?;
        test "$res" != 31 && exit "$res";
        export OPAMCLI=2.0;
        build_dir=$(opam var prefix)/.opam-switch/build;
        failed=$(ls "$build_dir");
        partial_fails="";
        for pkg in $failed; do
          if opam show -f x-ci-accept-failures: "$pkg" | grep -qF "\"centos-9\""; then
            echo "A package failed and has been disabled for CI using the 'x-ci-accept-failures' field.";
          fi;
          test "$pkg" != 'llvm.21-static' && partial_fails="$partial_fails $pkg";
        done;
        test "${partial_fails}" != "" && echo "opam-repo-ci detected dependencies failing: ${partial_fails}";
        exit 1" failed with exit status 10
2025-12-30 15:29.56: Job failed: Failed: Build failed
2025-12-30 15:29.56: Log analysis:
2025-12-30 15:29.56: >>> 
[ERROR] System package install failed with exit code 1 at command:
 (score = 20)
2025-12-30 15:29.56: >>> 
[ERROR] These packages are still missing: cmake3 epel-release llvm21-devel
 (score = 20)
2025-12-30 15:29.56: System package install failed with exit code 1 at command: