rpmbuild, frontend: activate Red Hat subscription on demand

Fix fedora-copr#2132

backend/copr_backend/background_worker_build.py

@@ -12,6 +12,8 @@
 import json
 import shlex
 
+from subprocess import PIPE
+from tempfile import NamedTemporaryFile
 from datetime import datetime
 from packaging import version
 from cachetools.func import ttl_cache
@@ -242,21 +244,31 @@ def _check_copr_builder(self):
             raise BuildRetry("Minimum version for builder is {}"
                              .format(MIN_BUILDER_VERSION))
 
-    def _check_mock_config(self):
-        config = "/etc/mock/{}.cfg".format(self.job.chroot)
-        command = "/usr/bin/test -f " + config
-        if self.job.chroot == "srpm-builds":
-            return
-        if self.ssh.run(command):
-            raise BuildRetry("Chroot config {} not found".format(config))
-
     def _check_vm(self):
         """
         Check that the VM is OK to start the build
         """
         self.log.info("Checking that builder machine is OK")
         self._check_copr_builder()
-        self._check_mock_config()
+
+        # We could open `self.job.backend_log` for appending and use it for
+        # stdout but I don't want to open the file for so long. This command
+        # can take 10 minutes to finish so I am afraid of data loss in case
+        # someone writes to the log in the meantime.
+        #
+        # Either way, the output won't be live and will appear only after this
+        # command finishes. Making it live is nontrivial but we have a good
+        # code for doing so in `resallocserver.manager.run_command`. Praiskup
+        # plans to generalize it into a separate package that we could
+        # eventually use here.
+        with NamedTemporaryFile(prefix="copr-builder-ready-") as tmp:
+            cmd = "copr-builder-ready " + self.job.chroot
+            rc = self.ssh.run(cmd, stdout=tmp, stderr=PIPE)
+            tmp.seek(0)
+            out = tmp.read().decode("utf-8")
+        self.log.info(out)
+        if rc:
+            raise BuildRetry("Builder wasn't ready, trying a new one")
 
     def _fill_build_info_file(self):
         """

backend/tests/test_background_worker_build.py

@@ -695,13 +695,13 @@ class _SideEffect:
         def __call__(self):
             self.counter += 1
             if self.counter == 1:
-                return (1, "err stdout", "err stderr")
-            return (0, "", "")
+                return (1, b"err stdout", "err stderr")
+            return (0, b"", "")
 
     config = f_build_rpm_case
     ssh = config.ssh
-    ssh.set_command("/usr/bin/test -f /etc/mock/fedora-30-x86_64.cfg",
-                    0, "", "", return_action=_SideEffect())
+    ssh.set_command("copr-builder-ready fedora-30-x86_64",
+                    0, b"", "", return_action=_SideEffect())
     worker = config.bw
     worker.process()
     assert_logs_exist([
@@ -859,8 +859,8 @@ def test_failed_build_retry(f_build_rpm_case, caplog):
         hosts[index].hostname = "1.2.3." + str(index)
     rhf.return_value.get_host.side_effect = hosts
     ssh = config.ssh
-    ssh.set_command("/usr/bin/test -f /etc/mock/fedora-30-x86_64.cfg",
-                    1, "", "not found")
+    ssh.set_command("copr-builder-ready fedora-30-x86_64",
+                    1, b"", "not found")
 
     config.bw.process()
     assert_logs_exist([

backend/tests/testlib/__init__.py

@@ -5,6 +5,7 @@
 import json
 import os
 import shutil
+from subprocess import PIPE
 from unittest.mock import MagicMock
 
 from copr_backend.background_worker_build import COMMANDS
@@ -130,8 +131,8 @@ def __init__(self, user=None, host=None, config_file=None, log=None):
         self.commands = {}
         self.set_command(COMMANDS["rpm_q_builder"],
                          0, "666\n", "")
-        self.set_command("/usr/bin/test -f /etc/mock/fedora-30-x86_64.cfg",
-                         0, "", "")
+        self.set_command("copr-builder-ready fedora-30-x86_64", 0, b"", "")
+        self.set_command("copr-builder-ready srpm-builds", 0, b"", "")
         self.set_command("copr-rpmbuild-log",
                          0, "build log stdout\n", "build log stderr\n")
         self.resultdir = "fedora-30-x86_64/00848963-example"
@@ -157,6 +158,10 @@ def get_command(self, cmd):
     def run(self, user_command, stdout=None, stderr=None, max_retries=0,
             subprocess_timeout=DEFAULT_SUBPROCESS_TIMEOUT):
         """ fake SSHConnection.run() """
+        if stdout == PIPE:
+            stdout = None
+        if stderr == PIPE:
+            stderr = None
         with open(os.devnull, "w") as devnull:
             out = stdout or devnull
             err = stderr or devnull

rpmbuild/bin/copr-builder-ready

@@ -0,0 +1,100 @@
+#! /usr/bin/python3
+
+"""
+Final checks that the builder machine is ready to be used
+
+Everything printed to STDOUT will be redirected to the copr-backend logs,
+STDERR will be ignored.
+"""
+
+import os
+import sys
+import time
+from fnmatch import fnmatch
+from subprocess import run, PIPE
+from copr_rpmbuild.config import Config
+
+
+def check_mock_config(chroot):
+    """
+    Does the mock config for this chroot exist?
+    """
+    if chroot == "srpm-builds":
+        return
+
+    config = "/etc/mock/{}.cfg".format(chroot)
+    if os.path.isfile(config):
+        return
+
+    print("Chroot config {} not found".format(config))
+    sys.exit(1)
+
+
+def subscription_required(chroot):
+    """
+    Is subscription required for this task?
+    """
+    config = Config()
+    config.load_config()
+
+    for pattern in config.rhsm:
+        if fnmatch(chroot, pattern):
+            return True
+    return False
+
+
+def active_subscription():
+    """
+    Is subscription active on this system?
+    """
+    # This implementation requires root privileges which we fortunately have
+    # when calling this script. In case this function needs to be re-written to
+    # be used under a normal user, it could be done by checking the existence of
+    # the `/etc/pki/consumer/cert.pem` file. However, it will break in the
+    # following corner cases:
+    #   - The system is registered and then halted for a long time and then
+    #     booted again after the entitlement is no longer valid
+    #   - The system is unregistered from the server (but not from the client
+    #     itself), making all the client files stale
+    cmd = ["subscription-manager", "status"]
+    return run(cmd, stdout=PIPE, stderr=PIPE, check=False) == 0
+
+
+def wait_for_subscription(timeout=600):
+    """
+    Wait until this system has an active subscription
+
+    Activating Red Hat subscription may take a lot of time and historically, the
+    subscription service used to be unreliable, so we should wait for the
+    subscription only when necessary.
+    """
+    start = time.time()
+    attempt = 1
+    while True:
+        print("Checking Red Hat subscription (attempt #{0})".format(attempt))
+        if active_subscription():
+            print("Red Hat subscription active")
+            return
+        if time.time() > start + timeout:
+            print("Waiting for Red Hat subscription timeouted!")
+            sys.exit(1)
+        time.sleep(30)
+        attempt += 1
+
+
+def main():
+    """
+    The entrypoint for this script
+    """
+    try:
+        chroot = sys.argv[1]
+        check_mock_config(chroot)
+        if subscription_required(chroot):
+            wait_for_subscription()
+    except RuntimeError as ex:
+        print(ex)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

rpmbuild/copr-rpmbuild.spec

@@ -232,6 +232,7 @@ install -d %{buildroot}%{_mandir}/man1
 install -p -m 644 man/copr-rpmbuild.1 %{buildroot}/%{_mandir}/man1/
 install -p -m 755 bin/copr-builder %buildroot%_bindir
 install -p -m 755 bin/copr-builder-cleanup %buildroot%_bindir
+install -p -m 755 bin/copr-builder-ready %buildroot%_bindir
 install -p -m 755 bin/copr-sources-custom %buildroot%_bindir
 install -p -m 755 bin/copr-rpmbuild-cancel %buildroot%_bindir
 install -p -m 755 bin/copr-rpmbuild-log %buildroot%_bindir
@@ -278,6 +279,7 @@ install -p -m 755 copr-update-builder %buildroot%_bindir
 %_bindir/copr-builder
 %_bindir/copr-update-builder
 %_bindir/copr-builder-cleanup
+%_bindir/copr-builder-ready
 %_sysconfdir/copr-builder
 %dir %mock_config_overrides
 %doc %mock_config_overrides/README

rpmbuild/copr-rpmbuild.yml

@@ -12,3 +12,8 @@
 #       cute
 #       multiline
 #       snippet
+#
+# Chroots that require active Red Hat subscription
+# rhsm:
+#   - rhel-*
+#   - epel-*

rpmbuild/copr_rpmbuild/config.py

@@ -12,8 +12,10 @@ class Config:
     """
     Configuration class for copr-rpmbuild
     """
+
     def __init__(self):
         self.tags_to_mock_snippet = []
+        self.rhsm = []
 
     def load_config(self):
         """
@@ -27,3 +29,4 @@ def load_config(self):
             pass
 
         self.tags_to_mock_snippet = config_data.get("tags_to_mock_snippet", [])
+        self.rhsm = config_data.get("rhsm", [])