Merge pull request #6914 from fstagni/80_VacuumPilots

[8.0] Add dirac-admin-add-pilot command

docs/source/AdministratorGuide/Systems/WorkloadManagement/Pilots/index.rst

@@ -168,7 +168,7 @@ If this is the case, then you need to:
 - provide a certificate, or a proxy, to start the pilot;
 - such certificate/proxy should have the `GenericPilot` property;
 - in case of multi-VO environment, the Pilot should set the `/Resources/Computing/CEDefaults/VirtualOrganization` (as done e.g. by `vm-pilot <https://github.com/DIRACGrid/DIRAC/blob/integration/src/DIRAC/WorkloadManagementSystem/Utilities/CloudBootstrap/vm-pilot#L122>`_);
-- find a way to start the pilots: VMDIRAC extension will make sure to create VirtualMachine contextualized to start Pilot3.
+- find a way to start the pilots: DIRAC will make sure to create VirtualMachine contextualized to start DIRAC Pilots.
 
 We have introduced a special command named "GetPilotVersion" that you should use,
 and possibly extend, in case you want to send/start pilots that don't know beforehand the (VO)DIRAC version they are going to install.
@@ -181,10 +181,8 @@ The main file in which you should look is dirac-pilot.py
 that also contains a good explanation on how the system works.
 
 You have to provide in this case a pilot wrapper script (which can be written in bash, for example) that will start your pilot script
-with the proper environment. If you are on a cloud site, often contextualization of your virtual machine is done by supplying
-a script like the following: https://github.com/DIRACGrid/Pilot/blob/master/Pilot/user_data_vm
-
-A simpler example using the LHCbPilot extension follows::
+with the proper environment.
+A simple example using the LHCbPilot extension follows::
 
   #!/bin/sh
   #

setup.cfg

@@ -305,6 +305,7 @@ console_scripts =
     dirac-transformation-verify-outputdata = DIRAC.TransformationSystem.scripts.dirac_transformation_verify_outputdata:main [admin]
     dirac-transformation-update-derived = DIRAC.TransformationSystem.scripts.dirac_transformation_update_derived:main [admin]
     # WorkloadManagementSystem
+    dirac-admin-add-pilot = DIRAC.WorkloadManagementSystem.scripts.dirac_admin_add_pilot:main [pilot]
     dirac-admin-kill-pilot = DIRAC.WorkloadManagementSystem.scripts.dirac_admin_kill_pilot:main [admin]
     dirac-admin-pilot-logging-info = DIRAC.WorkloadManagementSystem.scripts.dirac_admin_pilot_logging_info:main [admin]
     dirac-admin-show-task-queues = DIRAC.WorkloadManagementSystem.scripts.dirac_admin_show_task_queues:main [admin]

src/DIRAC/Resources/Computing/ARCComputingElement.py

@@ -48,11 +48,11 @@
 import os
 import stat
 import sys
+import uuid
 
 import arc  # Has to work if this module is called #pylint: disable=import-error
-from DIRAC import S_OK, S_ERROR, gConfig
+from DIRAC import S_OK, S_ERROR
 from DIRAC.Core.Utilities.Subprocess import shellCall
-from DIRAC.Core.Utilities.File import makeGuid
 from DIRAC.Core.Utilities.List import breakListIntoChunks
 from DIRAC.Core.Security.ProxyInfo import getVOfromProxyGroup
 from DIRAC.Resources.Computing.ComputingElement import ComputingElement
@@ -188,7 +188,7 @@ def _writeXRSL(self, executableFile, inputs, outputs):
         :param list inputs: path of the dependencies to include along with the executable
         :param list outputs: path of the outputs that we want to get at the end of the execution
         """
-        diracStamp = makeGuid()[:8]
+        diracStamp = uuid.uuid4().hex
         # Evaluate the number of processors to allocate
         nProcessors = self.ceParameters.get("NumberOfProcessors", 1)
 

src/DIRAC/Resources/Computing/HTCondorCEComputingElement.py

@@ -52,6 +52,7 @@
 import errno
 import threading
 import textwrap
+import uuid
 
 from DIRAC import S_OK, S_ERROR, gConfig
 from DIRAC.Resources.Computing.ComputingElement import ComputingElement
@@ -60,7 +61,6 @@
 from DIRAC.Core.Utilities.List import breakListIntoChunks
 from DIRAC.WorkloadManagementSystem.Client import PilotStatus
 from DIRAC.WorkloadManagementSystem.Client.PilotManagerClient import PilotManagerClient
-from DIRAC.Core.Utilities.File import makeGuid
 from DIRAC.FrameworkSystem.private.authorization.utils.Tokens import writeToTokenFile
 from DIRAC.Core.Security.Locations import getCAsLocation
 from DIRAC.Resources.Computing.BatchSystems.Condor import parseCondorStatus
@@ -300,9 +300,9 @@ def submitJob(self, executableFile, proxy, numberOfJobs=1):
         # The submitted pilots are going to have a common part of the stamp to construct a path to retrieve results
         # Then they also have an individual part to make them unique
         jobStamps = []
-        commonJobStampPart = makeGuid()[:3]
+        commonJobStampPart = uuid.uuid4().hex[:3]
         for _i in range(numberOfJobs):
-            jobStamp = commonJobStampPart + makeGuid()[:5]
+            jobStamp = commonJobStampPart + uuid.uuid4().hex[:29]
             jobStamps.append(jobStamp)
 
         # We randomize the location of the pilot output and log, because there are just too many of them

src/DIRAC/Resources/Computing/LocalComputingElement.py

@@ -37,15 +37,14 @@
 import tempfile
 import getpass
 import errno
+import uuid
 from urllib.parse import urlparse
 
 from DIRAC import S_OK, S_ERROR
-from DIRAC import gConfig
 
 from DIRAC.Resources.Computing.ComputingElement import ComputingElement
 from DIRAC.Resources.Computing.PilotBundle import bundleProxy, writeScript
 from DIRAC.Core.Utilities.List import uniqueElements
-from DIRAC.Core.Utilities.File import makeGuid
 from DIRAC.Core.Utilities.Subprocess import systemCall
 
 
@@ -170,7 +169,7 @@ def submitJob(self, executableFile, proxy=None, numberOfJobs=1):
 
         jobStamps = []
         for _i in range(numberOfJobs):
-            jobStamps.append(makeGuid()[:8])
+            jobStamps.append(uuid.uuid4().hex)
 
         batchDict = {
             "Executable": submitFile,

src/DIRAC/Resources/Computing/SSHComputingElement.py

@@ -65,14 +65,16 @@
 import errno
 import json
 import os
-import pexpect
 import shutil
 import stat
+import uuid
 from urllib.parse import urlparse
 from urllib.parse import quote
 from urllib.parse import unquote
 from shlex import quote as shlex_quote
 
+import pexpect
+
 import DIRAC
 from DIRAC import S_OK, S_ERROR
 from DIRAC import gLogger
@@ -81,7 +83,6 @@
 from DIRAC.Resources.Computing.PilotBundle import bundleProxy, writeScript
 from DIRAC.Resources.Computing.BatchSystems.executeBatch import executeBatchContent
 from DIRAC.Core.Utilities.List import uniqueElements
-from DIRAC.Core.Utilities.File import makeGuid
 from DIRAC.Core.Utilities.List import breakListIntoChunks
 
 
@@ -563,7 +564,7 @@ def _submitJobToHost(self, executableFile, numberOfJobs, host=None):
 
         jobStamps = []
         for _i in range(numberOfJobs):
-            jobStamps.append(makeGuid()[:8])
+            jobStamps.append(uuid.uuid4().hex)
 
         numberOfProcessors = self.ceParameters.get("NumberOfProcessors", 1)
         wholeNode = self.ceParameters.get("WholeNode", False)

src/DIRAC/Resources/Computing/test/Test_HTCondorCEComputingElement.py

@@ -2,11 +2,11 @@
 """
 tests for HTCondorCEComputingElement module
 """
+import uuid
 import pytest
 
 from DIRAC.Resources.Computing import HTCondorCEComputingElement as HTCE
 from DIRAC.Resources.Computing.BatchSystems import Condor
-from DIRAC.Core.Utilities.File import makeGuid
 from DIRAC import S_OK
 
 MODNAME = "DIRAC.Resources.Computing.HTCondorCEComputingElement"
@@ -127,9 +127,9 @@ def test__writeSub(mocker, localSchedd, optionsNotExpected, optionsExpected):
     mocker.patch(MODNAME + ".mkDir")
 
     jobStamps = []
-    commonJobStampPart = makeGuid()[:3]
+    commonJobStampPart = uuid.uuid4().hex[:3]
     for _i in range(42):
-        jobStamp = commonJobStampPart + makeGuid()[:5]
+        jobStamp = commonJobStampPart + uuid.uuid4().hex[:29]
         jobStamps.append(jobStamp)
 
     htce._HTCondorCEComputingElement__writeSub("dirac-install", 42, "", 1, jobStamps)  # pylint: disable=E1101