Skip to content

Commit

Permalink
feat: add RunNumber as a possible parametric sequence
Browse files Browse the repository at this point in the history
  • Loading branch information
fstagni committed Sep 24, 2024
1 parent 7852c14 commit 09ed430
Show file tree
Hide file tree
Showing 5 changed files with 35 additions and 3 deletions.
5 changes: 5 additions & 0 deletions src/DIRAC/Interfaces/API/Job.py
Original file line number Diff line number Diff line change
Expand Up @@ -904,6 +904,7 @@ def __setJobDefaults(self):
self._addParameter(self.workflow, "StdOutput", "JDL", self.stdout, "Standard output file")
self._addParameter(self.workflow, "StdError", "JDL", self.stderr, "Standard error file")
self._addParameter(self.workflow, "InputData", "JDL", "", "Default null input data value")
self._addParameter(self.workflow, "RunNumber", "JDL", "", "Default null input run value")
self._addParameter(self.workflow, "LogLevel", "JDL", self.logLevel, "Job Logging Level")
self._addParameter(self.workflow, "arguments", "string", "", "Arguments to executable Step")
# Those 2 below are need for on-site resolution
Expand All @@ -913,6 +914,9 @@ def __setJobDefaults(self):
self._addParameter(
self.workflow, "ParametricInputSandbox", "string", "", "Default null parametric input sandbox value"
)
self._addParameter(
self.workflow, "ParametricRunNumber", "string", "", "Default null parametric run number value"
)

#############################################################################

Expand Down Expand Up @@ -1021,6 +1025,7 @@ def _handleParameterSequences(self, paramsDict, arguments):
# If a parameter with the same name as the sequence name already exists
# and is a list, then extend it by the sequence value. If it is not a
# list, then replace it by the sequence value

if isinstance(paramsDict[pName]["value"], list):
currentParams = paramsDict[pName]["value"]
tmpList = []
Expand Down
7 changes: 7 additions & 0 deletions src/DIRAC/Interfaces/API/test/Test_JobAPI.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ def test_SimpleParametricJob():
]
job.setParameterSequence("InputData", inputDataList, addToWorkflow=True)

runNumberList = [123, 456, 789]
res = job.setParameterSequence("RunNumber", runNumberList, addToWorkflow=True)
assert res["OK"]

jdl = job._toJDL()

with open(join(dirname(__file__), "testWF.jdl")) as fd:
Expand All @@ -59,13 +63,16 @@ def test_SimpleParametricJob():
arguments = clad.getAttributeString("Arguments")
job_id = clad.getAttributeString("JOB_ID")
inputData = clad.getAttributeString("InputData")
runNumber = clad.getAttributeString("RunNumber")

assert job_id == "%(JOB_ID)s"
assert inputData == "%(InputData)s"
assert runNumber == "%(RunNumber)s"
assert "jobDescription.xml" in arguments
assert "-o LogLevel=DEBUG" in arguments
assert "-p JOB_ID=%(JOB_ID)s" in arguments
assert "-p InputData=%(InputData)s" in arguments
assert "-p RunNumber=%(RunNumber)s" in arguments


@pytest.mark.parametrize(
Expand Down
9 changes: 8 additions & 1 deletion src/DIRAC/Interfaces/API/test/testWF.jdl
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@

Arguments = "jobDescription.xml -o LogLevel=DEBUG -p JOB_ID=%(JOB_ID)s -p InputData=%(InputData)s";
Arguments = "jobDescription.xml -o LogLevel=DEBUG -p JOB_ID=%(JOB_ID)s -p InputData=%(InputData)s -p RunNumber=%(RunNumber)s";
Executable = "dirac-jobexec";
InputData = %(InputData)s;
InputSandbox = jobDescription.xml;
Expand Down Expand Up @@ -29,6 +29,13 @@
2,
3
};
Parameters.RunNumber =
{
123,
456,
789
};
Priority = 1;
RunNumber = %(RunNumber)s;
StdError = std.err;
StdOutput = std.out;
2 changes: 2 additions & 0 deletions src/DIRAC/Interfaces/API/test/testWF.xml
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,12 @@
<Parameter name="StdOutput" type="JDL" linked_module="" linked_parameter="" in="True" out="False" description="Standard output file"><value><![CDATA[std.out]]></value></Parameter>
<Parameter name="StdError" type="JDL" linked_module="" linked_parameter="" in="True" out="False" description="Standard error file"><value><![CDATA[std.err]]></value></Parameter>
<Parameter name="InputData" type="JDL" linked_module="" linked_parameter="" in="True" out="False" description="Default null input data value"><value><![CDATA[]]></value></Parameter>
<Parameter name="RunNumber" type="JDL" linked_module="" linked_parameter="" in="True" out="False" description="Default null input run value"><value><![CDATA[]]></value></Parameter>
<Parameter name="LogLevel" type="JDL" linked_module="" linked_parameter="" in="True" out="False" description="Job Logging Level"><value><![CDATA[INFO]]></value></Parameter>
<Parameter name="arguments" type="string" linked_module="" linked_parameter="" in="True" out="False" description="Arguments to executable Step"><value><![CDATA[]]></value></Parameter>
<Parameter name="ParametricInputData" type="string" linked_module="" linked_parameter="" in="True" out="False" description="Default null parametric input data value"><value><![CDATA[]]></value></Parameter>
<Parameter name="ParametricInputSandbox" type="string" linked_module="" linked_parameter="" in="True" out="False" description="Default null parametric input sandbox value"><value><![CDATA[]]></value></Parameter>
<Parameter name="ParametricRunNumber" type="string" linked_module="" linked_parameter="" in="True" out="False" description="Default null parametric run number value"><value><![CDATA[]]></value></Parameter>
<Parameter name="Owner" type="JDL" linked_module="" linked_parameter="" in="True" out="False" description="User specified ID"><value><![CDATA[ownerName]]></value></Parameter>
<Parameter name="OwnerGroup" type="JDL" linked_module="" linked_parameter="" in="True" out="False" description="User specified owner group."><value><![CDATA[ownerGroup]]></value></Parameter>
<Parameter name="Site" type="JDL" linked_module="" linked_parameter="" in="True" out="False" description="User specified destination site"><value><![CDATA[ANY]]></value></Parameter>
Expand Down
15 changes: 13 additions & 2 deletions src/DIRAC/TransformationSystem/Client/WorkflowTasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -211,8 +211,19 @@ def __prepareTasksBulk(self, transBody, taskDict, owner, ownerGroup, ownerDN):
self._logError("Invalid mixture of jobs with and without input data")
return S_ERROR(ETSDATA, "Invalid mixture of jobs with and without input data")

# Handle Run Number
runNumber = paramsDict.get("RunNumber")
if runNumber:
if isinstance(runNumber, str):
runNumber = runNumber.replace(" ", "").split(";")
self._logVerbose(f"Setting run number to {runNumber}", transID=transID, method=method)
seqDict["RunNumber"] = runNumber
elif paramSeqDict.get("RunNumber") is not None:
self._logError("Invalid mixture of jobs with and without run number")
return S_ERROR(ETSDATA, "Invalid mixture of jobs with and without run number")

for paramName, paramValue in paramsDict.items():
if paramName not in ("InputData", "Site", "TargetSE"):
if paramName not in ("InputData", "RunNumber", "Site", "TargetSE"):
if paramValue:
self._logVerbose(f"Setting {paramName} to {paramValue}", transID=transID, method=method)
seqDict[paramName] = paramValue
Expand Down Expand Up @@ -244,7 +255,7 @@ def __prepareTasksBulk(self, transBody, taskDict, owner, ownerGroup, ownerDN):
paramSeqDict.setdefault(pName, []).append(seq)

for paramName, paramSeq in paramSeqDict.items():
if paramName in ["JOB_ID", "PRODUCTION_ID", "InputData"] + outputParameterList:
if paramName in ["JOB_ID", "PRODUCTION_ID", "InputData", "RunNumber"] + outputParameterList:
res = oJob.setParameterSequence(paramName, paramSeq, addToWorkflow=paramName)
else:
res = oJob.setParameterSequence(paramName, paramSeq)
Expand Down

0 comments on commit 09ed430

Please sign in to comment.