From 7da83a2fb021742c02adb6bec064811a982a1ec8 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Fri, 12 Jun 2026 23:02:05 +0200 Subject: [PATCH 1/5] feat: interpret outputPath with LFN: prefix as absolute one --- src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py index 53c803b3400..bb2cf8f3811 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py @@ -1113,6 +1113,10 @@ def __getLFNfromOutputFile(self, outputFile, outputPath=""): # If output path is given, append it to the user path and put output files in this directory if outputPath.startswith("/"): outputPath = outputPath[1:] + # If output path is given with the LFN: prefix, take it as an absolute path + elif outputPath.startswith("LFN:"): + outputPath = outputPath[5:] + basePath = "" else: # By default the output path is constructed from the job id subdir = str(int(self.jobID / 1000)) From b464e93ea1d7cbf021425646a2b76129120ab04e Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Sat, 13 Jun 2026 00:34:06 +0200 Subject: [PATCH 2/5] feat: make output LFNs globbable --- .../JobWrapper/JobWrapper.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py index bb2cf8f3811..37cc2ff1db7 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py @@ -951,7 +951,17 @@ def __transferOutputDataFiles(self, outputData, outputSE, outputPath): else: nonlfnList.append(out) - # Check whether list of outputData has a globbable pattern + # Check whether the list of LFNs has globbable patterns + globbedLfnList = [] + for lfn in lfnList: + lfnPath = os.path.dirname(lfn) + lfnLocal = os.path.basename(lfn) + globbedLfnList += [os.path.join(lfnPath, gLfn) for gLfn in List.uniqueElements(getGlobbedFiles(lfnLocal))] + if globbedLfnList != lfnList and globbedLfnList: + self.log.info("Found a pattern in the output data LFN list, LFNs to upload are:", ", ".join(globbedLfnList)) + lfnList = globbedLfnList + + # Check whether the list of outputData has a globbable pattern globbedOutputList = List.uniqueElements(getGlobbedFiles(nonlfnList)) if globbedOutputList != nonlfnList and globbedOutputList: self.log.info( From 444c0cac4d40a6bec50828ee357153420ec10dd1 Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Sat, 13 Jun 2026 15:17:30 +0200 Subject: [PATCH 3/5] fix: fixed path evaluation --- src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py index 37cc2ff1db7..35498d2aa09 100755 --- a/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py +++ b/src/DIRAC/WorkloadManagementSystem/JobWrapper/JobWrapper.py @@ -1125,7 +1125,7 @@ def __getLFNfromOutputFile(self, outputFile, outputPath=""): outputPath = outputPath[1:] # If output path is given with the LFN: prefix, take it as an absolute path elif outputPath.startswith("LFN:"): - outputPath = outputPath[5:] + outputPath = outputPath[4:] basePath = "" else: # By default the output path is constructed from the job id From fbf46546bb0477ca367d50b6494634ef24c9f5ea Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Mon, 15 Jun 2026 16:13:02 +0200 Subject: [PATCH 4/5] feat: added documentation on outputData/Path/SE specification --- .../UserJobs/JDLReference/index.rst | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst b/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst index c0301b624ab..1ca6a8e2fcd 100644 --- a/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst +++ b/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst @@ -70,11 +70,11 @@ In this section all the attributes that can be used in the DIRAC JDL job descrip +---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ | *InputDataPolicy* | Job input data policy | InputDataPolicy = ``"DIRAC.WorkloadManagementSystem.Client.DownloadInputData";`` | +---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ -| *OutputData* | Job output data files | OutputData = ``{"output1","output2"};`` | +| *OutputData* [1] | Job output data files | OutputData = ``{"output1","output2"};`` | +---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ -| *OutputPath* | The output data path in the File Catalog | OutputPath = ``{"/myjobs/output"};`` | +| *OutputPath* [2] | The output data path in the File Catalog | OutputPath = ``{"/myjobs/output"};`` | +---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ -| *OutputSE* | The output data Storage Element | OutputSE = ``{"DIRAC-USER"};`` | +| *OutputSE* [3] | The output data Storage Element | OutputSE = ``{"DIRAC-USER"};`` | +---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ | | | :subtitle:`Parametric Jobs` | @@ -91,3 +91,24 @@ In this section all the attributes that can be used in the DIRAC JDL job descrip +---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ | *ParameterFactor* | Parameter multiplier | ParameterFactor = 1.1; (default 1.) | +---------------------+---------------------------------------------+-------------------------------------------------------------------------------------+ + +1. Elements of OutputData can be specified in several forms: + + - file names; in this case files with the specified names will be looked for in the job directory and uploaded + to a location specified by the OutputPath; + - file names with wild cards; same after the file names expansion; + - file names in a form "LFN:/vo/full/destination/path/file.name"; in this case the file will be uploaded + to the specified LFN path without taking into account the OutputPath. Note that file.name here can be also + specified with wild cards. + +2. The OutputPath can be specified in several ways + + - if not given it will be taken as the user's home directory + the job directory + for example "/lhcb/user/a/atsareg/1234/1234567", where 1234567 is the job ID; + - if given as path starting with "/", it will be appended to the user's home + directory; + - if given as "LFN:/output/path", it will be taken as an absolute path for + output files in the logical namespace. + +3. If multiple output SEs are specified, they will be tried one-by-one for each + output file until a successful file upload. From c4b8fd945d4dab51dcdfe5158c64ec48b1118cab Mon Sep 17 00:00:00 2001 From: Andrei Tsaregorodtsev Date: Mon, 15 Jun 2026 17:47:41 +0200 Subject: [PATCH 5/5] feat: improve the docs quality --- .../UserJobs/JDLReference/index.rst | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst b/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst index 1ca6a8e2fcd..a6e556a9317 100644 --- a/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst +++ b/docs/source/UserGuide/GettingStarted/UserJobs/JDLReference/index.rst @@ -94,21 +94,23 @@ In this section all the attributes that can be used in the DIRAC JDL job descrip 1. Elements of OutputData can be specified in several forms: - - file names; in this case files with the specified names will be looked for in the job directory and uploaded - to a location specified by the OutputPath; - - file names with wild cards; same after the file names expansion; - - file names in a form "LFN:/vo/full/destination/path/file.name"; in this case the file will be uploaded - to the specified LFN path without taking into account the OutputPath. Note that file.name here can be also - specified with wild cards. + - filenames; in this case files with the specified names will be looked for in the job directory and uploaded + to a location specified by the OutputPath (see below); + - filenames with wild cards, e.g. "*.log"; same after the filenames expansion; + - output data specified in a form "LFN:/vo/full/destination/path/filename"; in this case the file "filename" in + the job directory will be uploaded to the specified LFN path without taking into account the OutputPath. + Note that "filename" here can be also specified with wild cards, e.g. "LFN:/vo/full/destination/path/*.log". 2. The OutputPath can be specified in several ways - - if not given it will be taken as the user's home directory + the job directory + - if not given, it will be taken as the user's home directory + the job directory for example "/lhcb/user/a/atsareg/1234/1234567", where 1234567 is the job ID; - - if given as path starting with "/", it will be appended to the user's home - directory; + - if given as a path starting with "/", it will be appended to the user's home + directory, e.g. outputPath = "/my/analysis" will make output files to go to the + "/lhcb/user/a/atsareg/my/analysis" directory - if given as "LFN:/output/path", it will be taken as an absolute path for - output files in the logical namespace. + output files in the logical namespace. It is the responsibility of the user to make + sure that this path is accessible for writing for the user's data. 3. If multiple output SEs are specified, they will be tried one-by-one for each output file until a successful file upload.