Changes to avoid parallel execution issue on SLSA services

- Create custom-cache-tag for pkglistgen to enable separate cache dir
  name in case of parallel running
- Add custom-cache-tag to SLSA services
- Check for systemd service instead of process to ensure that SLSA
  services will not run in parallel
- Change pkglisgen timer to avoid overlap with relpkggen timer

generate-release-packages

@@ -9,13 +9,6 @@ logger() {
     echo "$1" >> ${LOG_DIR}/relpkggen.log
 }
 
-logger "[START] Start relpkggen service"
+logger "[START] Start osrt-relpkggen@${PROJECT}.service"
-if pgrep "osrt-pkglistgen"; then
+/usr/bin/osrt-pkglistgen -A ${API_URL} --debug update_and_solve -p ${PROJECT} -s target --only-release-packages --custom-cache-tag releasepackages --force >> ${LOG_DIR}/relpkggen.log 2>&1
-    logger "[WARNING] osrt-pkglistgen is running"
+logger "[FINISH] Finish osrt-relpkggen@${PROJECT}.service"
-    logger "[SKIP] Skip execution due to osrt-pkglistgen service is running"
-    exit 0
-fi
-
-logger "[RUNNING] Running osrt-relpkggen"
-/usr/bin/osrt-pkglistgen -A ${API_URL} --debug update_and_solve -p ${PROJECT} -s target --only-release-packages --force >> ${LOG_DIR}/relpkggen.log 2>&1
-logger "[FINISH] Finish relpkggen service"

pkglistgen/cli.py

@@ -49,6 +49,7 @@ class CommandLineInterface(ToolBase.CommandLineInterface):
     @cmdln.option('--stop-after-solve', action='store_true', help='only create group files')
     @cmdln.option('--staging', help='Only solve that one staging')
     @cmdln.option('--only-release-packages', action='store_true', help='Generate 000release-packages only')
+    @cmdln.option('--custom-cache-tag', help='add custom tag to cache dir to avoid issues when running in parallel')
     def do_update_and_solve(self, subcmd, opts):
         """${cmd_name}: update and solve for given scope
 
@@ -94,7 +95,8 @@ class CommandLineInterface(ToolBase.CommandLineInterface):
                                                          project=project, scope=scope, force=opts.force,
                                                          no_checkout=opts.no_checkout,
                                                          only_release_packages=opts.only_release_packages,
-                                                         stop_after_solve=opts.stop_after_solve)
+                                                         stop_after_solve=opts.stop_after_solve,
+                                                         custom_cache_tag=opts.custom_cache_tag)
             except MismatchedRepoException:
                 logging.error("Failed to create weakremovers.inc due to mismatch in repos - project most likey started building again.")
                 # for stagings we have to be strict on the exit value

pkglistgen/tool.py

@@ -611,7 +611,8 @@ class PkgListGen(ToolBase.ToolBase):
         force: bool,
         no_checkout: bool,
         only_release_packages: bool,
-        stop_after_solve: bool
+        stop_after_solve: bool,
+        custom_cache_tag
     ):
         self.all_architectures = target_config.get('pkglistgen-archs').split(' ')
         self.use_newest_version = str2bool(target_config.get('pkglistgen-use-newest-version', 'False'))
@@ -651,7 +652,10 @@ class PkgListGen(ToolBase.ToolBase):
 
         # Cache dir specific to hostname and project.
         host = urlparse(api.apiurl).hostname
-        cache_dir = CacheManager.directory('pkglistgen', host, project)
+        prefix_dir = 'pkglistgen'
+        if custom_cache_tag:
+            prefix_dir += '-' + custom_cache_tag
+        cache_dir = CacheManager.directory(prefix_dir, host, project)
 
         if not no_checkout:
             if os.path.exists(cache_dir):

systemd/osrt-pkglistgen@.timer

@@ -1,6 +1,6 @@
 [Timer]
 OnBootSec=420
-OnCalendar=*:0/10
+OnCalendar=*:05,15,25,35,45,55
 Unit=osrt-pkglistgen@.service
 
 [Install]

systemd/osrt-relpkggen@.service

@@ -7,6 +7,7 @@ SyslogIdentifier=osrt-slsa
 EnvironmentFile=/etc/openSUSE-release-tools/ibsapi
 Environment="OSC_CONFIG=/etc/openSUSE-release-tools/oscrc"
 WorkingDirectory=/var/lib/osrt-slsa/relpkggen
+ExecStartPre=/bin/bash -xc '/usr/bin/systemctl is-active --quiet osrt-pkglistgen@%i.service && exit 1 || exit 0'
 ExecStart=/usr/bin/osrt-generate-release-packages %i
 RuntimeMaxSec=120 hour
 

verify-build-and-generatelists

@@ -10,6 +10,7 @@ logger() {
 }
 
 polling_repo() {
+    logger "[CHECKING] Checking standard repository from ${PROJECT}"
     OUTPUT=$(/usr/share/openSUSE-release-tools/verify-repo-built-successful.py -A ${API_URL} -p ${PROJECT} -r standard 2>&1)
     RETURNCODE=$?
     if [ ${RETURNCODE} -eq 0 ]; then
@@ -34,25 +35,25 @@ polling_repo() {
     return ${RETURNCODE}
 }
 
-logger "[START] Start pkglistgen service: polling repository results"
+logger "[START] Start osrt-pkglistgen@${PROJECT}.service: polling repository results"
 polling_repo
 # Check if repository building
 if [ $? -eq 3 ]; then
     # Repository is building, waiting it finishes
     while true; do
-        logger "[WAITING] Waiting 20min before next repository status checking"
+        logger "[WAITING] Waiting 10min before next repository status check"
-        sleep 20m
+        sleep 10m
         polling_repo
         case $? in
             0)
                 # Stop polling repository results: Repository finished building
-                if pgrep "osrt-pkglistgen"; then
+                if systemctl is-active --quiet osrt-relpkggen@${PROJECT}.service; then
-                    logger "[WARNING] osrt-relpkggen is running"
+                    logger "[WARNING] osrt-relpkggen@${PROJECT}.service is running"
-                    logger "[WAITING] Waiting for osrt-relpkggen service finishes"
+                    logger "[WAITING] Waiting for osrt-relpkggen@${PROJECT}.service finishes"
                     continue
                 else
                     logger"[RUNNING] Running osrt-pkglistgen"
-                    /usr/bin/osrt-pkglistgen -A ${API_URL} --debug update_and_solve -p ${PROJECT} -s target --force >> ${LOG_DIR}/pkglistgen.log 2>&1
+                    /usr/bin/osrt-pkglistgen -A ${API_URL} --debug update_and_solve -p ${PROJECT} -s target --custom-cache-tag product --force >> ${LOG_DIR}/pkglistgen.log 2>&1
                     exit $?
                 fi
                 ;;
@@ -67,4 +68,4 @@ if [ $? -eq 3 ]; then
         esac
     done
 fi
-logger "[FINISH] Finish pkglistgen service"
+logger "[FINISH] Finish osrt-pkglistgen@${PROJECT}.service"