Refactor SLSA services to avoid parallel execution

- Refactor log
- Unify log for pkglistgen
- Replace external while true loop with a systemd timer for pkglistgen
- Add process check on verify-build-and-generatelists and
  generate-release-packages to avoid start pkglistgen when there is an
  instance that is already running it
- SLSA services must not share the same workdir

Makefile

@@ -11,7 +11,7 @@ VERSION = "build-$(shell date +%F)"
 all:
 
 install:
-	install -d -m 755 $(DESTDIR)$(bindir) $(DESTDIR)$(pkgdatadir) $(DESTDIR)$(unitdir) $(DESTDIR)$(oscplugindir) $(DESTDIR)$(sysconfdir)/$(package_name) $(DESTDIR)$(grafana_provisioning_dir)/dashboards $(DESTDIR)$(grafana_provisioning_dir)/datasources $(DESTDIR)$(logdir)/$(package_name) $(DESTDIR)$(varlibdir)/osrt-slsa
+	install -d -m 755 $(DESTDIR)$(bindir) $(DESTDIR)$(pkgdatadir) $(DESTDIR)$(unitdir) $(DESTDIR)$(oscplugindir) $(DESTDIR)$(sysconfdir)/$(package_name) $(DESTDIR)$(grafana_provisioning_dir)/dashboards $(DESTDIR)$(grafana_provisioning_dir)/datasources $(DESTDIR)$(logdir)/$(package_name) $(DESTDIR)$(varlibdir)/osrt-slsa/pkglistgen $(DESTDIR)$(varlibdir)/osrt-slsa/relpkggen
 	for i in $(pkgdata_SCRIPTS); do install -m 755 $$i $(DESTDIR)$(pkgdatadir); done
 	chmod 644 $(DESTDIR)$(pkgdatadir)/osc-*.py
 	for i in $(pkgdata_DATA); do cp -a $$i $(DESTDIR)$(pkgdatadir); done

generate-release-packages

@@ -4,5 +4,17 @@ PROJECT=$1
 LOG_DIR="/var/log/openSUSE-release-tools/${PROJECT}"
 [ ! -d "${LOG_DIR}" ] && mkdir ${LOG_DIR}
 
-date -Is >> ${LOG_DIR}/relpkggen.log
+logger() {
+    date -Is >> ${LOG_DIR}/relpkggen.log
+    echo "$1" >> ${LOG_DIR}/relpkggen.log
+}
+
+logger "[START] Start relpkgen service"
+if pgrep "osrt-pkglistgen"; then
+    logger "[SKIP] Skip execution due to osrt-pkglistgen service is running"
+    exit 0
+fi
+
+logger "[RUNNING] Running osrt-relpkggen"
 /usr/bin/osrt-pkglistgen -A ${API_URL} update_and_solve -p ${PROJECT} -s target --only-release-packages --force >> ${LOG_DIR}/relpkggen.log 2>&1
+logger "[FINISH] Finish relpkgen service"

systemd/osrt-pkglistgen@.service

@@ -6,7 +6,7 @@ User=osrt-slsa
 SyslogIdentifier=osrt-slsa
 EnvironmentFile=/etc/openSUSE-release-tools/ibsapi
 Environment="OSC_CONFIG=/etc/openSUSE-release-tools/oscrc"
-WorkingDirectory=/var/lib/osrt-slsa/
+WorkingDirectory=/var/lib/osrt-slsa/pkglistgen
 ExecStart=/usr/bin/osrt-verify-build-and-generatelists %i
 
 [Install]

systemd/osrt-pkglistgen@.timer

@@ -0,0 +1,7 @@
+[Timer]
+OnBootSec=240
+OnCalendar=*:0/20
+Unit=osrt-pkglistgen@.service
+
+[Install]
+WantedBy=timers.target

systemd/osrt-relpkggen@.service

@@ -6,7 +6,7 @@ User=osrt-slsa
 SyslogIdentifier=osrt-slsa
 EnvironmentFile=/etc/openSUSE-release-tools/ibsapi
 Environment="OSC_CONFIG=/etc/openSUSE-release-tools/oscrc"
-WorkingDirectory=/var/lib/osrt-slsa/
+WorkingDirectory=/var/lib/osrt-slsa/relpkggen
 ExecStart=/usr/bin/osrt-generate-release-packages %i
 RuntimeMaxSec=120 hour
 

verify-build-and-generatelists

@@ -4,62 +4,66 @@ PROJECT=$1
 LOG_DIR="/var/log/openSUSE-release-tools/${PROJECT}"
 [ ! -d "${LOG_DIR}" ] && mkdir ${LOG_DIR}
 
+logger() {
+    date -Is >> ${LOG_DIR}/pkglistgen.log
+    echo "$1" >> ${LOG_DIR}/pkglistgen.log
+}
+
 polling_repo() {
     # poll repository every 20min
     sleep 20m
     OUTPUT=$(/usr/share/openSUSE-release-tools/verify-repo-built-successful.py -A ${API_URL} -p ${PROJECT} -r standard 2>&1)
     RETURNCODE=$?
     if [ ${RETURNCODE} -eq 0 ]; then
-        REPO_STATUS="Repository is NOT building"
+        logger "[READY] Repository is NOT building"
     else
         case ${OUTPUT} in
             *"Repository "*" is not yet finished"*)
-                REPO_STATUS="Repository is building"
+                logger "[BUILDING] Repository is building"
                 RETURNCODE=3
                 ;;
             *"Repository "*" has "*" packages"*)
-                REPO_STATUS="Repository has package build failed"
+                logger "[FAILED] Repository has package build failed"
                 RETURNCODE=4
                 ;;
             *"Repository "*" has more disabled packages than succeeded"*)
-                REPO_STATUS="Repository has more disabled packages than succeeded"
+                logger "[ERROR] Repository has more disabled packages than succeeded"
                 RETURNCODE=5
                 ;;
         esac
-        date -Is >> ${LOG_DIR}/verify-build.log
+        logger "${OUTPUT}"
-        echo ${REPO_STATUS} >> ${LOG_DIR}/verify-build.log
-        echo ${OUTPUT} >> ${LOG_DIR}/verify-build.log
     fi
     return ${RETURNCODE}
 }
 
-while true; do
+logger "[START] Start pkglistgen service: polling repository results"
-    polling_repo
+polling_repo
-    # Check if repository building
+# Check if repository building
-    if [ $? -eq 3 ]; then
+if [ $? -eq 3 ]; then
     # Repository is building, waiting it finishes
     while true; do
         polling_repo
         case $? in
             0)
-                    # Stop polling repository results: Repository finished building, run pkglistgen
+                # Stop polling repository results: Repository finished building
-                    date -Is >> ${LOG_DIR}/pkglistgen.log
+                if pgrep "osrt-pkglistgen"; then
+                    logger "[WAITING] Waiting for osrt-relpkggen service finishes"
+                    continue
+                else
+                    logger"[RUNNING] Running osrt-pkglistgen"
                     /usr/bin/osrt-pkglistgen -A ${API_URL} --debug update_and_solve -p ${PROJECT} -s target --force >> ${LOG_DIR}/pkglistgen.log 2>&1
-                    break
+                    exit $?
+                fi
                 ;;
             3)
                 # Continue polling repository results: Repository is still building
                 continue
                 ;;
-                4)
+            *)
-                    # Stop polling repository results: Repository has package build failed
+                # Stop polling repository results: Failed polling results
-                    break
+                exit 1
-                    ;;
-                5)
-                    # Stop polling repository results: Repository has more disabled packages than succeeded
-                    break
                 ;;
         esac
     done
-    fi
+fi
-done
+logger "[FINISH] Finish pkglistgen service"