ironic-image/Dockerfile

@@ -69,10 +69,14 @@ RUN mkdir -p $GRUB_DIR
 
 COPY scripts/ /bin/
 COPY configure-nonroot.sh /bin/
-RUN set -euo pipefail; chmod +x /bin/configure-ironic.sh /bin/rundnsmasq /bin/runhttpd /bin/runironic /bin/runlogwatch.sh /bin/configure-nonroot.sh
+RUN set -euo pipefail; chmod +x /bin/configure-ironic.sh /bin/ironic-probe.sh /bin/rundatabase-upgrade /bin/rundnsmasq /bin/runhttpd /bin/runironic /bin/runlogwatch.sh /bin/runonline-data-migrations /bin/configure-nonroot.sh
+
+RUN mv /bin/ironic-probe.sh /bin/ironic-readiness
+RUN cp /bin/ironic-readiness /bin/ironic-liveness
 
 COPY ironic-config/inspector.ipxe.j2 ironic-config/httpd-ironic-api.conf.j2 \
-     ironic-config/ipxe_config.template /tmp/
+     ironic-config/ipxe_config.template ironic-config/dnsmasq.conf.j2 \
+     /tmp/
 
 # IRONIC #
 RUN cp /usr/share/ipxe/undionly.kpxe /tftpboot/undionly.kpxe
@@ -88,11 +92,7 @@ RUN if [ "$(uname -m)" = "aarch64" ]; then\
 COPY --from=base /tmp/esp-x86_64.img /tmp/uefi_esp-x86_64.img
 COPY --from=base /tmp/esp-aarch64.img /tmp/uefi_esp-arm64.img
 
-COPY ironic-config/ironic.conf.j2 /etc/ironic/
-COPY ironic-config/network-data-schema-empty.json /etc/ironic/
-
-# DNSMASQ
-COPY ironic-config/dnsmasq.conf.j2 /etc/
+COPY ironic-config/ironic.conf.j2 ironic-config/network-data-schema-empty.json /etc/ironic/
 
 # Workaround
 # Removing the 010-ironic.conf file that comes with the package 
@@ -101,8 +101,8 @@ RUN rm /etc/ironic/ironic.conf.d/010-ironic.conf
 # Custom httpd config, removes all but the bare minimum needed modules
 COPY ironic-config/httpd.conf.j2 /etc/httpd/conf/
 COPY ironic-config/httpd-modules.conf /etc/httpd/conf.modules.d/
-COPY ironic-config/apache2-vmedia.conf.j2 /etc/httpd-vmedia.conf.j2
-COPY ironic-config/apache2-ipxe.conf.j2 /etc/httpd-ipxe.conf.j2
+COPY ironic-config/apache2-vmedia.conf.j2 /tmp/httpd-vmedia.conf.j2
+COPY ironic-config/apache2-ipxe.conf.j2 /tmp/httpd-ipxe.conf.j2
 
 # configure non-root user and set relevant permissions
 RUN configure-nonroot.sh && rm -f /bin/configure-nonroot.sh

ironic-image/configure-nonroot.sh

@@ -1,53 +1,70 @@
 #!/usr/bin/bash
 
+# This script changes permissions to allow Ironic container to run as non-root
+# user. As the same image is used to run ironic, ironic-httpd, ironic-dsnmasq,
+# and ironic-log-watch via BMO's ironic k8s manifest, it has
+# to be configured to work with multiple different users and groups, while they
+# share files via bind mounts (/shared, /certs/*), which can only get one
+# group id as "fsGroup". Additionally, dnsmasq needs three capabilities to run
+# which we provide via "setcap", and "allowPrivilegeEscalation: true" in
+# manifest.
+
+set -eux
+
+# user and group are from ironic rpms (uid 997, gid 994)
 NONROOT_UID=10475
 NONROOT_GID=10475
-USER="ironic-suse"
+IRONIC_USER="ironic-suse"
+IRONIC_GROUP="ironic-suse"
 
-groupadd -r -g ${NONROOT_GID} ${USER}
+groupadd -r -g ${NONROOT_GID} ${IRONIC_GROUP}
 useradd -r -g ${NONROOT_GID} \
            -u ${NONROOT_UID} \
            -d /var/lib/ironic \
            -s /sbin/nologin \
-           ${USER}
+           ${IRONIC_USER}
 
-# create ironic's http_root directory
-mkdir -p /shared/html
-chown "${NONROOT_UID}":"${NONROOT_GID}" /shared/html
+# most containers mount /shared but dnsmasq can live without it
+mkdir -p /shared
+mkdir -p /data
+mkdir -p /conf
+chown "${IRONIC_USER}":"${IRONIC_GROUP}" /shared
+chown "${IRONIC_USER}":"${IRONIC_GROUP}" /data
+chown "${IRONIC_USER}":"${IRONIC_GROUP}" /conf
 
 # we'll bind mount shared ca and ironic certificate dirs here
 # that need to have correct ownership as the entire ironic in BMO
 # deployment shares a single fsGroup in manifest's securityContext
 mkdir -p /certs/ca
-chown "${NONROOT_UID}":"${NONROOT_GID}" /certs{,/ca}
+chown "${IRONIC_USER}":"${IRONIC_GROUP}" /certs{,/ca}
 chmod 2775 /certs{,/ca}
 
 # apache2 permission changes
-chown -R "${NONROOT_UID}":"${NONROOT_GID}" /etc/apache2
-chown -R "${NONROOT_UID}":"${NONROOT_GID}" /run
+chown -R "${IRONIC_USER}":"${IRONIC_GROUP}" /etc/apache2
+chown -R "${IRONIC_USER}":"${IRONIC_GROUP}" /run
 
 # ironic and httpd related changes
 mkdir -p /etc/httpd/conf.d
-chown -R "${NONROOT_UID}":"${NONROOT_GID}" /etc/ironic /etc/httpd /etc/httpd
-chown -R "${NONROOT_UID}":"${NONROOT_GID}" /var/log
+chown -R "${IRONIC_USER}":"${IRONIC_GROUP}" /etc/ironic /etc/httpd/conf /etc/httpd/conf.d
 chmod 2775 /etc/ironic /etc/httpd/conf /etc/httpd/conf.d
-chmod 664 /etc/ironic/* /etc/httpd/conf/* /etc/httpd/conf.d/*
+#chmod 664 /etc/ironic/* /etc/httpd/conf/* /etc/httpd/conf.d/*
+chmod 664 /etc/ironic/* /etc/httpd/conf/*
 
-chown -R "${NONROOT_UID}":"${NONROOT_GID}" /var/lib/ironic
+chown -R "${IRONIC_USER}":"${IRONIC_GROUP}" /var/lib/ironic
+chmod 2775 /var/lib/ironic
 chmod 664 /var/lib/ironic/ironic.sqlite
 
 # dnsmasq, and the capabilities required to run it as non-root user
-chown -R "${NONROOT_UID}":"${NONROOT_GID}" /etc/dnsmasq.conf /var/lib/dnsmasq
-chmod 2775 /var/lib/dnsmasq
-touch /var/lib/dnsmasq/dnsmasq.leases
-chmod 664 /etc/dnsmasq.conf /var/lib/dnsmasq/dnsmasq.leases
+chown -R "${IRONIC_USER}":"${IRONIC_GROUP}" /etc/dnsmasq.conf
+#handled at chart level
+#setcap "cap_net_raw,cap_net_admin,cap_net_bind_service=+eip" /usr/sbin/dnsmasq
 
 # ca-certificates permission changes
 touch /var/lib/ca-certificates/ca-bundle.pem.new
-chown -R "${NONROOT_UID}":"${NONROOT_GID}" /var/lib/ca-certificates/
+chown -R "${IRONIC_USER}":"${IRONIC_GROUP}" /var/lib/ca-certificates/
 chmod -R +w /var/lib/ca-certificates/
 
 # probes that are created before start
 touch /bin/ironic-{readi,live}ness
-chown root:"${NONROOT_GID}" /bin/ironic-{readi,live}ness
+chown root:"${IRONIC_GROUP}" /bin/ironic-{readi,live}ness
 chmod 775 /bin/ironic-{readi,live}ness

ironic-image/ironic-config/apache2-vmedia.conf.j2

@@ -10,15 +10,13 @@ Listen {{ env.VMEDIA_TLS_PORT }}
     SSLCertificateFile {{ env.IRONIC_VMEDIA_CERT_FILE }}
     SSLCertificateKeyFile {{ env.IRONIC_VMEDIA_KEY_FILE }}
 
-    <Directory "/shared">
-        AllowOverride None
-        Require all granted
+    <Directory ~ "/shared/html">
+         Order deny,allow
+         deny from all
     </Directory>
-
-    <Directory "/shared/html">
-        Options Indexes FollowSymLinks
-        AllowOverride None
-        Require all granted
+    <Directory ~ "/shared/html/(redfish|ilo)/">
+         Order allow,deny
+         allow from all
     </Directory>
 </VirtualHost>
 

ironic-image/ironic-config/dnsmasq.conf.j2

@@ -3,6 +3,7 @@ bind-dynamic
 enable-tftp
 tftp-root=/shared/tftpboot
 log-queries
+dhcp-leasefile=/data/dnsmasq/dnsmasq.leases
 
 # Configure listening for DNS (0 disables DNS)
 port={{ env.DNS_PORT }}
@@ -31,11 +32,11 @@ dhcp-match=ipxe,175
 # Client is already running iPXE; move to next stage of chainloading
 {%- if env.IPXE_TLS_SETUP == "true"  %}
 # iPXE with (U)EFI
-dhcp-boot=tag:efi,tag:ipxe,http://{{ env.IRONIC_URL_HOST }}:{{ env.HTTP_PORT }}/custom-ipxe/snponly.efi
+dhcp-boot=tag:efi,tag:ipxe,{{ env.IRONIC_HTTP_URL }}/custom-ipxe/snponly.efi
 # iPXE with BIOS
-dhcp-boot=tag:ipxe,http://{{ env.IRONIC_URL_HOST }}:{{ env.HTTP_PORT }}/custom-ipxe/undionly.kpxe
+dhcp-boot=tag:ipxe,{{ env.IRONIC_HTTP_URL }}/custom-ipxe/undionly.kpxe
 {% else %}
-dhcp-boot=tag:ipxe,http://{{ env.IRONIC_URL_HOST }}:{{ env.HTTP_PORT }}/boot.ipxe
+dhcp-boot=tag:ipxe,{{ env.IRONIC_HTTP_URL }}/boot.ipxe
 {% endif %}
 
 # Note: Need to test EFI booting
@@ -59,8 +60,8 @@ ra-param={{ env.PROVISIONING_INTERFACE }},0,0
 
 dhcp-vendorclass=set:pxe6,enterprise:343,PXEClient
 dhcp-userclass=set:ipxe6,iPXE
-dhcp-option=tag:pxe6,option6:bootfile-url,tftp://{{ env.IRONIC_URL_HOST }}/snponly.efi
-dhcp-option=tag:ipxe6,option6:bootfile-url,http://{{ env.IRONIC_URL_HOST }}:{{ env.HTTP_PORT }}/boot.ipxe
+dhcp-option=tag:pxe6,option6:bootfile-url,{{ env.IRONIC_TFTP_URL }}/snponly.efi
+dhcp-option=tag:ipxe6,option6:bootfile-url,{{ env.IRONIC_HTTP_URL }}/boot.ipxe
 
 # It can be used when setting DNS or GW variables.
 {%- if env["GATEWAY_IP"] is undefined %}

ironic-image/ironic-config/httpd-ironic-api.conf.j2

@@ -45,7 +45,7 @@ Listen {{ env.IRONIC_URL_HOST }}:{{ env.IRONIC_LISTEN_PORT }}
          {% if "IRONIC_HTPASSWD" in env and env.IRONIC_HTPASSWD | length %}
             AuthType Basic
             AuthName "Restricted area"
-            AuthUserFile "/etc/ironic/htpasswd"
+            AuthUserFile {{ env.HTPASSWD_FILE }}
             Require valid-user
          {% endif %}
     </Location>

ironic-image/ironic-config/httpd.conf.j2

@@ -1,10 +1,10 @@
-ServerRoot "/etc/httpd"
+ServerRoot {{ env.HTTPD_DIR }}
 {%- if env.LISTEN_ALL_INTERFACES | lower == "true" %}
 Listen {{ env.HTTP_PORT }}
 {% else %}
 Listen {{ env.IRONIC_URL_HOST }}:{{ env.HTTP_PORT }}
 {% endif %}
-Include conf.modules.d/*.conf
+Include /etc/httpd/conf.modules.d/*.conf
 User ironic-suse
 Group ironic-suse
 

ironic-image/ironic-config/inspector.ipxe.j2

@@ -5,6 +5,6 @@ echo In inspector.ipxe
 imgfree
 # NOTE(dtantsur): keep inspection kernel params in [mdns]params in
 # ironic-inspector-image and configuration in configure-ironic.sh
-kernel --timeout 60000 http://{{ env.IRONIC_URL_HOST }}:{{ env.HTTP_PORT }}/images/ironic-python-agent-${buildarch}.kernel ipa-insecure=1 ipa-inspection-collectors={{ env.IRONIC_IPA_COLLECTORS }} systemd.journald.forward_to_console=yes BOOTIF=${mac} ipa-debug=1 ipa-enable-vlan-interfaces={{ env.IRONIC_ENABLE_VLAN_INTERFACES }} ipa-inspection-dhcp-all-interfaces=1 ipa-collect-lldp=1 {{ env.INSPECTOR_EXTRA_ARGS }} initrd=ironic-python-agent.initramfs {% if env.IRONIC_RAMDISK_SSH_KEY %}sshkey="{{ env.IRONIC_RAMDISK_SSH_KEY|trim }}"{% endif %} {{ env.IRONIC_KERNEL_PARAMS|trim }} || goto retry_boot
-initrd --timeout 60000 http://{{ env.IRONIC_URL_HOST }}:{{ env.HTTP_PORT }}/images/ironic-python-agent-${buildarch}.initramfs || goto retry_boot
+kernel --timeout 60000 {{ env.IRONIC_HTTP_URL }}/images/ironic-python-agent-${buildarch}.kernel ipa-insecure={{ env.IPA_INSECURE }} ipa-inspection-collectors={{ env.IRONIC_IPA_COLLECTORS }} systemd.journald.forward_to_console=yes BOOTIF=${mac} ipa-debug=1 ipa-enable-vlan-interfaces={{ env.IRONIC_ENABLE_VLAN_INTERFACES }} ipa-inspection-dhcp-all-interfaces=1 ipa-collect-lldp=1 {{ env.INSPECTOR_EXTRA_ARGS }} initrd=ironic-python-agent-${buildarch}.initramfs {% if env.IRONIC_RAMDISK_SSH_KEY %}sshkey="{{ env.IRONIC_RAMDISK_SSH_KEY|trim }}"{% endif %} {{ env.IRONIC_KERNEL_PARAMS|trim }} || goto retry_boot
+initrd --timeout 60000 {{ env.IRONIC_HTTP_URL }}/images/ironic-python-agent-${buildarch}.initramfs || goto retry_boot
 boot

ironic-image/ironic-config/ironic.conf.j2

@@ -27,6 +27,7 @@ use_stderr = true
 hash_ring_algorithm = sha256
 my_ip = {{ env.IRONIC_IP }}
 host = {{ env.IRONIC_CONDUCTOR_HOST }}
+tempdir = {{ env.IRONIC_TMP_DATA_DIR }}
 
 # If a path to a certificate is defined, use that first for webserver
 {% if env.WEBSERVER_CACERT_FILE %}
@@ -49,6 +50,7 @@ deploy_logs_local_path = /shared/log/ironic/deploy
 # retries here works around such problems without affecting the normal path.
 # See https://bugzilla.redhat.com/show_bug.cgi?id=1822763
 max_command_attempts = 30
+certificates_path = {{ env.IRONIC_GEN_CERT_DIR }}
 
 [api]
 {% if env.IRONIC_REVERSE_PROXY_SETUP == "true" %}
@@ -83,7 +85,7 @@ send_sensor_data = {{ env.SEND_SENSOR_DATA }}
 # Power state is checked every 60 seconds and BMC activity should
 # be avoided more often than once every sixty seconds.
 send_sensor_data_interval = 160
-bootloader = {{ env.IRONIC_BOOT_BASE_URL }}/uefi_esp-{{ env.DEPLOY_ARCHITECTURE }}.img
+bootloader = {{ env.IRONIC_HTTP_URL }}/uefi_esp-{{ env.DEPLOY_ARCHITECTURE }}.img
 verify_step_priority_override = management.clear_job_queue:90
 # We don't use this feature, and it creates an additional load on the database
 node_history = False
@@ -95,16 +97,19 @@ deploy_kernel = file://{{ env.IRONIC_DEFAULT_KERNEL }}
 {% if env.IRONIC_DEFAULT_RAMDISK is defined %}
 deploy_ramdisk = file://{{ env.IRONIC_DEFAULT_RAMDISK }}
 {% endif %}
+{% if env.DISABLE_DEEP_IMAGE_INSPECTION | lower == "true" %}
+disable_deep_image_inspection = True
+{% endif %}
 
 [database]
-{% if env.IRONIC_USE_MARIADB | lower == "false" %}
-connection = sqlite:////var/lib/ironic/ironic.sqlite
+{% if env.IRONIC_USE_MARIADB | lower == "true" %}
+connection = {{ env.MARIADB_CONNECTION }}
+{% else %}
+connection = {{ env.LOCAL_DB_URI }}
 # Synchronous mode is required for data integrity in case of operating system
 # crash. In our case we restart the container from scratch, so we can save some
 # IO by not doing syncs all the time.
 sqlite_synchronous = False
-{% else %}
-connection = {{ env.MARIADB_CONNECTION }}
 {% endif %}
 
 [deploy]
@@ -112,7 +117,7 @@ default_boot_option = local
 erase_devices_metadata_priority = 10
 erase_devices_priority = 0
 http_root = /shared/html/
-http_url = {{ env.IRONIC_BOOT_BASE_URL }}
+http_url = {{ env.IRONIC_HTTP_URL }}
 fast_track = {{ env.IRONIC_FAST_TRACK }}
 {% if env.IRONIC_BOOT_ISO_SOURCE %}
 ramdisk_image_download_source = {{ env.IRONIC_BOOT_ISO_SOURCE }}
@@ -175,7 +180,7 @@ cipher_suite_versions = 3,17
 # unauthenticated connections from other processes in the same host since the
 # containers are in host networking.
 auth_strategy = http_basic
-http_basic_auth_user_file = /etc/ironic/htpasswd-rpc
+http_basic_auth_user_file = {{ env.IRONIC_RPC_HTPASSWD_FILE }}
 host_ip = {% if env.LISTEN_ALL_INTERFACES | lower == "true" %}::{% else %}{{ env.IRONIC_IP }}{% endif %}
 {% if env.IRONIC_TLS_SETUP == "true" %}
 use_ssl = true
@@ -196,7 +201,7 @@ images_path = /shared/html/tmp
 instance_master_path = /shared/html/master_images
 tftp_master_path = /shared/tftpboot/master_images
 tftp_root = /shared/tftpboot
-kernel_append_params = nofb nomodeset vga=normal ipa-insecure={{ env.IPA_INSECURE }} {% if env.ENABLE_FIPS_IPA %}fips={{ env.ENABLE_FIPS_IPA|trim }}{% endif %} {% if env.IRONIC_RAMDISK_SSH_KEY %}sshkey="{{ env.IRONIC_RAMDISK_SSH_KEY|trim }}"{% endif %} {{ env.IRONIC_KERNEL_PARAMS|trim }} systemd.journald.forward_to_console=yes net.ifnames={{ '0' if env.PREDICTABLE_NIC_NAMES == 'false' else '1' }}
+kernel_append_params = nofb nomodeset vga=normal ipa-insecure={{ env.IPA_INSECURE }} {% if env.ENABLE_FIPS_IPA %}fips={{ env.ENABLE_FIPS_IPA|trim }}{% endif %} {% if env.IRONIC_RAMDISK_SSH_KEY %}sshkey="{{ env.IRONIC_RAMDISK_SSH_KEY|trim }}"{% endif %} {{ env.IRONIC_KERNEL_PARAMS|trim }} systemd.journald.forward_to_console=yes
 # This makes networking boot templates generated even for nodes using local
 # boot (the default), ensuring that they boot correctly even if they start
 # netbooting for some reason (e.g. with the noop management interface).
@@ -209,14 +214,14 @@ ipxe_config_template = /tmp/ipxe_config.template
 
 [redfish]
 use_swift = false
-kernel_append_params = nofb nomodeset vga=normal ipa-insecure={{ env.IPA_INSECURE }} {% if env.ENABLE_FIPS_IPA %}fips={{ env.ENABLE_FIPS_IPA|trim }}{% endif %} {% if env.IRONIC_RAMDISK_SSH_KEY %}sshkey="{{ env.IRONIC_RAMDISK_SSH_KEY|trim }}"{% endif %} {{ env.IRONIC_KERNEL_PARAMS|trim }} systemd.journald.forward_to_console=yes net.ifnames={{ '0' if env.PREDICTABLE_NIC_NAMES == 'false' else '1' }}
+kernel_append_params = nofb nomodeset vga=normal ipa-insecure={{ env.IPA_INSECURE }} {% if env.ENABLE_FIPS_IPA %}fips={{ env.ENABLE_FIPS_IPA|trim }}{% endif %} {% if env.IRONIC_RAMDISK_SSH_KEY %}sshkey="{{ env.IRONIC_RAMDISK_SSH_KEY|trim }}"{% endif %} {{ env.IRONIC_KERNEL_PARAMS|trim }} systemd.journald.forward_to_console=yes
 
 [ilo]
-kernel_append_params = nofb nomodeset vga=normal ipa-insecure={{ env.IPA_INSECURE }} {% if env.ENABLE_FIPS_IPA %}fips={{ env.ENABLE_FIPS_IPA|trim }}{% endif %} {% if env.IRONIC_RAMDISK_SSH_KEY %}sshkey="{{ env.IRONIC_RAMDISK_SSH_KEY|trim }}"{% endif %} {{ env.IRONIC_KERNEL_PARAMS|trim }} systemd.journald.forward_to_console=yes net.ifnames={{ '0' if env.PREDICTABLE_NIC_NAMES == 'false' else '1' }}
+kernel_append_params = nofb nomodeset vga=normal ipa-insecure={{ env.IPA_INSECURE }} {% if env.ENABLE_FIPS_IPA %}fips={{ env.ENABLE_FIPS_IPA|trim }}{% endif %} {% if env.IRONIC_RAMDISK_SSH_KEY %}sshkey="{{ env.IRONIC_RAMDISK_SSH_KEY|trim }}"{% endif %} {{ env.IRONIC_KERNEL_PARAMS|trim }} systemd.journald.forward_to_console=yes
 use_web_server_for_images = true
 
 [irmc]
-kernel_append_params = nofb nomodeset vga=normal ipa-insecure={{ env.IPA_INSECURE }} {% if env.ENABLE_FIPS_IPA %}fips={{ env.ENABLE_FIPS_IPA|trim }}{% endif %} {% if env.IRONIC_RAMDISK_SSH_KEY %}sshkey="{{ env.IRONIC_RAMDISK_SSH_KEY|trim }}"{% endif %} {{ env.IRONIC_KERNEL_PARAMS|trim }} systemd.journald.forward_to_console=yes net.ifnames={{ '0' if env.PREDICTABLE_NIC_NAMES == 'false' else '1' }}
+kernel_append_params = nofb nomodeset vga=normal ipa-insecure={{ env.IPA_INSECURE }} {% if env.ENABLE_FIPS_IPA %}fips={{ env.ENABLE_FIPS_IPA|trim }}{% endif %} {% if env.IRONIC_RAMDISK_SSH_KEY %}sshkey="{{ env.IRONIC_RAMDISK_SSH_KEY|trim }}"{% endif %} {{ env.IRONIC_KERNEL_PARAMS|trim }} systemd.journald.forward_to_console=yes
 
 [service_catalog]
 endpoint_override = {{ env.IRONIC_BASE_URL }}

ironic-image/scripts/auth-common.sh

@@ -4,6 +4,12 @@ set -euxo pipefail
 
 export IRONIC_REVERSE_PROXY_SETUP=${IRONIC_REVERSE_PROXY_SETUP:-false}
 
+# CUSTOM_CONFIG_DIR is also managed in the ironic-common.sh, in order to
+# keep auth-common and ironic-common separate (to stay consistent with the
+# architecture) part of the ironic-common logic had to be duplicated
+CUSTOM_CONFIG_DIR="${CUSTOM_CONFIG_DIR:-/conf}"
+IRONIC_CONF_DIR="${CUSTOM_CONFIG_DIR}/ironic"
+
 # Backward compatibility
 if [[ "${IRONIC_DEPLOYMENT:-}" == "Conductor" ]]; then
     export IRONIC_EXPOSE_JSON_RPC=true
@@ -11,42 +17,74 @@ else
     export IRONIC_EXPOSE_JSON_RPC="${IRONIC_EXPOSE_JSON_RPC:-false}"
 fi
 
-IRONIC_HTPASSWD_FILE=/etc/ironic/htpasswd
+IRONIC_HTPASSWD_FILE="${IRONIC_CONF_DIR}/htpasswd"
+export IRONIC_RPC_HTPASSWD_FILE="${IRONIC_HTPASSWD_FILE}-rpc"
 if [[ -f "/auth/ironic/htpasswd" ]]; then
     IRONIC_HTPASSWD=$(</auth/ironic/htpasswd)
 fi
+if [[ -f "/auth/ironic-rpc/htpasswd" ]]; then
+    IRONIC_RPC_HTPASSWD=$(</auth/ironic-rpc/htpasswd)
+fi
 export IRONIC_HTPASSWD=${IRONIC_HTPASSWD:-${HTTP_BASIC_HTPASSWD:-}}
+export IRONIC_RPC_HTPASSWD=${IRONIC_RPC_HTPASSWD:-${IRONIC_HTPASSWD}}
 
-configure_client_basic_auth()
-{
-    local auth_config_file="/auth/$1/auth-config"
-    local dest="${2:-/etc/ironic/ironic.conf}"
-    if [[ -f "${auth_config_file}" ]]; then
-        # Merge configurations in the "auth" directory into the default ironic configuration file
-        crudini --merge "${dest}" < "${auth_config_file}"
-    fi
-}
+if [[ -n "${MARIADB_PASSWORD:-}" ]]; then
+    echo "WARNING: passing MARIADB_PASSWORD is deprecated, mount a secret under /auth/mariadb instead"
+elif [[ -f /auth/mariadb/password ]]; then
+    MARIADB_PASSWORD=$(</auth/mariadb/password)
+fi
+
+if [[ -z "${MARIADB_USER:-}" ]] && [[ -f /auth/mariadb/username ]]; then
+    MARIADB_USER=$(</auth/mariadb/username)
+fi
+
+IRONIC_CONFIG="${IRONIC_CONF_DIR}/ironic.conf"
 
 configure_json_rpc_auth()
 {
-    if [[ "${IRONIC_EXPOSE_JSON_RPC}" == "true" ]]; then
-        if [[ -z "${IRONIC_HTPASSWD}" ]]; then
+    if [[ "${IRONIC_EXPOSE_JSON_RPC}" != "true" ]]; then
+        return
+    fi
+
+    local auth_config_file="/auth/ironic-rpc/auth-config"
+    local username_file="/auth/ironic-rpc/username"
+    local password_file="/auth/ironic-rpc/password"
+    if [[ -f "${username_file}" ]] && [[ -f "${password_file}" ]]; then
+        crudini --set "${IRONIC_CONFIG}" json_rpc username "$(<${username_file})"
+        set +x
+        crudini --set "${IRONIC_CONFIG}" json_rpc password "$(<${password_file})"
+        set -x
+    elif [[ -f "${auth_config_file}" ]]; then
+        echo "WARNING: using auth-config is deprecated, mount a secret directly"
+        # Merge configurations in the "auth" directory into the default ironic configuration file
+        crudini --merge "${IRONIC_CONFIG}" < "${auth_config_file}"
+    else
+        echo "FATAL: no client-side credentials provided for JSON RPC"
+        echo "HINT: mount a secret with username and password fields under /auth/ironic-rpc"
+        exit 1
+    fi
+
+    if [[ -z "${IRONIC_RPC_HTPASSWD}" ]]; then
+        if [[ -f "${username_file}" ]] && [[ -f "${password_file}" ]]; then
+            htpasswd -c -i -B "${IRONIC_RPC_HTPASSWD_FILE}" "$(<${username_file})" <"${password_file}"
+        else
             echo "FATAL: enabling JSON RPC requires authentication"
+            echo "HINT: mount a secret with either username and password or htpasswd under /auth/ironic-rpc"
             exit 1
         fi
-        printf "%s\n" "${IRONIC_HTPASSWD}" > "${IRONIC_HTPASSWD_FILE}-rpc"
+    else
+        printf "%s\n" "${IRONIC_RPC_HTPASSWD}" > "${IRONIC_RPC_HTPASSWD_FILE}"
     fi
 }
 
 configure_ironic_auth()
 {
-    local config=/etc/ironic/ironic.conf
     # Configure HTTP basic auth for API server
     if [[ -n "${IRONIC_HTPASSWD}" ]]; then
         printf "%s\n" "${IRONIC_HTPASSWD}" > "${IRONIC_HTPASSWD_FILE}"
         if [[ "${IRONIC_REVERSE_PROXY_SETUP}" == "false" ]]; then
-            crudini --set "${config}" DEFAULT auth_strategy http_basic
-            crudini --set "${config}" DEFAULT http_basic_auth_user_file "${IRONIC_HTPASSWD_FILE}"
+            crudini --set "${IRONIC_CONFIG}" DEFAULT auth_strategy http_basic
+            crudini --set "${IRONIC_CONFIG}" DEFAULT http_basic_auth_user_file "${IRONIC_HTPASSWD_FILE}"
         fi
     fi
 }

ironic-image/scripts/configure-ironic.sh

@@ -19,10 +19,11 @@ export IRONIC_ENABLE_VLAN_INTERFACES=${IRONIC_ENABLE_VLAN_INTERFACES:-${IRONIC_I
 
 export HTTP_PORT=${HTTP_PORT:-80}
 
-export IRONIC_USE_MARIADB=${IRONIC_USE_MARIADB:-true}
-
-if [[ "$IRONIC_USE_MARIADB" == "true" ]]; then
-    MARIADB_PASSWORD=${MARIADB_PASSWORD}
+if [[ "${IRONIC_USE_MARIADB}" == true ]]; then
+    if [[ -z "${MARIADB_PASSWORD:-}" ]]; then
+        echo "FATAL: IRONIC_USE_MARIADB requires password, mount a secret under /auth/mariadb"
+        exit 1
+    fi
     MARIADB_DATABASE=${MARIADB_DATABASE:-ironic}
     MARIADB_USER=${MARIADB_USER:-ironic}
     MARIADB_HOST=${MARIADB_HOST:-127.0.0.1}
@@ -32,13 +33,9 @@ if [[ "$IRONIC_USE_MARIADB" == "true" ]]; then
     fi
 fi
 
-# TODO(dtantsur): remove the explicit default once we get
-# https://review.opendev.org/761185 in the repositories
-NUMPROC="$(grep -c "^processor" /proc/cpuinfo)"
-if [[ "$NUMPROC" -lt 4 ]]; then
-    NUMPROC=4
-fi
-export NUMWORKERS=${NUMWORKERS:-$NUMPROC}
+# zero makes it do cpu number detection on Ironic side
+export NUMWORKERS=${NUMWORKERS:-0}
+
 
 # Whether to enable fast_track provisioning or not
 export IRONIC_FAST_TRACK=${IRONIC_FAST_TRACK:-true}
@@ -57,8 +54,6 @@ wait_for_interface_or_ip
 # Hostname to use for the current conductor instance.
 export IRONIC_CONDUCTOR_HOST=${IRONIC_CONDUCTOR_HOST:-${IRONIC_URL_HOST}}
 
-export IRONIC_BASE_URL=${IRONIC_BASE_URL:-"${IRONIC_SCHEME}://${IRONIC_URL_HOST}:${IRONIC_ACCESS_PORT}"}
-
 if [[ -n "$IRONIC_EXTERNAL_IP" ]]; then
     export IRONIC_EXTERNAL_CALLBACK_URL=${IRONIC_EXTERNAL_CALLBACK_URL:-"${IRONIC_SCHEME}://${IRONIC_EXTERNAL_IP}:${IRONIC_ACCESS_PORT}"}
     if [[ "$IRONIC_VMEDIA_TLS_SETUP" == "true" ]]; then
@@ -74,9 +69,9 @@ if [[ -f "${IMAGE_CACHE_PREFIX}.kernel" ]] && [[ -f "${IMAGE_CACHE_PREFIX}.initr
     export IRONIC_DEFAULT_RAMDISK="${IMAGE_CACHE_PREFIX}.initramfs"
 fi
 
-if [[ -f /etc/ironic/ironic.conf ]]; then
+if [[ -f "${IRONIC_CONF_DIR}/ironic.conf" ]]; then
     # Make a copy of the original supposed empty configuration file
-    cp /etc/ironic/ironic.conf /etc/ironic/ironic.conf_orig
+    cp "${IRONIC_CONF_DIR}/ironic.conf" "${IRONIC_CONF_DIR}/ironic.conf.orig"
 fi
 
 # oslo.config also supports Config Opts From Environment, log them to stdout
@@ -85,34 +80,16 @@ env | grep "^OS_" || true
 
 mkdir -p /shared/html
 
-configure_json_rpc_auth
-
 if [[ -f /proc/sys/crypto/fips_enabled ]]; then
     ENABLE_FIPS_IPA=$(cat /proc/sys/crypto/fips_enabled)
     export ENABLE_FIPS_IPA
 fi
 
 # The original ironic.conf is empty, and can be found in ironic.conf_orig
-render_j2_config /etc/ironic/ironic.conf.j2 /etc/ironic/ironic.conf
+render_j2_config "/etc/ironic/ironic.conf.j2" \
+    "${IRONIC_CONF_DIR}/ironic.conf"
 
-configure_client_basic_auth ironic-rpc
+configure_json_rpc_auth
 
 # Make sure ironic traffic bypasses any proxies
 export NO_PROXY="${NO_PROXY:-},$IRONIC_IP"
-
-PROBE_CURL_ARGS=
-if [[ "${IRONIC_REVERSE_PROXY_SETUP}" == "true" ]]; then
-    if [[ "${IRONIC_PRIVATE_PORT}" == "unix" ]]; then
-        PROBE_URL="http://127.0.0.1:6385"
-        PROBE_CURL_ARGS="--unix-socket /shared/ironic.sock"
-    else
-        PROBE_URL="http://127.0.0.1:${IRONIC_PRIVATE_PORT}"
-    fi
-else
-        PROBE_URL="${IRONIC_BASE_URL}"
-fi
-export PROBE_CURL_ARGS
-export PROBE_URL
-
-PROBE_KIND=readiness render_j2_config /bin/ironic-probe.j2 /bin/ironic-readiness
-PROBE_KIND=liveness render_j2_config /bin/ironic-probe.j2 /bin/ironic-liveness

ironic-image/scripts/ironic-common.sh

@@ -2,11 +2,36 @@
 
 set -euxo pipefail
 
-IRONIC_IP="${IRONIC_IP:-}"
+# Export IRONIC_IP to avoid needing to lean on IRONIC_URL_HOST for consumption in
+# e.g. dnsmasq configuration
+export IRONIC_IP="${IRONIC_IP:-}"
 PROVISIONING_INTERFACE="${PROVISIONING_INTERFACE:-}"
 PROVISIONING_IP="${PROVISIONING_IP:-}"
 PROVISIONING_MACS="${PROVISIONING_MACS:-}"
 IPXE_CUSTOM_FIRMWARE_DIR="${IPXE_CUSTOM_FIRMWARE_DIR:-/shared/custom_ipxe_firmware}"
+CUSTOM_CONFIG_DIR="${CUSTOM_CONFIG_DIR:-/conf}"
+CUSTOM_DATA_DIR="${CUSTOM_DATA_DIR:-/data}"
+export DNSMASQ_CONF_DIR="${CUSTOM_CONFIG_DIR}/dnsmasq"
+export DNSMASQ_DATA_DIR="${CUSTOM_DATA_DIR}/dnsmasq"
+export DNSMASQ_TEMP_DIR="${CUSTOM_CONFIG_DIR}/dnsmasq"
+export HTTPD_DIR="${CUSTOM_CONFIG_DIR}/httpd"
+export HTTPD_CONF_DIR="${HTTPD_DIR}/conf"
+export HTTPD_CONF_DIR_D="${HTTPD_DIR}/conf.d"
+export IRONIC_CONF_DIR="${CUSTOM_CONFIG_DIR}/ironic"
+export IRONIC_DB_DIR="${CUSTOM_DATA_DIR}/db"
+export IRONIC_GEN_CERT_DIR="${CUSTOM_DATA_DIR}/auto_gen_certs"
+export IRONIC_TMP_DATA_DIR="${CUSTOM_DATA_DIR}/tmp"
+export PROBE_CONF_DIR="${CUSTOM_CONFIG_DIR}/probes"
+
+mkdir -p "${IRONIC_CONF_DIR}" "${PROBE_CONF_DIR}" "${HTTPD_CONF_DIR}" \
+    "${HTTPD_CONF_DIR_D}" "${DNSMASQ_CONF_DIR}" "${DNSMASQ_TEMP_DIR}" \
+    "${IRONIC_DB_DIR}" "${IRONIC_GEN_CERT_DIR}" "${DNSMASQ_DATA_DIR}" \
+    "${IRONIC_TMP_DATA_DIR}"
+
+export HTPASSWD_FILE="${IRONIC_CONF_DIR}/htpasswd"
+export LOCAL_DB_URI="sqlite:///${IRONIC_DB_DIR}/ironic.sqlite"
+
+export IRONIC_USE_MARIADB=${IRONIC_USE_MARIADB:-false}
 
 get_provisioning_interface()
 {
@@ -19,13 +44,13 @@ get_provisioning_interface()
     local interface="provisioning"
 
     if [[ -n "${PROVISIONING_IP}" ]]; then
-        if ip -br addr show | grep -qi " ${PROVISIONING_IP}/"; then
+        if ip -br addr show | grep -i " ${PROVISIONING_IP}/" &>/dev/null; then
             interface="$(ip -br addr show | grep -i " ${PROVISIONING_IP}/" | cut -f 1 -d ' ' | cut -f 1 -d '@')"
         fi
     fi
 
     for mac in ${PROVISIONING_MACS//,/ }; do
-        if ip -br link show up | grep -qi "$mac"; then
+        if ip -br link show up | grep -i "$mac" &>/dev/null; then
             interface="$(ip -br link show up | grep -i "$mac" | cut -f 1 -d ' ' | cut -f 1 -d '@')"
             break
         fi
@@ -42,9 +67,12 @@ export LISTEN_ALL_INTERFACES="${LISTEN_ALL_INTERFACES:-true}"
 # Wait for the interface or IP to be up, sets $IRONIC_IP
 wait_for_interface_or_ip()
 {
-    # If $PROVISIONING_IP is specified, then we wait for that to become available on an interface, otherwise we look at $PROVISIONING_INTERFACE for an IP
-    if [[ -n "$PROVISIONING_IP" ]]; then
-        # Convert the address using ipcalc which strips out the subnet. For IPv6 addresses, this will give the short-form address
+    # If $PROVISIONING_IP is specified, then we wait for that to become
+    # available on an interface, otherwise we look at $PROVISIONING_INTERFACE
+    # for an IP
+    if [[ -n "${PROVISIONING_IP}" ]]; then
+        # Convert the address using ipcalc which strips out the subnet.
+        # For IPv6 addresses, this will give the short-form address
         IRONIC_IP="$(ipcalc "${PROVISIONING_IP}" | grep "^Address:" | awk '{print $2}')"
         export IRONIC_IP
         until grep -F " ${IRONIC_IP}/" <(ip -br addr show); do
@@ -69,31 +97,37 @@ wait_for_interface_or_ip()
         export IPV=4
         export IRONIC_URL_HOST="$IRONIC_IP"
     fi
+
+    # Avoid having to construct full URL multiple times while allowing
+    # the override of IRONIC_HTTP_URL for environments in which IRONIC_IP
+    # is unreachable from hosts being provisioned.
+    export IRONIC_HTTP_URL="${IRONIC_HTTP_URL:-http://${IRONIC_URL_HOST}:${HTTP_PORT}}"
+    export IRONIC_TFTP_URL="${IRONIC_TFTP_URL:-tftp://${IRONIC_URL_HOST}}"
+    export IRONIC_BASE_URL=${IRONIC_BASE_URL:-"${IRONIC_SCHEME}://${IRONIC_URL_HOST}:${IRONIC_ACCESS_PORT}"}
 }
 
 render_j2_config()
 {
-    ls $1 # DEBUG
-    python3 -c 'import os; import sys; import jinja2; sys.stdout.write(jinja2.Template(sys.stdin.read()).render(env=os.environ))' < "$1"
-    python3 -c 'import os; import sys; import jinja2; sys.stdout.write(jinja2.Template(sys.stdin.read()).render(env=os.environ))' < "$1" > "$2"
-    ls $2 # DEBUG
+    python3.11 -c 'import os; import sys; import jinja2; sys.stdout.write(jinja2.Template(sys.stdin.read()).render(env=os.environ))' < "$1" > "$2"
 }
 
 run_ironic_dbsync()
 {
-    if [[ "${IRONIC_USE_MARIADB:-true}" == "true" ]]; then
+    if [[ "${IRONIC_USE_MARIADB}" == "true" ]]; then
         # It's possible for the dbsync to fail if mariadb is not up yet, so
         # retry until success
-        until ironic-dbsync --config-file /etc/ironic/ironic.conf upgrade; do
+        until ironic-dbsync --config-file "${IRONIC_CONF_DIR}/ironic.conf" upgrade; do
             echo "WARNING: ironic-dbsync failed, retrying"
             sleep 1
         done
     else
-        # SQLite does not support some statements. Fortunately, we can just create
-        # the schema in one go if not already created, instead of going through an upgrade
-        DB_VERSION="$(ironic-dbsync --config-file /etc/ironic/ironic.conf version)"
+        # SQLite does not support some statements. Fortunately, we can just
+        # create the schema in one go if not already created, instead of going
+        # through an upgrade
+        cp "/var/lib/ironic/ironic.sqlite" "${IRONIC_DB_DIR}/ironic.sqlite"
+        DB_VERSION="$(ironic-dbsync --config-file "${IRONIC_CONF_DIR}/ironic.conf" version)"
         if [[ "${DB_VERSION}" == "None" ]]; then
-            ironic-dbsync --config-file /etc/ironic/ironic.conf create_schema
+            ironic-dbsync --config-file "${IRONIC_CONF_DIR}/ironic.conf" create_schema
         fi
     fi
 }

ironic-image/scripts/ironic-probe.j2

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-set -eu -o pipefail
-
-curl -sSf {{ env.PROBE_CURL_ARGS }} "{{ env.PROBE_URL }}"
-
-# TODO(dtantsur): when PROBE_KIND==readiness, try the conductor and driver API
-# to make sure the conductor is ready. This requires having access to secrets
-# since these endpoints are authenticated.

ironic-image/scripts/ironic-probe.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+# shellcheck disable=SC1091
+. /bin/ironic-common.sh
+# shellcheck disable=SC1091
+. /bin/auth-common.sh
+
+PROBE_CURL_ARGS=
+if [[ "${IRONIC_REVERSE_PROXY_SETUP}" == "true" ]]; then
+    if [[ "${IRONIC_PRIVATE_PORT}" == "unix" ]]; then
+        PROBE_URL="http://127.0.0.1:6385"
+        PROBE_CURL_ARGS="--unix-socket /shared/ironic.sock"
+    else
+        PROBE_URL="http://127.0.0.1:${IRONIC_PRIVATE_PORT}"
+    fi
+else
+        PROBE_URL="${IRONIC_BASE_URL}"
+fi
+
+# shellcheck disable=SC2086
+curl -sSf ${PROBE_CURL_ARGS} "${PROBE_URL}"

ironic-image/scripts/rundatabase-upgrade

@@ -0,0 +1,10 @@
+#!/usr/bin/bash
+
+set -euxo pipefail
+
+# shellcheck disable=SC1091
+. /bin/configure-ironic.sh
+
+# NOTE(dtantsur): no retries here: this script is supposed to be run as a Job
+# that is retried on failure.
+exec ironic-dbsync --config-file "${IRONIC_CONF_DIR}/ironic.conf" upgrade

ironic-image/scripts/rundnsmasq

@@ -13,7 +13,11 @@ export DNS_PORT=${DNS_PORT:-0}
 
 wait_for_interface_or_ip
 if [[ "${DNS_IP:-}" == "provisioning" ]]; then
-    export DNS_IP="$IRONIC_URL_HOST"
+    if [[ "${IPV}" == "4" ]]; then
+      export DNS_IP="${IRONIC_IP}"
+    else
+      export DNS_IP="[${IRONIC_IP}]"
+    fi
 fi
 
 mkdir -p /shared/tftpboot
@@ -32,12 +36,12 @@ fi
 # Template and write dnsmasq.conf
 # we template via /tmp as sed otherwise creates temp files in /etc directory
 # where we can't write
-python3 -c 'import os; import sys; import jinja2; sys.stdout.write(jinja2.Template(sys.stdin.read()).render(env=os.environ))' </etc/dnsmasq.conf.j2 >/tmp/dnsmasq.conf
+python3.11 -c 'import os; import sys; import jinja2; sys.stdout.write(jinja2.Template(sys.stdin.read()).render(env=os.environ))' <"/tmp/dnsmasq.conf.j2" >"${DNSMASQ_TEMP_DIR}/dnsmasq_temp.conf"
 
 for iface in $(echo "$DNSMASQ_EXCEPT_INTERFACE" | tr ',' ' '); do
-    sed -i -e "/^interface=.*/ a\except-interface=${iface}" /tmp/dnsmasq.conf
+    sed -i -e "/^interface=.*/ a\except-interface=${iface}" "${DNSMASQ_TEMP_DIR}/dnsmasq_temp.conf"
 done
-cat /tmp/dnsmasq.conf > /etc/dnsmasq.conf
-rm /tmp/dnsmasq.conf
+cat "${DNSMASQ_TEMP_DIR}/dnsmasq_temp.conf" > "${DNSMASQ_CONF_DIR}/dnsmasq.conf"
+rm "${DNSMASQ_TEMP_DIR}/dnsmasq_temp.conf"
 
-exec /usr/sbin/dnsmasq -d -q -C /etc/dnsmasq.conf
+exec /usr/sbin/dnsmasq -d -q -C "${DNSMASQ_CONF_DIR}/dnsmasq.conf"

ironic-image/scripts/runhttpd

@@ -28,25 +28,29 @@ wait_for_interface_or_ip
 mkdir -p /shared/html
 chmod 0777 /shared/html
 
-IRONIC_BASE_URL="${IRONIC_SCHEME}://${IRONIC_URL_HOST}"
-
-INSPECTOR_EXTRA_ARGS=" ipa-inspection-callback-url=${IRONIC_BASE_URL}:${IRONIC_ACCESS_PORT}/v1/continue_inspection"
+INSPECTOR_EXTRA_ARGS=" ipa-inspection-callback-url=${IRONIC_BASE_URL}/v1/continue_inspection"
 
 if [[ "$IRONIC_FAST_TRACK" == "true" ]]; then
-    INSPECTOR_EXTRA_ARGS+=" ipa-api-url=${IRONIC_BASE_URL}:${IRONIC_ACCESS_PORT}"
+    INSPECTOR_EXTRA_ARGS+=" ipa-api-url=${IRONIC_BASE_URL}"
 fi
 export INSPECTOR_EXTRA_ARGS
 
 # Copy files to shared mount
 render_j2_config /tmp/inspector.ipxe.j2 /shared/html/inspector.ipxe
 cp /tmp/uefi_esp*.img /shared/html/
+# cp -r /etc/httpd/* "${HTTPD_DIR}"
+if [[ -f "${HTTPD_CONF_DIR}/httpd.conf" ]]; then
+    mv "${HTTPD_CONF_DIR}/httpd.conf" "${HTTPD_CONF_DIR}/httpd.conf.example"
+fi
 
 # Render the core httpd config
-render_j2_config /etc/httpd/conf/httpd.conf.j2 /etc/httpd/conf/httpd.conf
+render_j2_config "/etc/httpd/conf/httpd.conf.j2" \
+    "${HTTPD_CONF_DIR}/httpd.conf"
 
 if [[ "$IRONIC_TLS_SETUP" == "true" ]]; then
     if [[ "${IRONIC_REVERSE_PROXY_SETUP}" == "true" ]]; then
-        render_j2_config /tmp/httpd-ironic-api.conf.j2 /etc/httpd/conf.d/ironic.conf
+        render_j2_config "/tmp/httpd-ironic-api.conf.j2" \
+            "${HTTPD_CONF_DIR_D}/ironic.conf"
     fi
 else
     export IRONIC_REVERSE_PROXY_SETUP="false" # If TLS is not used, we have no reason to use the reverse proxy
@@ -56,33 +60,24 @@ write_htpasswd_files
 
 # Render httpd TLS configuration for /shared/html/<redifsh;ilo>
 if [[ "$IRONIC_VMEDIA_TLS_SETUP" == "true" ]]; then
-    render_j2_config /etc/httpd-vmedia.conf.j2 /etc/httpd/conf.d/vmedia.conf
+    render_j2_config "/tmp/httpd-vmedia.conf.j2" \
+        "${HTTPD_CONF_DIR_D}/vmedia.conf"
 fi
 
 # Render httpd TLS configuration for /shared/html
 if [[ "$IPXE_TLS_SETUP" == "true" ]]; then
     mkdir -p /shared/html/custom-ipxe
     chmod 0777 /shared/html/custom-ipxe
-    render_j2_config "/etc/httpd-ipxe.conf.j2" "/etc/httpd/conf.d/ipxe.conf"
+    render_j2_config "/tmp/httpd-ipxe.conf.j2" "${HTTPD_CONF_DIR_D}/ipxe.conf"
     cp "${IPXE_CUSTOM_FIRMWARE_DIR}/undionly.kpxe" \
        "${IPXE_CUSTOM_FIRMWARE_DIR}/snponly.efi" \
        "/shared/html/custom-ipxe"
 fi
 
 # Set up inotify to kill the container (restart) whenever cert files for ironic api change
-if [[ "$IRONIC_TLS_SETUP" == "true" ]] && [[ "${RESTART_CONTAINER_CERTIFICATE_UPDATED}" == "true" ]]; then
-    # shellcheck disable=SC2034
-    python3.11 -m pyinotify --raw-format -e IN_DELETE_SELF -v "${IRONIC_CERT_FILE}" | while read -r file event; do
-        kill -WINCH $(pgrep httpd)
-    done &
-fi
+configure_restart_on_certificate_update "${IRONIC_TLS_SETUP}" httpd "${IRONIC_CERT_FILE}"
 
 # Set up inotify to kill the container (restart) whenever cert of httpd for /shared/html/<redifsh;ilo> path change
-if [[ "$IRONIC_VMEDIA_TLS_SETUP" == "true" ]] && [[ "${RESTART_CONTAINER_CERTIFICATE_UPDATED}" == "true" ]]; then
-    # shellcheck disable=SC2034
-    python3.11 -m pyinotify --raw-format -e IN_DELETE_SELF -v "${IRONIC_VMEDIA_CERT_FILE}" | while read -r file event; do
-        kill -WINCH $(pgrep httpd)
-    done &
-fi
+configure_restart_on_certificate_update "${IRONIC_VMEDIA_TLS_SETUP}" httpd "${IRONIC_VMEDIA_CERT_FILE}"
 
-exec /usr/sbin/httpd -DFOREGROUND -f /etc/httpd/conf/httpd.conf
+exec /usr/sbin/httpd -DFOREGROUND -f "${HTTPD_CONF_DIR}/httpd.conf"

ironic-image/scripts/runironic

@@ -1,23 +1,18 @@
 #!/usr/bin/bash
 
-# This setting must go before configure-ironic since it has different defaults.
-export IRONIC_USE_MARIADB=${IRONIC_USE_MARIADB:-false}
-
 # shellcheck disable=SC1091
 . /bin/configure-ironic.sh
 
 # Ramdisk logs
 mkdir -p /shared/log/ironic/deploy
 
-run_ironic_dbsync
-
-if [[ "$IRONIC_TLS_SETUP" == "true" ]] && [[ "${RESTART_CONTAINER_CERTIFICATE_UPDATED}" == "true" ]]; then
-    # shellcheck disable=SC2034
-    python3.11 -m pyinotify --raw-format -e IN_DELETE_SELF -v "${IRONIC_CERT_FILE}" | while read -r file event; do
-        kill $(pgrep ironic)
-    done &
+# Allows skipping dbsync if it's done by an external job
+if [[ "${IRONIC_SKIP_DBSYNC:-false}" != true ]]; then
+    run_ironic_dbsync
 fi
 
+configure_restart_on_certificate_update "${IRONIC_TLS_SETUP}" ironic "${IRONIC_CERT_FILE}"
+
 configure_ironic_auth
 
-exec /usr/bin/ironic
+exec /usr/bin/ironic --config-dir "${IRONIC_CONF_DIR}"

ironic-image/scripts/runlogwatch.sh

@@ -3,17 +3,15 @@
 # Ramdisk logs path
 LOG_DIR="/shared/log/ironic/deploy"
 
-# The ironic container creates the directory, wait for
-# it to exist before running inotifywait or it can fail causing
-# a spurious restart
-while [ ! -d "${LOG_DIR}" ]; do
-  echo "Waiting for ${LOG_DIR}"
-  sleep 5
-done
+mkdir -p "${LOG_DIR}"
 
+# shellcheck disable=SC2034
 python3.11 -m pyinotify --raw-format -e IN_CLOSE_WRITE -v "${LOG_DIR}" |
-    while read -r path _action file; do
-        echo "************ Contents of ${path}/${file} ramdisk log file bundle **************"
-        tar -xOzvvf "${path}/${file}" | sed -e "s/^/${file}: /"
-        rm -f "${path}/${file}"
+    while read -r event dir mask maskname filename filepath pathname wd; do
+        #NOTE(elfosardo): a pyinotify event looks like this:
+        # <Event dir=False mask=0x8 maskname=IN_CLOSE_WRITE name=mylogs.gzip path=/shared/log/ironic/deploy pathname=/shared/log/ironic/deploy/mylogs.gzip wd=1 >
+        FILENAME=$(echo "${filename}" | cut -d'=' -f2-)
+        echo "************ Contents of ${LOG_DIR}/${FILENAME} ramdisk log file bundle **************"
+        tar -xOzvvf "${LOG_DIR}/${FILENAME}" | sed -e "s/^/${FILENAME}: /"
+        rm -f "${LOG_DIR}/${FILENAME}"
     done

ironic-image/scripts/runonline-data-migrations

@@ -0,0 +1,10 @@
+#!/usr/bin/bash
+
+set -euxo pipefail
+
+# shellcheck disable=SC1091
+. /bin/configure-ironic.sh
+
+# NOTE(dtantsur): no retries here: this script is supposed to be run as a Job
+# that is retried on failure.
+exec ironic-dbsync --config-file "${IRONIC_CONF_DIR}/ironic.conf" online_data_migrations

ironic-image/scripts/tls-common.sh

@@ -95,3 +95,21 @@ if [[ -f "$MARIADB_CACERT_FILE" ]]; then
 else
     export MARIADB_TLS_ENABLED="false"
 fi
+
+configure_restart_on_certificate_update()
+{
+    local enabled="$1"
+    local service="$2"
+    local cert_file="$3"
+    local signal="TERM"
+
+    if [[ "${enabled}" == "true" ]] && [[ "${RESTART_CONTAINER_CERTIFICATE_UPDATED}" == "true" ]]; then
+        if [[ "${service}" == httpd ]]; then
+            signal="WINCH"
+        fi
+        python3 -m pyinotify --raw-format -e IN_DELETE_SELF -v "${cert_file}" |
+            while read -r; do
+                pkill "-${signal}" "${service}"
+            done &
+    fi
+}