From 5beb6541c39335fd83aaec4f2e83d26c9f217c31 Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Tue, 11 Jul 2017 20:48:53 -0500 Subject: [PATCH 1/9] dist/spec: initial specfile including relevant sub-packages. Definitely some areas that should be improved, but those can be done in coordination with Makefile updates once a base spec file exists. --- dist/openSUSE-release-tools.spec | 281 +++++++++++++++++++++++++++++++ 1 file changed, 281 insertions(+) create mode 100644 dist/openSUSE-release-tools.spec diff --git a/dist/openSUSE-release-tools.spec b/dist/openSUSE-release-tools.spec new file mode 100644 index 00000000..153549a7 --- /dev/null +++ b/dist/openSUSE-release-tools.spec @@ -0,0 +1,281 @@ +# +# spec file for package openSUSE-release-tools +# +# Copyright (c) 2017 SUSE LINUX GmbH, Nuernberg, Germany. +# +# All modifications and additions to the file contributed by third parties +# remain the property of their copyright owners, unless otherwise agreed +# upon. The license for this file, and modifications and additions to the +# file, is the same license as for the pristine package itself (unless the +# license for the pristine package is not an Open Source License, in which +# case the license is the MIT License). An "Open Source License" is a +# license that conforms to the Open Source Definition (Version 1.9) +# published by the Open Source Initiative. + +# Please submit bugfixes or comments via http://bugs.opensuse.org/ +# + + +%global __provides_exclude ^perl.* +%define source_dir osc-plugin-factory +%define osc_plugin_dir %{_prefix}/lib/osc-plugins +%define announcer_filename factory-package-news +Name: openSUSE-release-tools +Version: 0 +Release: 0 +Summary: Tools to aid in staging and release work for openSUSE/SUSE +License: GPL-2.0+ and MIT +Group: Development/Tools/Other +Url: https://github.com/openSUSE/osc-plugin-factory +# _service:tar/filename does not seem to add version like docs indicate. +Source: %{name}.tar.xz +BuildArch: noarch +BuildRequires: osc +BuildRequires: python-PyYAML +BuildRequires: python-cmdln +BuildRequires: python-colorama +BuildRequires: python-lxml +BuildRequires: python-pycurl +BuildRequires: python-python-dateutil +BuildRequires: python-pyxdg +BuildRequires: python-urlgrabber +%if 0%{?is_opensuse} +# Testing only requirements installed for `make check`. +BuildRequires: libxml2-tools +BuildRequires: python-httpretty +BuildRequires: python-mock +BuildRequires: python-nose +%endif + +# Spec related requirements. +%if 0%{?is_opensuse} +BuildRequires: apache-rpm-macros +%else +%define apache_sysconfdir %{_sysconfdir}/apache2 +%endif +BuildRequires: apache2-devel +BuildRequires: rsyslog +BuildRequires: systemd-rpm-macros + +Requires: python-PyYAML +Requires: python-cmdln +Requires: python-colorama +Requires: python-lxml +Requires: python-pycurl +Requires: python-python-dateutil +Requires: python-pyxdg +Requires: python-urlgrabber + +# Spec related requirements. +Requires: osclib = %{version} + +%description +Tools to aid in staging and release work for openSUSE/SUSE + +The toolset consists of a variety of stand-alone scripts, review bots, osc +plugins, and automation aids. + +%package devel +Summary: Development requirements for openSUSE-release-tools +Group: Development/Tools/Other +BuildArch: noarch +Requires: libxml2-tools +Requires: python-httpretty +Requires: python-mock +Requires: python-nose + +%description devel +Development requirements for openSUSE-release-tools to be used in conjunction +with a git clone of the development repository available from %{url}. + +%package abichecker +Summary: ABI review bot +Group: Development/Tools/Other +BuildArch: noarch +Requires: osclib = %{version} + +%description abichecker +ABI review bot for checking OBS requests. + +%package announcer +Summary: Release announcer +Group: Development/Tools/Other +BuildArch: noarch +# TODO Update requirements. + +%description announcer +OBS product release announcer for generating email diffs summaries. + +%package totest-manager +Summary: Manages \$product:ToTest repository +Group: Development/Tools/Other +BuildArch: noarch +# TODO Update requirements. +Requires: osclib = %{version} + +%description totest-manager +Manages \$product:ToTest repository workflow and openQA interaction + +%package -n osclib +Summary: Supplemental osc libraries +Group: Development/Tools/Other +BuildArch: noarch +# TODO Update requirements, but for now base deps. +Requires: %{name} = %{version} + +%description -n osclib +Supplemental osc libraries utilized by release tools. + +%package -n osc-plugin-check_dups +Summary: OSC plugin to check for duplicate requests +Group: Development/Tools/Other +BuildArch: noarch +Requires: osc + +%description -n osc-plugin-check_dups +OSC plugin to check for duplicate requests, see `osc check_dups --help`. + +%package -n osc-plugin-check_repo +Summary: OSC plugin for checking request build cycles and installability +Group: Development/Tools/Other +BuildArch: noarch +Requires: osclib = %{version} + +%description -n osc-plugin-check_repo +OSC plugin for checking request build cycles and installability, see `osc +check_repo --help`. + +%package -n osc-plugin-cycle +Summary: OSC plugin for cycle visualization +Group: Development/Tools/Other +BuildArch: noarch +Requires: osc + +%description -n osc-plugin-cycle +OSC plugin for cycle visualization, see `osc cycle --help`. + +%package -n osc-plugin-staging +Summary: OSC plugin for the staging workflow +Group: Development/Tools/Other +BuildArch: noarch +Requires: osclib = %{version} + +%description -n osc-plugin-staging +OSC plugin for the staging workflow, see `osc staging --help`. + +%prep +%setup -q -n "%{name}" + +%build +make %{?_smp_mflags} + +%check +%if 0%{?is_opensuse} +make check +%endif + +%install +%make_install + +# TODO Correct makefile to actually install source. +mkdir -p %{buildroot}%{_datadir}/%{source_dir}/%{announcer_filename} + +%pre announcer +%service_add_pre %{announcer_filename}.service + +%post announcer +%service_add_post %{announcer_filename}.service + +%preun announcer +%service_del_preun %{announcer_filename}.service + +%postun announcer +%service_del_postun %{announcer_filename}.service + +%pre totest-manager +%service_add_pre opensuse-totest-manager.service + +%post totest-manager +%service_add_post opensuse-totest-manager.service + +%preun totest-manager +%service_del_preun opensuse-totest-manager.service + +%postun totest-manager +%service_del_postun opensuse-totest-manager.service + +%pre -n osc-plugin-check_repo +%service_add_pre opensuse-repo-checker.target + +%post -n osc-plugin-check_repo +%service_add_post opensuse-repo-checker.target + +%preun -n osc-plugin-check_repo +%service_del_preun opensuse-repo-checker.target + +%postun -n osc-plugin-check_repo +%service_del_postun opensuse-repo-checker.target + +%files +%defattr(-,root,root,-) +%doc README.asciidoc +%{_datadir}/%{source_dir} +%exclude %{_datadir}/%{source_dir}/abichecker +%exclude %{_datadir}/%{source_dir}/%{announcer_filename} +%exclude %{_datadir}/%{source_dir}/totest-manager.py +%exclude %{_datadir}/%{source_dir}/osclib +%exclude %{_datadir}/%{source_dir}/osc-check_dups.py +%exclude %{_datadir}/%{source_dir}/osc-check_repo.py +%exclude %{_datadir}/%{source_dir}/osc-cycle.py +%exclude %{_datadir}/%{source_dir}/osc-staging.py +# Should be in osc package, but ironically it is using its deprecated directory. +%dir %{osc_plugin_dir} + +%files devel +%defattr(-,root,root,-) +# Non-empty for older products. +%doc README.asciidoc + +%files abichecker +%{apache_sysconfdir}/vhosts.d/opensuse-abi-checker.conf.in +%{_datadir}/%{source_dir}/abichecker +%{_tmpfilesdir}/opensuse-abi-checker.conf +%{_unitdir}/opensuse-abi-checker.service + +%files announcer +%defattr(-,root,root,-) +%doc %{announcer_filename}/README.asciidoc +%{apache_sysconfdir}/conf.d/%{announcer_filename}.conf.in +%{_datadir}/%{source_dir}/%{announcer_filename} +%config(noreplace) %{_sysconfdir}/rsyslog.d/%{announcer_filename}.conf +%{_unitdir}/%{announcer_filename}.service +%{_unitdir}/%{announcer_filename}.timer + +%files totest-manager +%{_unitdir}/opensuse-totest-manager.service +%{_datadir}/%{source_dir}/totest-manager.py + +%files -n osclib +%{_datadir}/%{source_dir}/osclib +%{osc_plugin_dir}/osclib + +%files -n osc-plugin-check_dups +%{_datadir}/%{source_dir}/osc-check_dups.py +%{osc_plugin_dir}/osc-check_dups.py + +%files -n osc-plugin-check_repo +%{_datadir}/%{source_dir}/osc-check_repo.py +%{osc_plugin_dir}/osc-check_repo.py +%{_unitdir}/opensuse-repo-checker.target +%{_unitdir}/opensuse-repo-checker@.service +%{_unitdir}/opensuse-repo-checker@.timer + +%files -n osc-plugin-cycle +%{_datadir}/%{source_dir}/osc-cycle.py +%{osc_plugin_dir}/osc-cycle.py + +%files -n osc-plugin-staging +%{_datadir}/%{source_dir}/osc-staging.py +%{osc_plugin_dir}/osc-staging.py + +%changelog From 3d640e98fb34992fce3a8f32d04375243cfb3184 Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 12 Jul 2017 20:24:10 -0500 Subject: [PATCH 2/9] dist/spec: drop osc-plugin-check_repo now that new repo checker was merged. --- dist/openSUSE-release-tools.spec | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/dist/openSUSE-release-tools.spec b/dist/openSUSE-release-tools.spec index 153549a7..5e6a1673 100644 --- a/dist/openSUSE-release-tools.spec +++ b/dist/openSUSE-release-tools.spec @@ -135,16 +135,6 @@ Requires: osc %description -n osc-plugin-check_dups OSC plugin to check for duplicate requests, see `osc check_dups --help`. -%package -n osc-plugin-check_repo -Summary: OSC plugin for checking request build cycles and installability -Group: Development/Tools/Other -BuildArch: noarch -Requires: osclib = %{version} - -%description -n osc-plugin-check_repo -OSC plugin for checking request build cycles and installability, see `osc -check_repo --help`. - %package -n osc-plugin-cycle Summary: OSC plugin for cycle visualization Group: Development/Tools/Other @@ -204,18 +194,6 @@ mkdir -p %{buildroot}%{_datadir}/%{source_dir}/%{announcer_filename} %postun totest-manager %service_del_postun opensuse-totest-manager.service -%pre -n osc-plugin-check_repo -%service_add_pre opensuse-repo-checker.target - -%post -n osc-plugin-check_repo -%service_add_post opensuse-repo-checker.target - -%preun -n osc-plugin-check_repo -%service_del_preun opensuse-repo-checker.target - -%postun -n osc-plugin-check_repo -%service_del_postun opensuse-repo-checker.target - %files %defattr(-,root,root,-) %doc README.asciidoc @@ -225,7 +203,6 @@ mkdir -p %{buildroot}%{_datadir}/%{source_dir}/%{announcer_filename} %exclude %{_datadir}/%{source_dir}/totest-manager.py %exclude %{_datadir}/%{source_dir}/osclib %exclude %{_datadir}/%{source_dir}/osc-check_dups.py -%exclude %{_datadir}/%{source_dir}/osc-check_repo.py %exclude %{_datadir}/%{source_dir}/osc-cycle.py %exclude %{_datadir}/%{source_dir}/osc-staging.py # Should be in osc package, but ironically it is using its deprecated directory. @@ -263,13 +240,6 @@ mkdir -p %{buildroot}%{_datadir}/%{source_dir}/%{announcer_filename} %{_datadir}/%{source_dir}/osc-check_dups.py %{osc_plugin_dir}/osc-check_dups.py -%files -n osc-plugin-check_repo -%{_datadir}/%{source_dir}/osc-check_repo.py -%{osc_plugin_dir}/osc-check_repo.py -%{_unitdir}/opensuse-repo-checker.target -%{_unitdir}/opensuse-repo-checker@.service -%{_unitdir}/opensuse-repo-checker@.timer - %files -n osc-plugin-cycle %{_datadir}/%{source_dir}/osc-cycle.py %{osc_plugin_dir}/osc-cycle.py From 8603a73971cf16cc79e1d40419d93f3d77fa2878 Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 12 Jul 2017 20:25:16 -0500 Subject: [PATCH 3/9] dist/spec: move into package subdirectory in prepration for travis-ci. --- dist/{ => package}/openSUSE-release-tools.spec | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dist/{ => package}/openSUSE-release-tools.spec (100%) diff --git a/dist/openSUSE-release-tools.spec b/dist/package/openSUSE-release-tools.spec similarity index 100% rename from dist/openSUSE-release-tools.spec rename to dist/package/openSUSE-release-tools.spec From bd5296d68013c99d8148b2cd4a73c2593148ac88 Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 12 Jul 2017 20:25:38 -0500 Subject: [PATCH 4/9] Makefile: provide package and package-clean targets. --- Makefile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Makefile b/Makefile index c01b7dcf..e657c3c3 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ pkgdata_SCRIPTS=$(wildcard *.py *.pl *.sh) pkgdata_SCRIPTS+=bs_mirrorfull findfileconflicts pkgdata_DATA+=bs_copy osclib $(wildcard *.pm *.testcase) repocheckerhome = /var/lib/opensuse-repo-checker +package_name = openSUSE-release-tools all: @@ -25,4 +26,12 @@ test: # to see more add -v -d -s --nologcapture $(wildcard /usr/bin/nosetests-2.*) +package: + touch dist/package/$(package_name).changes + tar -cJf dist/package/$(package_name).tar.xz --exclude=.git* --exclude=dist/package/*.tar.xz --transform 's,^\.,$(package_name),' . + +package-clean: + rm -f dist/package/$(package_name).changes + rm -f dist/package/$(package_name).tar.xz + .PHONY: all install test check From 06d69e5d5b79e7f78f94016cc3e21c9365198ad8 Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 12 Jul 2017 20:27:14 -0500 Subject: [PATCH 5/9] travis & Makefile: drop old repo checker directory creation. --- .travis.yml | 2 -- Makefile | 2 -- 2 files changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 66fc9eab..d214e268 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,8 +6,6 @@ install: # needed to install osc from git in requirements.txt - pip install pycurl urlgrabber - pip install -r requirements.txt - - sudo mkdir -p /var/cache/repo-checker - - sudo chown $USER /var/cache/repo-checker script: - nosetests --with-coverage --cover-package=osclib --cover-inclusive after_success: diff --git a/Makefile b/Makefile index e657c3c3..1b0d2cbe 100644 --- a/Makefile +++ b/Makefile @@ -5,14 +5,12 @@ include Makefile.common pkgdata_SCRIPTS=$(wildcard *.py *.pl *.sh) pkgdata_SCRIPTS+=bs_mirrorfull findfileconflicts pkgdata_DATA+=bs_copy osclib $(wildcard *.pm *.testcase) -repocheckerhome = /var/lib/opensuse-repo-checker package_name = openSUSE-release-tools all: install: install -d -m 755 $(DESTDIR)$(pkgdatadir) $(DESTDIR)$(unitdir) $(DESTDIR)$(oscplugindir) - install -d -m 755 $(DESTDIR)$(repocheckerhome) for i in $(pkgdata_SCRIPTS); do install -m 755 $$i $(DESTDIR)$(pkgdatadir); done chmod 644 $(DESTDIR)$(pkgdatadir)/osc-*.py for i in $(pkgdata_DATA); do cp -a $$i $(DESTDIR)$(pkgdatadir); done From 04c8e091d4e9ccb136b6bc9f10f7ba051ed2abe7 Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 12 Jul 2017 20:28:41 -0500 Subject: [PATCH 6/9] travis: provide a separate job for build spec. --- .travis.yml | 36 ++++++++++++++++++++++++------------ dist/ci/Dockerfile | 28 ++++++++++++++++++++++++++++ dist/ci/spec.sh | 32 ++++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 12 deletions(-) create mode 100644 dist/ci/Dockerfile create mode 100755 dist/ci/spec.sh diff --git a/.travis.yml b/.travis.yml index d214e268..95712251 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,24 @@ -language: python -python: - - 2.7 -install: - - pip install python-coveralls - # needed to install osc from git in requirements.txt - - pip install pycurl urlgrabber - - pip install -r requirements.txt -script: - - nosetests --with-coverage --cover-package=osclib --cover-inclusive -after_success: - - coveralls +matrix: + include: + - env: TEST_SUITE=distribution + sudo: required + services: + - docker + language: bash + before_install: + - docker build -f dist/ci/Dockerfile -t spec . + script: + - docker run -it spec ./dist/ci/spec.sh + - env: TEST_SUITE=nosetests + sudo: false + language: python + python: 2.7 + install: + # needed to install osc from git in requirements.txt + - pip install pycurl urlgrabber + - pip install -r requirements.txt + - pip install python-coveralls + script: + - nosetests --with-coverage --cover-package=osclib --cover-inclusive + after_success: + - coveralls diff --git a/dist/ci/Dockerfile b/dist/ci/Dockerfile new file mode 100644 index 00000000..1040bec0 --- /dev/null +++ b/dist/ci/Dockerfile @@ -0,0 +1,28 @@ +# https://docs.docker.com/engine/reference/builder/ + +FROM boombatower/opensuse:tumbleweed +MAINTAINER Jimmy Berry + +RUN zypper ref && zypper -n in --no-recommends \ + obs-service-source_validator \ + rpm-build \ + apache-rpm-macros \ + apache2-devel \ + libxml2-tools \ + make \ + osc \ + python-PyYAML \ + python-cmdln \ + python-colorama \ + python-httpretty \ + python-lxml \ + python-mock \ + python-nose \ + python-python-dateutil \ + python-pyxdg \ + rsyslog + +RUN mkdir -p /usr/src/app +WORKDIR /usr/src/app + +COPY . /usr/src/app diff --git a/dist/ci/spec.sh b/dist/ci/spec.sh new file mode 100755 index 00000000..10a762c8 --- /dev/null +++ b/dist/ci/spec.sh @@ -0,0 +1,32 @@ +#!/bin/bash +# Adapted from: https://github.com/openSUSE/snapper/blob/master/.travis.tumbleweed.sh. + +set -e -x + +make +make package + +# Validate package. +(cd dist/package && /usr/lib/obs/service/source_validator) + +# Build package (--nocheck as test suite runs separately). +cp dist/package/* /usr/src/packages/SOURCES/ +rpmbuild --nocheck -bb -D "jobs `nproc`" dist/package/*.spec + +# Install to test scripts. +rpm -iv --force --nodeps /usr/src/packages/RPMS/*/*.rpm + +# Ensure the staging plugin starts. +cat << eom > ~/.oscrc +[general] +[https://api.opensuse.org] +user = example +pass = example +eom + +osc staging --version + +# Upgrade and uninstall to test scripts. +rpm -Uv --force --nodeps /usr/src/packages/RPMS/*/*.rpm +# get the plain package names and remove all packages at once +rpm -ev --nodeps `rpm -q --qf '%{NAME} ' -p /usr/src/packages/RPMS/**/*.rpm` From b7b3f928a6175e13a031f7039bcb5a1715514c96 Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Wed, 12 Jul 2017 20:29:43 -0500 Subject: [PATCH 7/9] travis: skip expensive spec build if no changes made to relevant files. --- .travis.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.travis.yml b/.travis.yml index 95712251..b5c148dc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,15 @@ matrix: - docker language: bash before_install: + - | + # Always run on custom branches in main project as diff does not work. + if [ "$TRAVIS_BRANCH" == "master" ] || [ "$TRAVIS_PULL_REQUEST" != "false" ] ; then + echo "Checking to see if skippable..." + if ! git diff --name-only "$TRAVIS_COMMIT_RANGE" | grep -qE '^(\.travis.yml|dist\/.*)$' ; then + echo "No distribution related changes, stopping build process." + exit + fi + fi - docker build -f dist/ci/Dockerfile -t spec . script: - docker run -it spec ./dist/ci/spec.sh From 6cca8f6ef30dbac61d19cd7fc45e85df21886821 Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Thu, 13 Jul 2017 17:07:00 -0500 Subject: [PATCH 8/9] travis: drop distribution related change check. Managed to get the spec building quick enough it does not matter and the check would need to be more complicated to not miss a variety of cases. --- .travis.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index b5c148dc..95712251 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,15 +6,6 @@ matrix: - docker language: bash before_install: - - | - # Always run on custom branches in main project as diff does not work. - if [ "$TRAVIS_BRANCH" == "master" ] || [ "$TRAVIS_PULL_REQUEST" != "false" ] ; then - echo "Checking to see if skippable..." - if ! git diff --name-only "$TRAVIS_COMMIT_RANGE" | grep -qE '^(\.travis.yml|dist\/.*)$' ; then - echo "No distribution related changes, stopping build process." - exit - fi - fi - docker build -f dist/ci/Dockerfile -t spec . script: - docker run -it spec ./dist/ci/spec.sh From 23fd8e19c98b697a6c51d9d03d1a5630cdcfd86b Mon Sep 17 00:00:00 2001 From: Jimmy Berry Date: Thu, 13 Jul 2017 17:07:53 -0500 Subject: [PATCH 9/9] dist/ci/Dockerfile: hard-code mirror until TW mirror issue resolved. --- dist/ci/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dist/ci/Dockerfile b/dist/ci/Dockerfile index 1040bec0..e70efaeb 100644 --- a/dist/ci/Dockerfile +++ b/dist/ci/Dockerfile @@ -3,6 +3,11 @@ FROM boombatower/opensuse:tumbleweed MAINTAINER Jimmy Berry +# hard-code mirror until TW mirror issue is resolved +RUN zypper ar http://mirror.datto.com/opensuse/tumbleweed/repo/oss/ oss2 && \ + zypper mr --disable oss && \ + zypper lr -U + RUN zypper ref && zypper -n in --no-recommends \ obs-service-source_validator \ rpm-build \