1
0
mirror of https://github.com/openSUSE/osc.git synced 2025-01-12 16:56:15 +01:00

- bump version (0.7)

- initial support for local builds (subcommand 'build')
This commit is contained in:
Dr. Peter Poeml 2006-07-14 17:39:46 +00:00
parent d6af7d2f85
commit 20a13e7b3b
8 changed files with 632 additions and 5 deletions

8
NEWS
View File

@ -1,5 +1,11 @@
since 0.6:
since 0.7:
...
0.7:
- initial support for local builds (subcommand 'build')
- better error handling
- new subcommands buildconfig, buildinfo, repos
- remove requirement on pyxml package
- editmeta: add examples for package/project templates
- add support for streaming the build log (thanks to Christoph Thiel)

6
README
View File

@ -113,6 +113,12 @@ Update package meta data with metadata taken from spec file
osc updatepacmetafromspec <dir>
There are other commands, which you may not need (they may be useful in scripts):
osc repos
osc buildconfig
osc buildinfo
HINT FOR W3M USERS

330
osc/build.py Normal file
View File

@ -0,0 +1,330 @@
#!/usr/bin/python
# Copyright (C) 2006 Peter Poeml. All rights reserved.
# This program is free software; it may be used, copied, modified
# and distributed under the terms of the GNU General Public Licence,
# either version 2, or (at your option) any later version.
import os
import sys
import ConfigParser
import cElementTree as ET
from tempfile import NamedTemporaryFile
from osc.fetch import *
DEFAULTS = { 'packagecachedir': '/var/tmp/osbuild-packagecache',
'su-wrapper': 'su -c',
'build-cmd': '/usr/bin/build',
'build-root': '/var/tmp/build-root',
# default list of download URLs, which will be tried in order
'urllist': [
# the normal repo server, redirecting to mirrors
'http://software.opensuse.org/download/%(project)s/%(repository)s/%(arch)s/%(filename)s',
# direct access to "full" tree
'http://api.opensuse.org/rpm/%(project)s/%(repository)s/_repository/%(buildarch)s/%(name)s',
],
}
text_config_incomplete = """
Your configuration is not complete.
Make sure that you have a [general] section in %%s:
(You can copy&paste it. Some commented defaults are shown.)
[general]
# Downloaded packages are cached here. Must be writable by you.
#packagecachedir: %(packagecachedir)s
# Wrapper to call build as root (sudo, su -, ...)
#su-wrapper: %(su-wrapper)s
# rootdir to setup the chroot environment
#build-root: %(build-root)s
Note:
Configuration can be overridden by envvars, e.g.
OSC_SU_WRAPPER overrides the setting of su-wrapper.
""" % DEFAULTS
change_personality = {
'i686': 'linux32',
'i586': 'linux32',
'ppc': 'powerpc32',
's390': 's390',
}
can_also_build = {
'x86_64': ['i686', 'i586'],
'i686': ['i586'],
'ppc64': ['ppc'],
's390x': ['s390'],
}
hostarch = os.uname()[4]
if hostarch == 'i686': # FIXME
hostarch = 'i586'
class Buildinfo:
"""represent the contents of a buildinfo file"""
def __init__(self, filename):
tree = ET.parse(filename)
root = tree.getroot()
if root.find('error') != None:
sys.stderr.write('buildinfo is borken... it says:\n')
error = root.find('error').text
sys.stderr.write(error + '\n')
sys.exit(1)
# are we building .rpm or .deb?
# need the right suffix for downloading
# if a package named debhelper is in the dependencies, it must be .deb
self.pacsuffix = 'rpm'
for node in root.findall('dep'):
if node.text == 'debhelper':
self.pacsuffix = 'deb'
break
self.buildarch = root.find('arch').text
self.deps = []
for node in root.findall('bdep'):
p = Pac(node.get('name'),
node.get('version'),
node.get('release'),
node.get('project'),
node.get('repository'),
node.get('arch'),
self.buildarch, # buildarch is used only for the URL to access the full tree...
self.pacsuffix)
self.deps.append(p)
self.pdeps = []
for node in root.findall('pdep'):
self.pdeps.append(node.text)
class Pac:
"""represent a package to be downloaded"""
def __init__(self, name, version, release, project, repository, arch, buildarch, pacsuffix):
self.name = name
self.version = version
self.release = release
self.arch = arch
self.project = project
self.repository = repository
self.buildarch = buildarch
self.pacsuffix = pacsuffix
# build a map to fill our the URL templates
self.mp = {}
self.mp['name'] = self.name
self.mp['version'] = self.version
self.mp['release'] = self.release
self.mp['arch'] = self.arch
self.mp['project'] = self.project
self.mp['repository'] = self.repository
self.mp['buildarch'] = self.buildarch
self.mp['pacsuffix'] = self.pacsuffix
self.filename = '%(name)s-%(version)s-%(release)s.%(arch)s.%(pacsuffix)s' % self.mp
self.mp['filename'] = self.filename
def makeurls(self, cachedir, urllist):
self.urllist = []
# build up local URL
# by using the urlgrabber with local urls, we basically build up a cache.
# the cache has no validation, since the package servers don't support etags,
# or if-modified-since, so the caching is simply name-based (on the assumption
# that the filename is suitable as identifier)
self.localdir = '%s/%s/%s/%s' % (cachedir, self.project, self.repository, self.arch)
self.fullfilename=os.path.join(self.localdir, self.filename)
self.url_local = 'file://%s/' % self.fullfilename
# first, add the local URL
self.urllist.append(self.url_local)
# remote URLs
for url in urllist:
self.urllist.append(url % self.mp)
def __str__(self):
return self.name
def get_build_conf():
auth_dict = { } # to hold multiple usernames and passwords
conffile = os.path.expanduser('~/.oscrc')
if not os.path.exists(conffile):
print >>sys.stderr, 'Error:'
print >>sys.stderr, 'You need to create ~/.oscrc.'
print >>sys.stderr, 'Running the osc command will do this for you.'
sys.exit(1)
config = ConfigParser.ConfigParser(DEFAULTS)
config.read(conffile)
if not config.has_section('general'):
# FIXME: it might be sufficient to just assume defaults?
print >>sys.stderr, text_config_incomplete % conffile
sys.exit(1)
for host in [ x for x in config.sections() if x != 'general' ]:
auth_dict[host] = dict(config.items(host))
config = dict(config.items('general'))
# make it possible to override configuration of the rc file
for var in ['OSC_PACKAGECACHEDIR', 'BUILD_ROOT']:
val = os.getenv(var)
if val:
if var.startswith('OSC_'): var = var[4:]
var = var.lower().replace('_', '-')
config[var] = val
return config, auth_dict
def get_built_files(pacdir, pactype):
if pactype == 'rpm':
b_built = os.popen('find %s -name *.rpm' \
% os.path.join(pacdir, 'RPMS')).read().strip()
s_built = os.popen('find %s -name *.rpm' \
% os.path.join(pacdir, 'SRPMS')).read().strip()
else:
b_built = os.popen('find %s -name *.deb' \
% os.path.join(pacdir, 'DEBS')).read().strip()
s_built = None
return s_built, b_built
def main(argv):
global config
config, auth = get_build_conf()
if len(argv) < 2:
print 'you have to choose a repo to build on'
print 'possible repositories are:'
(i, o) = os.popen4(['osc', 'repos'])
i.close()
for line in o.readlines():
if line.split()[1] == hostarch or line.split()[1] in can_also_build[hostarch]:
print line.strip()
sys.exit(1)
repo = argv[1]
arch = argv[2]
spec = argv[3]
buildargs = []
buildargs += argv[4:]
if not os.path.exists(spec):
print >>sys.stderr, 'Error: specfile \'%s\' does not exist.' % spec
sys.exit(1)
print 'Getting buildinfo from server'
bi_file = NamedTemporaryFile(suffix='.xml', prefix='buildinfo.', dir = '/tmp')
os.system('osc buildinfo %s %s > %s' % (repo, arch, bi_file.name))
bi = Buildinfo(bi_file.name)
print 'Updating cache of required packages'
fetcher = Fetcher(cachedir = config['packagecachedir'],
urllist = config['urllist'],
auth_dict = auth)
# now update the package cache
fetcher.run(bi)
if bi.pacsuffix == 'rpm':
"""don't know how to verify .deb packages. They are verified on install
anyway, I assume... verifying package now saves time though, since we don't
even try to set up the buildroot if it wouldn't work."""
print 'Verifying integrity of cached packages'
verify_pacs([ i.fullfilename for i in bi.deps ])
print 'Writing build configuration'
buildconf = [ '%s %s\n' % (i.name, i.fullfilename) for i in bi.deps ]
buildconf.append('preinstall: ' + ' '.join(bi.pdeps) + '\n')
rpmlist = NamedTemporaryFile(prefix='rpmlist.', dir = '/tmp')
rpmlist.writelines(buildconf)
rpmlist.flush()
os.fsync(rpmlist)
print 'Getting buildconfig from server'
bc_file = NamedTemporaryFile(prefix='buildconfig.', dir = '/tmp')
os.system('osc buildconfig %s %s > %s' % (repo, arch, bc_file.name))
print 'Running build'
buildargs = ' '.join(buildargs)
cmd = '%s --root=%s --rpmlist=%s --dist=%s %s %s' \
% (config['build-cmd'],
config['build-root'],
rpmlist.name,
bc_file.name,
spec,
buildargs)
if config['su-wrapper'].startswith('su '):
tmpl = '%s \'%s\''
else:
tmpl = '%s %s'
cmd = tmpl % (config['su-wrapper'], cmd)
if hostarch != bi.buildarch:
cmd = change_personality[bi.buildarch] + ' ' + cmd
print cmd
os.system(cmd)
pacdirlink = os.readlink(os.path.join(config['build-root'], '.build.packages'))
pacdir = os.path.join(config['build-root'] + pacdirlink)
if os.path.exists(pacdir):
(s_built, b_built) = get_built_files(pacdir, bi.pacsuffix)
print
#print 'built source packages:'
if s_built: print s_built
#print 'built binary packages:'
print b_built

View File

@ -607,6 +607,28 @@ usage: 1. osc repos # package = current dir
print platform
def build(args):
"""build: build a package _locally_
You need to call the command inside a package directory.
usage: osc build <platform> <arch> <specfile>
"""
if args is None or len(args) < 3:
print 'missing argument'
print build.func_doc
print 'Valid arguments are:'
print
repos(None)
print
sys.exit(1)
import osc.build
osc.build.main(sys.argv[1:])
def history(args):
"""history: Shows the build history of a package (NOT IMPLEMENTED YET)
@ -662,10 +684,12 @@ usage: osc help [SUBCOMMAND...]
print '\n'.join(lines)
# all commands and aliases are defined here
# a function with the respective name is assumed to exist
cmd_dict = {
add: ['add'],
addremove: ['addremove'],
build: ['build'],
buildconfig: ['buildconfig'],
buildinfo: ['buildinfo'],
commit: ['commit', 'ci', 'checkin'],

View File

@ -5,7 +5,7 @@
# and distributed under the terms of the GNU General Public Licence,
# either version 2, or (at your option) any later version.
__version__ = '0.6'
__version__ = '0.7'
import os
import sys
@ -700,7 +700,7 @@ def check_store_version(dir):
sys.exit(1)
if v != __version__:
if v in ['0.2', '0.3', '0.4', '0.5']:
if v in ['0.2', '0.3', '0.4', '0.5', '0.6']:
# version is fine, no migration needed
f = open(versionfile, 'w')
f.write(__version__ + '\n')

168
osc/fetch.py Normal file
View File

@ -0,0 +1,168 @@
#!/usr/bin/python
# Copyright (C) 2006 Peter Poeml. All rights reserved.
# This program is free software; it may be used, copied, modified
# and distributed under the terms of the GNU General Public Licence,
# either version 2, or (at your option) any later version.
import sys, os
import urllib2
from urlgrabber.grabber import URLGrabber, URLGrabError
from urlgrabber.mirror import MirrorGroup
try:
from meter import TextMeter
except:
TextMeter = None
def join_url(self, base_url, rel_url):
"""to override _join_url of MirrorGroup, because we want to
pass full URLs instead of base URL where relative_url is added later...
IOW, we make MirrorGroup ignore relative_url"""
return base_url
class Fetcher:
def __init__(self, cachedir = '/tmp', auth_dict = {}, urllist = []):
__version__ = '0.1'
__user_agent__ = 'osbuild/%s' % __version__
# set up progress bar callback
if sys.stdout.isatty() and TextMeter:
self.progress_obj = TextMeter(fo=sys.stdout)
else:
self.progress_obj = None
self.cachedir = cachedir
self.urllist = urllist
passmgr = urllib2.HTTPPasswordMgrWithDefaultRealm()
for host in auth_dict.keys():
passmgr.add_password(None, host, auth_dict[host]['user'], auth_dict[host]['pass'])
authhandler = urllib2.HTTPBasicAuthHandler(passmgr)
self.gr = URLGrabber(user_agent=__user_agent__,
keepalive=1,
opener = urllib2.build_opener(authhandler),
progress_obj=self.progress_obj,
failure_callback=(self.failureReport,(),{}),
)
def failureReport(self, errobj):
"""failure output for failovers from urlgrabber"""
#log(0, '%s: %s' % (errobj.url, str(errobj.exception)))
#log(0, 'Trying other mirror.')
print 'Trying upstream server for %s (%s), since it is not on %s.' \
% (self.curpac, self.curpac.project, errobj.url.split('/')[2])
raise errobj.exception
def fetch(self, pac):
# for use by the failure callback
self.curpac = pac
MirrorGroup._join_url = join_url
mg = MirrorGroup(self.gr, pac.urllist)
try:
# it returns the filename
ret = mg.urlgrab(pac.filename,
filename=pac.fullfilename,
text = '(%s) %s' %(pac.project, pac.filename))
except URLGrabError, e:
print
print >>sys.stderr, 'Error:', e.strerror
print >>sys.stderr, 'Failed to retrieve %s from the following locations (in order):' % pac.filename
print >>sys.stderr, '\n'.join(pac.urllist)
sys.exit(1)
def dirSetup(self, pac):
dir = os.path.join(self.cachedir, pac.localdir)
if not os.path.exists(dir):
os.makedirs(dir, mode=0755)
def run(self, buildinfo):
for i in buildinfo.deps:
i.makeurls(self.cachedir, self.urllist)
if os.path.exists(os.path.join(i.localdir, i.fullfilename)):
#print 'cached:', i.fullfilename
pass
else:
self.dirSetup(i)
try:
# if there isn't a progress bar, there is no output at all
if not self.progress_obj:
print '(%s) %s' % (i.project, i.filename)
self.fetch(i)
except KeyboardInterrupt:
print 'Cancelled by user (ctrl-c)'
print 'Exiting.'
if os.path.exists(i.fullfilename):
print 'Cleaning up incomplete file', i.fullfilename
os.unlink(i.fullfilename)
sys.exit(0)
def verify_pacs(pac_list):
"""Take a list of rpm filenames and run rpm -K on them.
In case of failure, exit.
Check all packages in one go, since this takes only 6 seconds on my Athlon 700
instead of 20 when calling 'rpm -K' for each of them.
"""
# we can use os.popen4 because we don't care about the return value.
# we check the output anyway, and rpm always writes to stdout.
(i, o) = os.popen4(['/bin/rpm', '-K'] + pac_list)
i.close()
for line in o.readlines():
if not 'OK' in line:
print
print >>sys.stderr, 'The following package could not be verified:'
print >>sys.stderr, line
sys.exit(1)
if 'NOT OK' in line:
print
print >>sys.stderr, 'The following package could not be verified:'
print >>sys.stderr, line
if 'MISSING KEYS' in line:
missing_key = line.split('#')[-1].split(')')[0]
print >>sys.stderr, """
- If the key is missing, install it first.
For example, do the following as root:
gpg --keyserver pgp.mit.edu --recv-keys %s
gpg --armor --export %s > keyfile
rpm --import keyfile
Then, just start the build again.
""" %(missing_key, missing_key)
else:
print >>sys.stderr, """
- If the signature is wrong, you may try deleting the package manually
and re-run this program, so it is fetched again.
"""
sys.exit(1)

93
osc/meter.py Normal file
View File

@ -0,0 +1,93 @@
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330,
# Boston, MA 02111-1307 USA
# this is basically a copy of python-urlgrabber's TextMeter class,
# with support added for dynamical sizing according to screen size.
# it uses getScreenWidth() scrapped from smart.
# Peter Poeml <poeml@suse.de>
from urlgrabber.progress import BaseMeter, format_time, format_number
import sys, os
def getScreenWidth():
import termios, struct, fcntl
s = struct.pack('HHHH', 0, 0, 0, 0)
try:
x = fcntl.ioctl(1, termios.TIOCGWINSZ, s)
except IOError:
return 80
return struct.unpack('HHHH', x)[1]
class TextMeter(BaseMeter):
def __init__(self, fo=sys.stderr):
BaseMeter.__init__(self)
self.fo = fo
try:
width = int(os.environ['COLUMNS'])
except (KeyError, ValueError):
width = getScreenWidth()
#self.unsized_templ = '\r%-60.60s %5sB %s '
self.unsized_templ = '\r%%-%s.%ss %%5sB %%s ' % (width *4/3, width*4/3)
#self.sized_templ = '\r%-45.45s %3i%% |%-15.15s| %5sB %8s '
self.sized_templ = '\r%%-%s.%ss %%3i%%%% |%%-%s.%ss| %%5sB %%8s ' %(width*4/10, width*4/10, width/3, width/3)
self.bar_length = width/3
def _do_update(self, amount_read, now=None):
etime = self.re.elapsed_time()
fetime = format_time(etime)
fread = format_number(amount_read)
#self.size = None
if self.text is not None:
text = self.text
else:
text = self.basename
if self.size is None:
out = self.unsized_templ % \
(text, fread, fetime)
else:
rtime = self.re.remaining_time()
frtime = format_time(rtime)
frac = self.re.fraction_read()
bar = '='*int(self.bar_length * frac)
out = self.sized_templ % \
(text, frac*100, bar, fread, frtime) + 'ETA '
self.fo.write(out)
self.fo.flush()
def _do_end(self, amount_read, now=None):
total_time = format_time(self.re.elapsed_time())
total_size = format_number(amount_read)
if self.text is not None:
text = self.text
else:
text = self.basename
if self.size is None:
out = self.unsized_templ % \
(text, total_size, total_time)
else:
bar = '=' * self.bar_length
out = self.sized_templ % \
(text, 100, bar, total_size, total_time) + ' '
self.fo.write(out + '\n')
self.fo.flush()

View File

@ -3,7 +3,7 @@
from distutils.core import setup
setup(name='osc',
version='0.6',
version='0.7',
description='opensuse commander',
author='Peter Poeml',
author_email='poeml@suse.de',