Merge pull request #2657 from dmcgowan/add-travis-ci

Update build to use travis

.gometalinter.json

@@ -0,0 +1,11 @@
+{
+  "Vendor": true,
+  "Deadline": "2m",
+  "Sort": ["linter", "severity", "path", "line"],
+  "EnableGC": true,
+  "Enable": [
+    "gofmt",
+    "golint",
+    "vet"
+  ]
+}

.travis.yml

@@ -0,0 +1,52 @@
+dist: trusty
+sudo: required
+# setup travis so that we can run containers for integration tests
+services:
+  - docker
+
+language: go
+
+go:
+  - "1.9.x"
+  - "1.10.x"
+
+go_import_path: github.com/docker/distribution
+
+addons:
+  apt:
+    packages:
+      - python-minimal
+
+
+env:
+  - TRAVIS_GOOS=linux DOCKER_BUILDTAGS="include_oss include_gcs" TRAVIS_CGO_ENABLED=1
+
+before_install:
+  - uname -r
+  - sudo apt-get -q update
+
+install:
+  - go get -u github.com/vbatts/git-validation
+    # TODO: Add enforcement of license
+    # - go get -u github.com/kunalkushwaha/ltag
+  - cd $TRAVIS_BUILD_DIR
+
+script:
+  - export GOOS=$TRAVIS_GOOS
+  - export CGO_ENABLED=$TRAVIS_CGO_ENABLED
+  - DCO_VERBOSITY=-q script/validate/dco
+  - GOOS=linux script/setup/install-dev-tools
+  - script/validate/vendor
+  - go build -i .
+  - make check
+  - make build
+  - make binaries
+    # Currently takes too long
+    #- if [ "$GOOS" = "linux" ]; then make test-race ; fi
+  - if [ "$GOOS" = "linux" ]; then make coverage ; fi
+
+after_success:
+  - bash <(curl -s https://codecov.io/bash) -F linux
+
+before_deploy:
+    # Run tests with storage driver configurations

Makefile

@@ -1,9 +1,21 @@
-# Set an output prefix, which is the local directory if not specified
+# Root directory of the project (absolute path).
-PREFIX?=$(shell pwd)
+ROOTDIR=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))
-
 
 # Used to populate version variable in main package.
 VERSION=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always)
+REVISION=$(shell git rev-parse HEAD)$(shell if ! git diff --no-ext-diff --quiet --exit-code; then echo .m; fi)
+
+
+PKG=github.com/docker/distribution
+
+# Project packages.
+PACKAGES=$(shell go list -tags "${BUILDTAGS}" ./... | grep -v /vendor/)
+INTEGRATION_PACKAGE=${PKG}
+COVERAGE_PACKAGES=$(filter-out ${PKG}/registry/storage/driver/%,${PACKAGES})
+
+
+# Project binaries.
+COMMANDS=registry digest registry-api-descriptor-template
 
 # Allow turning off function inlining and variable registerization
 ifeq (${DISABLE_OPTIMIZATION},true)
@@ -11,89 +23,83 @@ ifeq (${DISABLE_OPTIMIZATION},true)
 	VERSION:="$(VERSION)-noopt"
 endif
 
-GO_LDFLAGS=-ldflags "-X `go list ./version`.Version=$(VERSION)"
+WHALE = "+"
 
-.PHONY: all build binaries clean dep-restore dep-save dep-validate fmt lint test test-full vet
+# Go files
+#
+TESTFLAGS_RACE=
+GOFILES=$(shell find . -type f -name '*.go')
+GO_TAGS=$(if $(BUILDTAGS),-tags "$(BUILDTAGS)",)
+GO_LDFLAGS=-ldflags '-s -w -X $(PKG)/version.Version=$(VERSION) -X $(PKG)/version.Revision=$(REVISION) -X $(PKG)/version.Package=$(PKG) $(EXTRA_LDFLAGS)'
+
+BINARIES=$(addprefix bin/,$(COMMANDS))
+
+# Flags passed to `go test`
+TESTFLAGS ?= -v $(TESTFLAGS_RACE)
+TESTFLAGS_PARALLEL ?= 8
+
+.PHONY: all build binaries check clean test test-race test-full integration coverage
 .DEFAULT: all
-all: fmt vet lint build test binaries
+
+all: binaries
 
 AUTHORS: .mailmap .git/HEAD
 	 git log --format='%aN <%aE>' | sort -fu > $@
 
 # This only needs to be generated by hand when cutting full releases.
 version/version.go:
+	@echo "$(WHALE) $@"
 	./version/version.sh > $@
 
-# Required for go 1.5 to build
+check: ## run all linters (TODO: enable "unused", "varcheck", "ineffassign", "unconvert", "staticheck", "goimports", "structcheck")
-GO15VENDOREXPERIMENT := 1
+	@echo "$(WHALE) $@"
+	gometalinter --config .gometalinter.json ./...
 
-# Go files
+test: ## run tests, except integration test with test.short
-GOFILES=$(shell find . -type f -name '*.go')
+	@echo "$(WHALE) $@"
+	@go test ${GO_TAGS} -test.short ${TESTFLAGS} $(filter-out ${INTEGRATION_PACKAGE},${PACKAGES})
 
-# Package list
+test-race: ## run tests, except integration test with test.short and race
-PKGS=$(shell go list -tags "${DOCKER_BUILDTAGS}" ./... | grep -v ^github.com/docker/distribution/vendor/)
+	@echo "$(WHALE) $@"
+	@go test ${GO_TAGS} -race -test.short ${TESTFLAGS} $(filter-out ${INTEGRATION_PACKAGE},${PACKAGES})
 
-# Resolving binary dependencies for specific targets
+test-full: ## run tests, except integration tests
-GOLINT=$(shell which golint || echo '')
+	@echo "$(WHALE) $@"
-VNDR=$(shell which vndr || echo '')
+	@go test ${GO_TAGS} ${TESTFLAGS} $(filter-out ${INTEGRATION_PACKAGE},${PACKAGES})
 
-${PREFIX}/bin/registry: $(GOFILES)
+integration: ## run integration tests
-	@echo "+ $@"
+	@echo "$(WHALE) $@"
-	@go build -tags "${DOCKER_BUILDTAGS}" -o $@ ${GO_LDFLAGS}  ${GO_GCFLAGS} ./cmd/registry
+	@go test ${TESTFLAGS} -parallel ${TESTFLAGS_PARALLEL} ${INTEGRATION_PACKAGE}
 
-${PREFIX}/bin/digest:  $(GOFILES)
+coverage: ## generate coverprofiles from the unit tests
-	@echo "+ $@"
+	@echo "$(WHALE) $@"
-	@go build -tags "${DOCKER_BUILDTAGS}" -o $@ ${GO_LDFLAGS}  ${GO_GCFLAGS} ./cmd/digest
+	@rm -f coverage.txt
+	@go test ${GO_TAGS} -i ${TESTFLAGS} $(filter-out ${INTEGRATION_PACKAGE},${COVERAGE_PACKAGES}) 2> /dev/null
+	@( for pkg in $(filter-out ${INTEGRATION_PACKAGE},${COVERAGE_PACKAGES}); do \
+		go test ${GO_TAGS} ${TESTFLAGS} \
+			-cover \
+			-coverprofile=profile.out \
+			-covermode=atomic $$pkg || exit; \
+		if [ -f profile.out ]; then \
+			cat profile.out >> coverage.txt; \
+			rm profile.out; \
+		fi; \
+	done )
 
-${PREFIX}/bin/registry-api-descriptor-template: $(GOFILES)
+FORCE:
-	@echo "+ $@"
-	@go build -o $@ ${GO_LDFLAGS} ${GO_GCFLAGS} ./cmd/registry-api-descriptor-template
 
-docs/spec/api.md: docs/spec/api.md.tmpl ${PREFIX}/bin/registry-api-descriptor-template
+# Build a binary from a cmd.
-	./bin/registry-api-descriptor-template $< > $@
+bin/%: cmd/% FORCE
+	@echo "$(WHALE) $@${BINARY_SUFFIX}"
+	@go build ${GO_GCFLAGS} ${GO_BUILD_FLAGS} -o $@${BINARY_SUFFIX} ${GO_LDFLAGS} ${GO_TAGS}  ./$<
 
-vet:
+binaries: $(BINARIES) ## build binaries
-	@echo "+ $@"
+	@echo "$(WHALE) $@"
-	@go vet -tags "${DOCKER_BUILDTAGS}" $(PKGS)
-
-fmt:
-	@echo "+ $@"
-	@test -z "$$(gofmt -s -l . 2>&1 | grep -v ^vendor/ | tee /dev/stderr)" || \
-		(echo >&2 "+ please format Go code with 'gofmt -s'" && false)
-
-lint:
-	@echo "+ $@"
-	$(if $(GOLINT), , \
-		$(error Please install golint: `go get -u github.com/golang/lint/golint`))
-	@test -z "$$($(GOLINT) ./... 2>&1 | grep -v ^vendor/ | tee /dev/stderr)"
 
 build:
-	@echo "+ $@"
+	@echo "$(WHALE) $@"
-	@go build -tags "${DOCKER_BUILDTAGS}" -v ${GO_LDFLAGS} $(PKGS)
+	@go build ${GO_GCFLAGS} ${GO_BUILD_FLAGS} ${GO_LDFLAGS} ${GO_TAGS} $(PACKAGES)
 
-test:
+clean: ## clean up binaries
-	@echo "+ $@"
+	@echo "$(WHALE) $@"
-	@go test -test.short -tags "${DOCKER_BUILDTAGS}" $(PKGS)
+	@rm -f $(BINARIES)
-
-test-full:
-	@echo "+ $@"
-	@go test -tags "${DOCKER_BUILDTAGS}" $(PKGS)
-
-binaries: ${PREFIX}/bin/registry ${PREFIX}/bin/digest ${PREFIX}/bin/registry-api-descriptor-template
-	@echo "+ $@"
-
-clean:
-	@echo "+ $@"
-	@rm -rf "${PREFIX}/bin/registry" "${PREFIX}/bin/digest" "${PREFIX}/bin/registry-api-descriptor-template"
-
-dep-validate:
-	@echo "+ $@"
-	$(if $(VNDR), , \
-		$(error Please install vndr: go get github.com/lk4d4/vndr))
-	@rm -Rf .vendor.bak
-	@mv vendor .vendor.bak
-	@$(VNDR)
-	@test -z "$$(diff -r vendor .vendor.bak 2>&1 | tee /dev/stderr)" || \
-		(echo >&2 "+ inconsistent dependencies! what you have in vendor.conf does not match with what you have in vendor" && false)
-	@rm -Rf vendor
-	@mv .vendor.bak vendor

circle.yml

@@ -1,94 +0,0 @@
-# Pony-up!
-machine:
-  pre:
-  # Install gvm
-    - bash < <(curl -s -S -L https://raw.githubusercontent.com/moovweb/gvm/1.0.22/binscripts/gvm-installer)
-  # Install codecov for coverage
-    - pip install --user codecov
-
-  post:
-  # go
-    - gvm install go1.10 --prefer-binary --name=stable
-
-  environment:
-  # Convenient shortcuts to "common" locations
-    CHECKOUT: /home/ubuntu/$CIRCLE_PROJECT_REPONAME
-    BASE_DIR: src/github.com/$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME
-  # Trick circle brainflat "no absolute path" behavior
-    BASE_STABLE: ../../../$HOME/.gvm/pkgsets/stable/global/$BASE_DIR
-    DOCKER_BUILDTAGS: "include_oss include_gcs"
-  # Workaround Circle parsing dumb bugs and/or YAML wonkyness
-    CIRCLE_PAIN: "mode: set"
-
-  hosts:
-  # Not used yet
-    fancy: 127.0.0.1
-
-dependencies:
-  pre:
-  # Copy the code to the gopath of all go versions
-    - >
-      gvm use stable &&
-      mkdir -p "$(dirname $BASE_STABLE)" &&
-      cp -R "$CHECKOUT" "$BASE_STABLE"
-
-  override:
-  # Install dependencies for every copied clone/go version
-    - gvm use stable && go get github.com/lk4d4/vndr:
-        pwd: $BASE_STABLE
-
-  post:
-  # For the stable go version, additionally install linting tools
-    - >
-      gvm use stable &&
-      go get github.com/axw/gocov/gocov github.com/golang/lint/golint
-
-test:
-  pre:
-  # Output the go versions we are going to test
-    # - gvm use old && go version
-    - gvm use stable && go version
-
-  # Ensure validation of dependencies
-    - git fetch origin:
-        pwd: $BASE_STABLE
-    - gvm use stable && if test -n "`git diff --stat=1000 origin/master | grep -E \"^[[:space:]]*vendor\"`"; then make dep-validate; fi:
-        pwd: $BASE_STABLE
-
-  # First thing: build everything. This will catch compile errors, and it's
-  # also necessary for go vet to work properly (see #807).
-    - gvm use stable && go install $(go list ./... | grep -v "/vendor/"):
-        pwd: $BASE_STABLE
-
-  # FMT
-    - gvm use stable && make fmt:
-        pwd: $BASE_STABLE
-
-   # VET
-    - gvm use stable && make vet:
-        pwd: $BASE_STABLE
-
-  # LINT
-    - gvm use stable && make lint:
-        pwd: $BASE_STABLE
-
-  override:
-  # Test stable, and report
-     - gvm use stable; export ROOT_PACKAGE=$(go list .); go list -tags "$DOCKER_BUILDTAGS" ./... | grep -v "/vendor/" | xargs -L 1 -I{} bash -c 'export PACKAGE={}; go test -tags "$DOCKER_BUILDTAGS" -test.short -coverprofile=$GOPATH/src/$PACKAGE/coverage.out -coverpkg=$(./coverpkg.sh $PACKAGE $ROOT_PACKAGE) $PACKAGE':
-         timeout: 1000
-         pwd: $BASE_STABLE
-
-  # Test stable with race
-     - gvm use stable; export ROOT_PACKAGE=$(go list .); go list -tags "$DOCKER_BUILDTAGS" ./... | grep -v "/vendor/" | grep -v "registry/handlers" | grep -v "registry/storage/driver" | xargs -L 1 -I{} bash -c 'export PACKAGE={}; go test -race -tags "$DOCKER_BUILDTAGS" -test.short $PACKAGE':
-         timeout: 1000
-         pwd: $BASE_STABLE
-  post:
-  # Report to codecov
-    - bash <(curl -s https://codecov.io/bash):
-        pwd: $BASE_STABLE
-
-  ## Notes
-  # Do we want these as well?
-  # - go get code.google.com/p/go.tools/cmd/goimports
-  # - test -z "$(goimports -l -w ./... | tee /dev/stderr)"
-  # http://labix.org/gocheck

coverpkg.sh

@@ -1,7 +0,0 @@
-#!/usr/bin/env bash
-# Given a subpackage and the containing package, figures out which packages
-# need to be passed to `go test -coverpkg`:  this includes all of the
-# subpackage's dependencies within the containing package, as well as the
-# subpackage itself.
-DEPENDENCIES="$(go list -f $'{{range $f := .Deps}}{{$f}}\n{{end}}' ${1} | grep ${2} | grep -v github.com/docker/distribution/vendor)"
-echo "${1} ${DEPENDENCIES}" | xargs echo -n | tr ' ' ','

script/setup/install-dev-tools

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+#
+# Install developer tools to $GOBIN (or $GOPATH/bin if unset)
+#
+set -eu -o pipefail
+
+go get -u github.com/alecthomas/gometalinter
+gometalinter --install >/dev/null
+go get -u github.com/LK4D4/vndr
+go get -u github.com/cpuguy83/go-md2man

script/validate/dco

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+if ! command -v git-validation; then
+    >&2 echo "ERROR: git-validation not found. Install with:"
+    >&2 echo "    go get -u github.com/vbatts/git-validation"
+    exit 1
+fi
+
+verbosity="${DCO_VERBOSITY--v}"
+GIT_CHECK_EXCLUDE="./vendor:./script/validate/template" git-validation "$verbosity" -run DCO,short-subject,dangling-whitespace

script/validate/vendor

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -eu -o pipefail
+
+rm -rf vendor/
+vndr |& grep -v -i clone
+
+DIFF_PATH="vendor/"
+DIFF=$(git status --porcelain -- "$DIFF_PATH")
+
+if [ "$DIFF" ]; then
+    echo
+    echo "These files were modified:"
+    echo
+    echo "$DIFF"
+    echo
+    exit 1
+else
+    echo "$DIFF_PATH is correct"
+fi

version/version.go

@@ -9,3 +9,7 @@ var Package = "github.com/docker/distribution"
 // build, it will be replaced by the actual version. The value here will be
 // used if the registry is run after a go get based install.
 var Version = "v2.6.0+unknown"
+
+// Revision is filled with the VCS (e.g. git) revision being used to build
+// the program at linking time.
+var Revision = ""

version/version.sh

@@ -19,4 +19,8 @@ var Package = "$(go list)"
 // build, it will be replaced by the actual version. The value here will be
 // used if the registry is run after a go get based install.
 var Version = "$(git describe --match 'v[0-9]*' --dirty='.m' --always)+unknown"
+
+// Revision is filled with the VCS (e.g. git) revision being used to build
+// the program at linking time.
+var Revision = ""
 EOF