Add api to delete blob and manifest (#12006)

* Add api to delete blob and manifest

Enable the capability of registry controller to delete blob and manifest
Signed-off-by: wang yan <wangyan@vmware.com>

make/photon/prepare/templates/registryctl/config.yml.jinja

@@ -12,3 +12,4 @@ protocol: "http"
 port: 8080
 {% endif %}
 log_level: "INFO"
+registry_config: "/etc/registry/config.yml"

src/go.mod

@@ -2,9 +2,10 @@ module github.com/goharbor/harbor/src
 
 go 1.13
 
-replace github.com/goharbor/harbor => ../
-
 require (
+	github.com/Azure/azure-sdk-for-go v37.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest v0.9.3 // indirect
+	github.com/Azure/go-autorest/autorest/to v0.3.0 // indirect
 	github.com/Masterminds/semver v1.4.2
 	github.com/Unknwon/goconfig v0.0.0-20160216183935-5f601ca6ef4d // indirect
 	github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect
@@ -19,6 +20,7 @@ require (
 	github.com/cenkalti/backoff v2.1.1+incompatible // indirect
 	github.com/cloudflare/cfssl v0.0.0-20190510060611-9c027c93ba9e // indirect
 	github.com/coreos/go-oidc v2.1.0+incompatible
+	github.com/denverdino/aliyungo v0.0.0-20191227032621-df38c6fa730c // indirect
 	github.com/dghubble/sling v1.1.0
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
 	github.com/docker/distribution v2.7.1+incompatible
@@ -50,6 +52,7 @@ require (
 	github.com/lib/pq v1.3.0
 	github.com/mattn/go-runewidth v0.0.4 // indirect
 	github.com/miekg/pkcs11 v0.0.0-20170220202408-7283ca79f35e // indirect
+	github.com/ncw/swift v1.0.49 // indirect
 	github.com/olekukonko/tablewriter v0.0.1
 	github.com/opencontainers/go-digest v1.0.0-rc1
 	github.com/opencontainers/image-spec v1.0.1
@@ -76,3 +79,8 @@ require (
 	k8s.io/client-go v0.17.3
 	k8s.io/helm v2.16.3+incompatible
 )
+
+replace (
+	github.com/Azure/go-autorest => github.com/Azure/go-autorest v13.3.3+incompatible
+	github.com/goharbor/harbor => ../
+)

src/lib/errors/const.go

@@ -11,6 +11,8 @@ const (
 	BadRequestCode = "BAD_REQUEST"
 	// ForbiddenCode ...
 	ForbiddenCode = "FORBIDDEN"
+	// MethodNotAllowedCode ...
+	MethodNotAllowedCode = "METHOD_NOT_ALLOWED"
 	// PreconditionCode ...
 	PreconditionCode = "PRECONDITION"
 	// GeneralCode ...
@@ -59,6 +61,11 @@ func ForbiddenError(err error) *Error {
 	return New("forbidden").WithCode(ForbiddenCode).WithCause(err)
 }
 
+// MethodNotAllowedError is error for the case of forbidden
+func MethodNotAllowedError(err error) *Error {
+	return New("method not allowed").WithCode(MethodNotAllowedCode).WithCause(err)
+}
+
 // PreconditionFailedError is error for the case of precondition failed
 func PreconditionFailedError(err error) *Error {
 	return New("precondition failed").WithCode(PreconditionCode).WithCause(err)

src/registryctl/api/base.go

@@ -16,24 +16,36 @@ package api
 
 import (
 	"encoding/json"
+	"github.com/goharbor/harbor/src/lib/errors"
+	server_error "github.com/goharbor/harbor/src/server/error"
 	"net/http"
 )
 
-func handleInternalServerError(w http.ResponseWriter) {
-	http.Error(w, http.StatusText(http.StatusInternalServerError),
-		http.StatusInternalServerError)
+// HandleInternalServerError ...
+func HandleInternalServerError(w http.ResponseWriter, err error) {
+	HandleError(w, errors.UnknownError(err))
 }
 
-func handleUnauthorized(w http.ResponseWriter) {
-	http.Error(w, http.StatusText(http.StatusUnauthorized),
-		http.StatusUnauthorized)
+// HandleNotMethodAllowed ...
+func HandleNotMethodAllowed(w http.ResponseWriter) {
+	HandleError(w, errors.MethodNotAllowedError(nil))
 }
 
-// response status code will be written automatically if there is an error
-func writeJSON(w http.ResponseWriter, v interface{}) error {
+// HandleBadRequest ...
+func HandleBadRequest(w http.ResponseWriter, err error) {
+	HandleError(w, errors.BadRequestError(err))
+}
+
+// HandleError ...
+func HandleError(w http.ResponseWriter, err error) {
+	server_error.SendError(w, err)
+}
+
+// WriteJSON response status code will be written automatically if there is an error
+func WriteJSON(w http.ResponseWriter, v interface{}) error {
 	b, err := json.Marshal(v)
 	if err != nil {
-		handleInternalServerError(w)
+		HandleInternalServerError(w, err)
 		return err
 	}
 

src/registryctl/api/base_test.go

@@ -15,17 +15,36 @@
 package api
 
 import (
+	"github.com/goharbor/harbor/src/lib/errors"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 )
 
-func TestHandleInternalServerError(t *testing.T) {
+func TestHandleError(t *testing.T) {
 	w := httptest.NewRecorder()
-	handleInternalServerError(w)
+	HandleInternalServerError(w, errors.New("internal"))
 
 	if w.Code != http.StatusInternalServerError {
 		t.Errorf("unexpected status code: %d != %d", w.Code, http.StatusInternalServerError)
 	}
 
+	w = httptest.NewRecorder()
+	HandleBadRequest(w, errors.New("BadRequest"))
+	if w.Code != http.StatusBadRequest {
+		t.Errorf("unexpected status code: %d != %d", w.Code, http.StatusBadRequest)
+	}
+
+	w = httptest.NewRecorder()
+	HandleNotMethodAllowed(w)
+	if w.Code != http.StatusMethodNotAllowed {
+		t.Errorf("unexpected status code: %d != %d", w.Code, http.StatusMethodNotAllowed)
+	}
+
+	w = httptest.NewRecorder()
+	HandleError(w, errors.New("handle error"))
+	if w.Code != http.StatusInternalServerError {
+		t.Errorf("unexpected status code: %d != %d", w.Code, http.StatusInternalServerError)
+	}
+
 }

src/registryctl/api/health.go

@@ -22,7 +22,7 @@ import (
 
 // Health ...
 func Health(w http.ResponseWriter, r *http.Request) {
-	if err := writeJSON(w, "healthy"); err != nil {
+	if err := WriteJSON(w, "healthy"); err != nil {
 		log.Errorf("Failed to write response: %v", err)
 		return
 	}

src/registryctl/api/registry/blob/blob.go

+package blob
+
+import (
+	"errors"
+	"github.com/docker/distribution/registry/storage"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/registryctl/api"
+	"github.com/gorilla/mux"
+	"net/http"
+)
+
+// NewHandler returns the handler to handler blob request
+func NewHandler(storageDriver storagedriver.StorageDriver) http.Handler {
+	return &handler{
+		storageDriver: storageDriver,
+	}
+}
+
+type handler struct {
+	storageDriver storagedriver.StorageDriver
+}
+
+// ServeHTTP ...
+func (h *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	switch req.Method {
+	case http.MethodDelete:
+		h.delete(w, req)
+	default:
+		api.HandleNotMethodAllowed(w)
+	}
+}
+
+// DeleteBlob ...
+func (h *handler) delete(w http.ResponseWriter, r *http.Request) {
+	ref := mux.Vars(r)["reference"]
+	if ref == "" {
+		api.HandleBadRequest(w, errors.New("no reference specified"))
+		return
+	}
+	// don't parse the reference here as RemoveBlob does.
+	cleaner := storage.NewVacuum(r.Context(), h.storageDriver)
+	if err := cleaner.RemoveBlob(ref); err != nil {
+		log.Infof("failed to remove blob: %s, with error:%v", ref, err)
+		api.HandleError(w, err)
+		return
+	}
+}

src/registryctl/api/registry/blob/blob_test.go

+package blob
+
+import (
+	"github.com/docker/distribution/registry/storage/driver/inmemory"
+	"github.com/docker/distribution/testutil"
+	"github.com/goharbor/harbor/src/registryctl/api/registry/test"
+	"github.com/gorilla/mux"
+	"github.com/stretchr/testify/assert"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestDeletionBlob(t *testing.T) {
+	inmemoryDriver := inmemory.New()
+
+	registry := test.CreateRegistry(t, inmemoryDriver)
+	repo := test.MakeRepository(t, registry, "blobdeletion")
+
+	// Create random layers
+	randomLayers1, err := testutil.CreateRandomLayers(3)
+	if err != nil {
+		t.Fatalf("failed to make layers: %v", err)
+	}
+
+	randomLayers2, err := testutil.CreateRandomLayers(3)
+	if err != nil {
+		t.Fatalf("failed to make layers: %v", err)
+	}
+
+	// Upload all layers
+	err = testutil.UploadBlobs(repo, randomLayers1)
+	if err != nil {
+		t.Fatalf("failed to upload layers: %v", err)
+	}
+
+	err = testutil.UploadBlobs(repo, randomLayers2)
+	if err != nil {
+		t.Fatalf("failed to upload layers: %v", err)
+	}
+
+	req, err := http.NewRequest(http.MethodDelete, "", nil)
+	varMap := make(map[string]string, 1)
+	varMap["reference"] = test.GetKeys(randomLayers1)[0].String()
+	req = mux.SetURLVars(req, varMap)
+
+	blobHandler := NewHandler(inmemoryDriver)
+	rec := httptest.NewRecorder()
+	blobHandler.ServeHTTP(rec, req)
+	assert.True(t, rec.Result().StatusCode == 200)
+
+	// layer1 is deleted and layer2 is still there
+	blobs := test.AllBlobs(t, registry)
+	for dgst := range randomLayers1 {
+		if _, ok := blobs[dgst]; !ok {
+			t.Logf("random layer 1 blob missing is correct as it has been deleted: %v", dgst)
+		}
+	}
+	for dgst := range randomLayers2 {
+		if _, ok := blobs[dgst]; !ok {
+			t.Fatalf("random layer 2 blob missing: %v", dgst)
+		}
+	}
+}

src/registryctl/api/registry.go → src/registryctl/api/registry/gc/gc.go

-// Copyright Project Harbor Authors
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//    http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package api
+package gc
 
 import (
 	"bytes"
+	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/registryctl/api"
 	"net/http"
+	"os/exec"
 	"time"
+)
 
-	"os/exec"
+// NewHandler returns the handler to handler blob request
+func NewHandler(registryConf string) http.Handler {
+	return &handler{
+		registryConf: registryConf,
+	}
+}
 
-	"github.com/goharbor/harbor/src/lib/log"
-)
+type handler struct {
+	registryConf string
+}
 
-const (
-	regConf = "/etc/registry/config.yml"
-)
+// ServeHTTP ...
+func (h *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	switch req.Method {
+	case http.MethodPost:
+		h.start(w, req)
+	default:
+		api.HandleNotMethodAllowed(w)
+	}
+}
 
-// GCResult ...
-type GCResult struct {
+// Result ...
+type Result struct {
 	Status    bool      `json:"status"`
 	Msg       string    `json:"msg"`
 	StartTime time.Time `json:"starttime"`
 	EndTime   time.Time `json:"endtime"`
 }
 
-// StartGC ...
-func StartGC(w http.ResponseWriter, r *http.Request) {
-	cmd := exec.Command("/bin/bash", "-c", "registry_DO_NOT_USE_GC garbage-collect --delete-untagged=false "+regConf)
+// start ...
+func (h *handler) start(w http.ResponseWriter, r *http.Request) {
+	cmd := exec.Command("/bin/bash", "-c", "registry_DO_NOT_USE_GC garbage-collect --delete-untagged=false "+h.registryConf)
 	var outBuf, errBuf bytes.Buffer
 	cmd.Stdout = &outBuf
 	cmd.Stderr = &errBuf
@@ -47,12 +49,12 @@ func StartGC(w http.ResponseWriter, r *http.Request) {
 	log.Debugf("Start to execute garbage collection...")
 	if err := cmd.Run(); err != nil {
 		log.Errorf("Fail to execute GC: %v, command err: %s", err, errBuf.String())
-		handleInternalServerError(w)
+		api.HandleInternalServerError(w, err)
 		return
 	}
 
-	gcr := GCResult{true, outBuf.String(), start, time.Now()}
-	if err := writeJSON(w, gcr); err != nil {
+	gcr := Result{true, outBuf.String(), start, time.Now()}
+	if err := api.WriteJSON(w, gcr); err != nil {
 		log.Errorf("failed to write response: %v", err)
 		return
 	}

src/registryctl/api/registry/manifest/manifest.go

+package manifest
+
+import (
+	"github.com/docker/distribution/registry/storage"
+	storagedriver "github.com/docker/distribution/registry/storage/driver"
+	"github.com/goharbor/harbor/src/lib/errors"
+	"github.com/goharbor/harbor/src/lib/log"
+	"github.com/goharbor/harbor/src/registryctl/api"
+	"github.com/gorilla/mux"
+	"github.com/opencontainers/go-digest"
+	"net/http"
+)
+
+// NewHandler returns the handler to handler manifest request
+func NewHandler(storageDriver storagedriver.StorageDriver) http.Handler {
+	return &handler{
+		storageDriver: storageDriver,
+	}
+}
+
+type handler struct {
+	storageDriver storagedriver.StorageDriver
+}
+
+// ServeHTTP ...
+func (h *handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	switch req.Method {
+	case http.MethodDelete:
+		h.delete(w, req)
+	default:
+		api.HandleNotMethodAllowed(w)
+	}
+}
+
+// delete deletes manifest ...
+func (h *handler) delete(w http.ResponseWriter, r *http.Request) {
+	ref := mux.Vars(r)["reference"]
+	if ref == "" {
+		api.HandleBadRequest(w, errors.New("no reference specified"))
+		return
+	}
+	dgst, err := digest.Parse(ref)
+	if err != nil {
+		api.HandleBadRequest(w, errors.Wrap(err, "not supported reference"))
+		return
+	}
+	repoName := mux.Vars(r)["name"]
+	if repoName == "" {
+		api.HandleBadRequest(w, errors.New("no repository name specified"))
+		return
+	}
+	// let the tags as empty here, as it non-blocking GC. The tags deletion will be handled via DELETE /v2/manifest
+	var tags []string
+	cleaner := storage.NewVacuum(r.Context(), h.storageDriver)
+	if err := cleaner.RemoveManifest(repoName, dgst, tags); err != nil {
+		log.Infof("failed to remove manifest: %s, with error:%v", ref, err)
+		api.HandleInternalServerError(w, err)
+		return
+	}
+}

src/registryctl/api/registry/manifest/manifest_test.go

+package manifest
+
+import (
+	"fmt"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/registry/storage/driver/inmemory"
+	"github.com/docker/distribution/testutil"
+	"github.com/goharbor/harbor/src/registryctl/api/registry/test"
+	"github.com/gorilla/mux"
+	"github.com/stretchr/testify/assert"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestDeleteManifest(t *testing.T) {
+	ctx := context.Background()
+	inmemoryDriver := inmemory.New()
+
+	registry := test.CreateRegistry(t, inmemoryDriver)
+	repo := test.MakeRepository(t, registry, "mftest")
+
+	// Create random layers
+	randomLayers, err := testutil.CreateRandomLayers(3)
+	if err != nil {
+		t.Fatalf("failed to make layers: %v", err)
+	}
+
+	// Upload all layers
+	err = testutil.UploadBlobs(repo, randomLayers)
+	if err != nil {
+		t.Fatalf("failed to upload layers: %v", err)
+	}
+
+	sharedKey := test.GetAnyKey(randomLayers)
+	manifest, err := testutil.MakeSchema2Manifest(repo, append(test.GetKeys(randomLayers), sharedKey))
+	if err != nil {
+		t.Fatalf("failed to make manifest: %v", err)
+	}
+
+	manifestService := test.MakeManifestService(t, repo)
+	_, err = manifestService.Put(ctx, manifest)
+	if err != nil {
+		t.Fatalf("manifest upload failed: %v", err)
+	}
+
+	manifestDigest, err := manifestService.Put(ctx, manifest)
+	if err != nil {
+		t.Fatalf("manifest upload failed: %v", err)
+	}
+
+	req, err := http.NewRequest(http.MethodDelete, "http://api/registry/{name}/manifests/{reference}/?tags=1,2,3", nil)
+	varMap := make(map[string]string, 1)
+	varMap["reference"] = manifestDigest.String()
+	varMap["name"] = fmt.Sprintf("%v", repo.Named())
+	req = mux.SetURLVars(req, varMap)
+
+	manifestHandler := NewHandler(inmemoryDriver)
+	rec := httptest.NewRecorder()
+	manifestHandler.ServeHTTP(rec, req)
+	assert.True(t, rec.Result().StatusCode == 200)
+
+	// check that all of the layers of manifest are deleted.
+	blobs := test.AllBlobs(t, registry)
+	for dgst := range randomLayers {
+		if _, ok := blobs[dgst]; !ok {
+			t.Fatalf("random layer blob missing: %v", dgst)
+		}
+	}
+}

src/registryctl/api/registry/test/util.go

+package test
+
+import (
+	"github.com/docker/distribution"
+	"github.com/docker/distribution/context"
+	"github.com/docker/distribution/reference"
+	"github.com/docker/distribution/registry/storage"
+	"github.com/docker/distribution/registry/storage/driver"
+	"github.com/docker/libtrust"
+	"github.com/opencontainers/go-digest"
+	"io"
+	"testing"
+)
+
+// CreateRegistry ...
+func CreateRegistry(t *testing.T, driver driver.StorageDriver, options ...storage.RegistryOption) distribution.Namespace {
+	ctx := context.Background()
+	k, err := libtrust.GenerateECP256PrivateKey()
+	if err != nil {
+		t.Fatal(err)
+	}
+	options = append([]storage.RegistryOption{storage.EnableDelete, storage.Schema1SigningKey(k), storage.EnableSchema1}, options...)
+	registry, err := storage.NewRegistry(ctx, driver, options...)
+	if err != nil {
+		t.Fatalf("Failed to construct namespace")
+	}
+	return registry
+}
+
+// MakeRepository ...
+func MakeRepository(t *testing.T, registry distribution.Namespace, name string) distribution.Repository {
+	ctx := context.Background()
+
+	// Initialize a dummy repository
+	named, err := reference.WithName(name)
+	if err != nil {
+		t.Fatalf("Failed to parse name %s:  %v", name, err)
+	}
+
+	repo, err := registry.Repository(ctx, named)
+	if err != nil {
+		t.Fatalf("Failed to construct repository: %v", err)
+	}
+	return repo
+}
+
+// AllBlobs ...
+func AllBlobs(t *testing.T, registry distribution.Namespace) map[digest.Digest]struct{} {
+	ctx := context.Background()
+	blobService := registry.Blobs()
+	allBlobsMap := make(map[digest.Digest]struct{})
+	err := blobService.Enumerate(ctx, func(dgst digest.Digest) error {
+		allBlobsMap[dgst] = struct{}{}
+		return nil
+	})
+	if err != nil {
+		t.Fatalf("Error getting all blobs: %v", err)
+	}
+	return allBlobsMap
+}
+
+// GetAnyKey ...
+func GetAnyKey(digests map[digest.Digest]io.ReadSeeker) (d digest.Digest) {
+	for d = range digests {
+		break
+	}
+	return
+}
+
+// GetAnyKeys ...
+func GetKeys(digests map[digest.Digest]io.ReadSeeker) (ds []digest.Digest) {
+	for d := range digests {
+		ds = append(ds, d)
+	}
+	return
+}
+
+// MakeManifestService ...
+func MakeManifestService(t *testing.T, repository distribution.Repository) distribution.ManifestService {
+	ctx := context.Background()
+
+	manifestService, err := repository.Manifests(ctx)
+	if err != nil {
+		t.Fatalf("Failed to construct manifest store: %v", err)
+	}
+	return manifestService
+}

src/registryctl/client/client.go

@@ -17,6 +17,7 @@ package client
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/goharbor/harbor/src/lib/errors"
 	"io/ioutil"
 	"net/http"
 	"strings"
@@ -25,7 +26,12 @@ import (
 	"github.com/goharbor/harbor/src/common/http/modifier/auth"
 	"github.com/goharbor/harbor/src/common/utils"
 	"github.com/goharbor/harbor/src/lib/log"
-	"github.com/goharbor/harbor/src/registryctl/api"
+	"github.com/goharbor/harbor/src/registryctl/api/registry/gc"