Unverified Commit 179a04da authored by Steven Zou's avatar Steven Zou Committed by GitHub
Browse files

Merge pull request #9369 from steven-zou/feature/pluggable_scanner_s3_merge

Implement the pluggable scanner framework - stage 3
parents 21a0dd1d 6e8e601c
/*Table for keeping the plug scanner registration*/
CREATE TABLE scanner_registration
(
id SERIAL PRIMARY KEY NOT NULL,
uuid VARCHAR(64) UNIQUE NOT NULL,
url VARCHAR(256) UNIQUE NOT NULL,
name VARCHAR(128) UNIQUE NOT NULL,
description VARCHAR(1024) NULL,
auth VARCHAR(16) NOT NULL,
access_cred VARCHAR(512) NULL,
disabled BOOLEAN NOT NULL DEFAULT FALSE,
is_default BOOLEAN NOT NULL DEFAULT FALSE,
skip_cert_verify BOOLEAN NOT NULL DEFAULT FALSE,
create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
/*Table for keeping the scan report. The report details are stored as JSON*/
CREATE TABLE scan_report
(
id SERIAL PRIMARY KEY NOT NULL,
uuid VARCHAR(64) UNIQUE NOT NULL,
digest VARCHAR(256) NOT NULL,
registration_uuid VARCHAR(64) NOT NULL,
mime_type VARCHAR(256) NOT NULL,
job_id VARCHAR(32),
status VARCHAR(16) NOT NULL,
status_code INTEGER DEFAULT 0,
status_rev BIGINT DEFAULT 0,
report JSON,
start_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
end_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
UNIQUE(digest, registration_uuid, mime_type)
)
/*Table for keeping the plug scanner registration*/
CREATE TABLE scanner_registration
(
id SERIAL PRIMARY KEY NOT NULL,
uuid VARCHAR(64) UNIQUE NOT NULL,
url VARCHAR(256) UNIQUE NOT NULL,
name VARCHAR(128) UNIQUE NOT NULL,
description VARCHAR(1024) NULL,
auth VARCHAR(16) NOT NULL,
access_cred VARCHAR(512) NULL,
disabled BOOLEAN NOT NULL DEFAULT FALSE,
is_default BOOLEAN NOT NULL DEFAULT FALSE,
skip_cert_verify BOOLEAN NOT NULL DEFAULT FALSE,
create_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
update_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP
);
/*Table for keeping the scan report. The report details are stored as JSON*/
CREATE TABLE scan_report
(
id SERIAL PRIMARY KEY NOT NULL,
uuid VARCHAR(64) UNIQUE NOT NULL,
digest VARCHAR(256) NOT NULL,
registration_uuid VARCHAR(64) NOT NULL,
mime_type VARCHAR(256) NOT NULL,
job_id VARCHAR(64),
track_id VARCHAR(64),
status VARCHAR(1024) NOT NULL,
status_code INTEGER DEFAULT 0,
status_rev BIGINT DEFAULT 0,
report JSON,
start_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
end_time TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
UNIQUE(digest, registration_uuid, mime_type)
);
/** Add table for immutable tag **/
CREATE TABLE immutable_tag_rule
(
......
......@@ -54,11 +54,11 @@ type RepositoryQuery struct {
// TagResp holds the information of one image tag
type TagResp struct {
TagDetail
Signature *model.Target `json:"signature"`
ScanOverview *ImgScanOverview `json:"scan_overview,omitempty"`
Labels []*Label `json:"labels"`
PushTime time.Time `json:"push_time"`
PullTime time.Time `json:"pull_time"`
Signature *model.Target `json:"signature"`
ScanOverview map[string]interface{} `json:"scan_overview,omitempty"`
Labels []*Label `json:"labels"`
PushTime time.Time `json:"push_time"`
PullTime time.Time `json:"pull_time"`
}
// TagDetail ...
......
......@@ -54,9 +54,10 @@ const (
ResourceRepositoryTag = Resource("repository-tag")
ResourceRepositoryTagLabel = Resource("repository-tag-label")
ResourceRepositoryTagManifest = Resource("repository-tag-manifest")
ResourceRepositoryTagScanJob = Resource("repository-tag-scan-job")
ResourceRepositoryTagVulnerability = Resource("repository-tag-vulnerability")
ResourceRepositoryTagScanJob = Resource("repository-tag-scan-job") // TODO: remove
ResourceRepositoryTagVulnerability = Resource("repository-tag-vulnerability") // TODO: remove
ResourceRobot = Resource("robot")
ResourceNotificationPolicy = Resource("notification-policy")
ResourceScan = Resource("scan")
ResourceSelf = Resource("") // subresource for self
)
......@@ -162,6 +162,9 @@ var (
{Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionDelete},
{Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionList},
{Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionRead},
{Resource: rbac.ResourceScan, Action: rbac.ActionCreate},
{Resource: rbac.ResourceScan, Action: rbac.ActionRead},
}
)
......
......@@ -119,6 +119,9 @@ var (
{Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionDelete},
{Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionList},
{Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionRead},
{Resource: rbac.ResourceScan, Action: rbac.ActionCreate},
{Resource: rbac.ResourceScan, Action: rbac.ActionRead},
},
"master": {
......@@ -201,6 +204,9 @@ var (
{Resource: rbac.ResourceRobot, Action: rbac.ActionList},
{Resource: rbac.ResourceNotificationPolicy, Action: rbac.ActionList},
{Resource: rbac.ResourceScan, Action: rbac.ActionCreate},
{Resource: rbac.ResourceScan, Action: rbac.ActionRead},
},
"developer": {
......@@ -251,6 +257,9 @@ var (
{Resource: rbac.ResourceRobot, Action: rbac.ActionRead},
{Resource: rbac.ResourceRobot, Action: rbac.ActionList},
{Resource: rbac.ResourceScan, Action: rbac.ActionCreate},
{Resource: rbac.ResourceScan, Action: rbac.ActionRead},
},
"guest": {
......
......@@ -211,8 +211,17 @@ func init() {
scannerAPI := &ScannerAPI{}
beego.Router("/api/scanners", scannerAPI, "post:Create;get:List")
beego.Router("/api/scanners/:uuid", scannerAPI, "get:Get;delete:Delete;put:Update;patch:SetAsDefault")
beego.Router("/api/scanners/:uuid/metadata", scannerAPI, "get:Metadata")
beego.Router("/api/scanners/ping", scannerAPI, "post:Ping")
// Add routes for project level scanner
beego.Router("/api/projects/:pid([0-9]+)/scanner", scannerAPI, "get:GetProjectScanner;put:SetProjectScanner")
proScannerAPI := &ProjectScannerAPI{}
beego.Router("/api/projects/:pid([0-9]+)/scanner", proScannerAPI, "get:GetProjectScanner;put:SetProjectScanner")
// Add routes for scan
scanAPI := &ScanAPI{}
beego.Router("/api/repositories/*/tags/:tag/scan", scanAPI, "post:Scan;get:Report")
beego.Router("/api/repositories/*/tags/:tag/scan/:uuid/log", scanAPI, "get:Log")
// syncRegistry
if err := SyncRegistry(config.GlobalProjectMgr); err != nil {
......
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/pkg/scan/api/scanner"
"github.com/pkg/errors"
)
// ProjectScannerAPI provides rest API for managing the project level scanner(s).
type ProjectScannerAPI struct {
// The base controller to provide common utilities
BaseController
// Scanner controller for operating scanner registrations.
c scanner.Controller
// ID of the project
pid int64
}
// Prepare sth. for the subsequent actions
func (sa *ProjectScannerAPI) Prepare() {
// Call super prepare method
sa.BaseController.Prepare()
// Check access permissions
if !sa.RequireAuthenticated() {
return
}
// Get ID of the project
pid, err := sa.GetInt64FromPath(":pid")
if err != nil {
sa.SendBadRequestError(errors.Wrap(err, "project scanner API"))
return
}
// Check if the project exists
exists, err := sa.ProjectMgr.Exists(pid)
if err != nil {
sa.SendInternalServerError(errors.Wrap(err, "project scanner API"))
return
}
if !exists {
sa.SendNotFoundError(errors.Errorf("project with id %d", sa.pid))
return
}
sa.pid = pid
sa.c = scanner.DefaultController
}
// GetProjectScanner gets the project level scanner
func (sa *ProjectScannerAPI) GetProjectScanner() {
// Check access permissions
if !sa.RequireProjectAccess(sa.pid, rbac.ActionRead, rbac.ResourceConfiguration) {
return
}
r, err := sa.c.GetRegistrationByProject(sa.pid)
if err != nil {
sa.SendInternalServerError(errors.Wrap(err, "scanner API: get project scanners"))
return
}
if r != nil {
sa.Data["json"] = r
} else {
sa.Data["json"] = make(map[string]interface{})
}
sa.ServeJSON()
}
// SetProjectScanner sets the project level scanner
func (sa *ProjectScannerAPI) SetProjectScanner() {
// Check access permissions
if !sa.RequireProjectAccess(sa.pid, rbac.ActionUpdate, rbac.ResourceConfiguration) {
return
}
body := make(map[string]string)
if err := sa.DecodeJSONReq(&body); err != nil {
sa.SendBadRequestError(errors.Wrap(err, "scanner API: set project scanners"))
return
}
uuid, ok := body["uuid"]
if !ok || len(uuid) == 0 {
sa.SendBadRequestError(errors.New("missing scanner uuid when setting project scanner"))
return
}
if err := sa.c.SetRegistrationByProject(sa.pid, uuid); err != nil {
sa.SendInternalServerError(errors.Wrap(err, "scanner API: set project scanners"))
return
}
}
// Copyright Project Harbor Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api
import (
"fmt"
"net/http"
"testing"
"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
sc "github.com/goharbor/harbor/src/pkg/scan/api/scanner"
"github.com/stretchr/testify/suite"
)
// ProScannerAPITestSuite is test suite for testing the project scanner API
type ProScannerAPITestSuite struct {
suite.Suite
originC sc.Controller
mockC *MockScannerAPIController
}
// TestProScannerAPI is the entry of ProScannerAPITestSuite
func TestProScannerAPI(t *testing.T) {
suite.Run(t, new(ProScannerAPITestSuite))
}
// SetupSuite prepares testing env
func (suite *ProScannerAPITestSuite) SetupTest() {
suite.originC = sc.DefaultController
m := &MockScannerAPIController{}
sc.DefaultController = m
suite.mockC = m
}
// TearDownTest clears test case env
func (suite *ProScannerAPITestSuite) TearDownTest() {
// Restore
sc.DefaultController = suite.originC
}
// TestScannerAPIProjectScanner tests the API of getting/setting project level scanner
func (suite *ProScannerAPITestSuite) TestScannerAPIProjectScanner() {
suite.mockC.On("SetRegistrationByProject", int64(1), "uuid").Return(nil)
// Set
body := make(map[string]interface{}, 1)
body["uuid"] = "uuid"
runCodeCheckingCases(suite.T(), &codeCheckingCase{
request: &testingRequest{
url: fmt.Sprintf("/api/projects/%d/scanner", 1),
method: http.MethodPut,
credential: projAdmin,
bodyJSON: body,
},
code: http.StatusOK,
})
r := &scanner.Registration{
ID: 1004,
UUID: "uuid",
Name: "TestScannerAPIProjectScanner",
Description: "JUST FOR TEST",
URL: "https://a.b.c",
}
suite.mockC.On("GetRegistrationByProject", int64(1)).Return(r, nil)
// Get
rr := &scanner.Registration{}
err := handleAndParse(&testingRequest{
url: fmt.Sprintf("/api/projects/%d/scanner", 1),
method: http.MethodGet,
credential: projAdmin,
}, rr)
require.NoError(suite.T(), err)
assert.Equal(suite.T(), r.Name, rr.Name)
assert.Equal(suite.T(), r.UUID, rr.UUID)
}
......@@ -25,6 +25,11 @@ import (
"strings"
"time"
"github.com/goharbor/harbor/src/jobservice/logger"
"github.com/goharbor/harbor/src/pkg/scan/api/scan"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/goharbor/harbor/src/common"
......@@ -40,7 +45,6 @@ import (
"github.com/goharbor/harbor/src/core/config"
notifierEvt "github.com/goharbor/harbor/src/core/notifier/event"
coreutils "github.com/goharbor/harbor/src/core/utils"
"github.com/goharbor/harbor/src/pkg/scan"
"github.com/goharbor/harbor/src/replication"
"github.com/goharbor/harbor/src/replication/event"
"github.com/goharbor/harbor/src/replication/model"
......@@ -397,6 +401,13 @@ func (ra *RepositoryAPI) GetTag() {
return
}
project, err := ra.ProjectMgr.Get(projectName)
if err != nil {
ra.ParseAndHandleError(fmt.Sprintf("failed to get the project %s",
projectName), err)
return
}
client, err := coreutils.NewRepositoryClientForUI(ra.SecurityCtx.GetUsername(), repository)
if err != nil {
ra.SendInternalServerError(fmt.Errorf("failed to initialize the client for %s: %v",
......@@ -414,7 +425,7 @@ func (ra *RepositoryAPI) GetTag() {
return
}
result := assembleTagsInParallel(client, repository, []string{tag},
result := assembleTagsInParallel(client, project.ProjectID, repository, []string{tag},
ra.SecurityCtx.GetUsername())
ra.Data["json"] = result[0]
ra.ServeJSON()
......@@ -523,14 +534,14 @@ func (ra *RepositoryAPI) GetTags() {
}
projectName, _ := utils.ParseRepository(repoName)
exist, err := ra.ProjectMgr.Exists(projectName)
project, err := ra.ProjectMgr.Get(projectName)
if err != nil {
ra.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %s",
ra.ParseAndHandleError(fmt.Sprintf("failed to get the project %s",
projectName), err)
return
}
if !exist {
if project == nil {
ra.SendNotFoundError(fmt.Errorf("project %s not found", projectName))
return
}
......@@ -587,8 +598,13 @@ func (ra *RepositoryAPI) GetTags() {
return
}
ra.Data["json"] = assembleTagsInParallel(client, repoName, tags,
ra.SecurityCtx.GetUsername())
ra.Data["json"] = assembleTagsInParallel(
client,
project.ProjectID,
repoName,
tags,
ra.SecurityCtx.GetUsername(),
)
ra.ServeJSON()
}
......@@ -607,7 +623,7 @@ func simpleTags(tags []string) []*models.TagResp {
// get config, signature and scan overview and assemble them into one
// struct for each tag in tags
func assembleTagsInParallel(client *registry.Repository, repository string,
func assembleTagsInParallel(client *registry.Repository, projectID int64, repository string,
tags []string, username string) []*models.TagResp {
var err error
signatures := map[string][]notarymodel.Target{}
......@@ -621,8 +637,15 @@ func assembleTagsInParallel(client *registry.Repository, repository string,
c := make(chan *models.TagResp)
for _, tag := range tags {
go assembleTag(c, client, repository, tag, config.WithClair(),
config.WithNotary(), signatures)
go assembleTag(
c,
client,
projectID,
repository,
tag,
config.WithNotary(),
signatures,
)
}
result := []*models.TagResp{}
var item *models.TagResp
......@@ -636,8 +659,8 @@ func assembleTagsInParallel(client *registry.Repository, repository string,
return result
}
func assembleTag(c chan *models.TagResp, client *registry.Repository,
repository, tag string, clairEnabled, notaryEnabled bool,
func assembleTag(c chan *models.TagResp, client *registry.Repository, projectID int64,
repository, tag string, notaryEnabled bool,
signatures map[string][]notarymodel.Target) {
item := &models.TagResp{}
// labels
......@@ -659,8 +682,9 @@ func assembleTag(c chan *models.TagResp, client *registry.Repository,
}
// scan overview
if clairEnabled {
item.ScanOverview = getScanOverview(item.Digest, item.Name)
so := getSummary(projectID, repository, item.Digest)
if len(so) > 0 {
item.ScanOverview = so
}
// signature, compare both digest and tag
......@@ -968,73 +992,6 @@ func (ra *RepositoryAPI) GetSignatures() {
ra.ServeJSON()
}
// ScanImage handles request POST /api/repository/$repository/tags/$tag/scan to trigger image scan manually.
func (ra *RepositoryAPI) ScanImage() {
if !config.WithClair() {
log.Warningf("Harbor is not deployed with Clair, scan is disabled.")
ra.SendInternalServerError(errors.New("harbor is not deployed with Clair, scan is disabled"))
return
}
repoName := ra.GetString(":splat")
tag := ra.GetString(":tag")
projectName, _ := utils.ParseRepository(repoName)
exist, err := ra.ProjectMgr.Exists(projectName)
if err != nil {
ra.ParseAndHandleError(fmt.Sprintf("failed to check the existence of project %s",
projectName), err)
return
}
if !exist {
ra.SendNotFoundError(fmt.Errorf("project %s not found", projectName))
return
}
if !ra.SecurityCtx.IsAuthenticated() {
ra.SendUnAuthorizedError(errors.New("Unauthorized"))
return
}
if !ra.RequireProjectAccess(projectName, rbac.ActionCreate, rbac.ResourceRepositoryTagScanJob) {
return
}
err = coreutils.TriggerImageScan(repoName, tag)
if err != nil {
log.Errorf("Error while calling job service to trigger image scan: %v", err)
ra.SendInternalServerError(errors.New("Failed to scan image, please check log for details"))
return
}
}
// VulnerabilityDetails fetch vulnerability info from clair, transform to Harbor's format and return to client.
func (ra *RepositoryAPI) VulnerabilityDetails() {
if !config.WithClair() {
log.Warningf("Harbor is not deployed with Clair, it's not impossible to get vulnerability details.")
ra.SendInternalServerError(errors.New("harbor is not deployed with Clair, it's not impossible to get vulnerability details"))
return
}
repository := ra.GetString(":splat")
tag := ra.GetString(":tag")
exist, digest, err := ra.checkExistence(repository, tag)
if err != nil {
ra.SendInternalServerError(fmt.Errorf("failed to check the existence of resource, error: %v", err))
return
}
if !exist {
ra.SendNotFoundError(fmt.Errorf("resource: %s:%s not found", repository, tag))
return
}
projectName, _ := utils.ParseRepository(repository)
if !ra.RequireProjectAccess(projectName, rbac.ActionList, rbac.ResourceRepositoryTagVulnerability) {
return
}
res, err := scan.VulnListByDigest(digest)
if err != nil {
log.Errorf("Failed to get vulnerability list for image: %s:%s", repository, tag)
}
ra.Data["json"] = res
ra.ServeJSON()
}
func getSignatures(username, repository string) (map[string][]notarymodel.Target, error) {
targets, err := notary.GetInternalTargets(config.InternalNotaryEndpoint(),
username, repository)
......@@ -1079,33 +1036,19 @@ func (ra *RepositoryAPI) checkExistence(repository, tag string) (bool, string, e
return true, digest, nil
}
// will return nil when it failed to get data. The parm "tag" is for logging only.
func getScanOverview(digest string, tag string) *models.ImgScanOverview {
if len(digest) == 0 {
log.Debug("digest is nil")
return nil
func getSummary(pid int64, repository string, digest string) map[string]interface{} {
// At present, only get harbor native report as default behavior.
artifact := &v1.Artifact{
NamespaceID: pid,
Repository: repository,
Digest: digest,
MimeType: v1.MimeTypeDockerArtifact,
}
data, err := dao.GetImgScanOverview(digest)
sum, err := scan.DefaultController.GetSummary(artifact, []string{v1.MimeTypeNativeReport})
if err != nil {
log.Errorf("Failed to get scan result for tag:%s, digest: %s, error: %v", tag, digest, err)
}
if data == nil {
return nil
logger.Errorf("Failed to get scan report summary with error: %s", err)
}
job, err := dao.GetScanJob(data.JobID)
if err != nil {
log.Errorf("Failed to get scan job for id:%d, error: %v", data.JobID, err)
return nil