Unverified Commit eb317fb8 authored by Ted Guan's avatar Ted Guan Committed by GitHub
Browse files

tag retention webhook support (#12749)


Signed-off-by: default avatarguanxiatao <guanxiatao@corp.netease.com>
parent af0f36a1
......@@ -28,6 +28,7 @@ func init() {
notifier.Subscribe(event.TopicScanningCompleted, &scan.Handler{})
notifier.Subscribe(event.TopicDeleteArtifact, &scan.DelArtHandler{})
notifier.Subscribe(event.TopicReplication, &artifact.ReplicationHandler{})
notifier.Subscribe(event.TopicTagRetention, &artifact.RetentionHandler{RetentionController: artifact.DefaultRetentionControllerFunc})
// replication
notifier.Subscribe(event.TopicPushArtifact, &replication.Handler{})
......
......@@ -8,6 +8,7 @@ import (
commonModels "github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/controller/event"
"github.com/goharbor/harbor/src/controller/event/handler/util"
ctlModel "github.com/goharbor/harbor/src/controller/event/model"
"github.com/goharbor/harbor/src/controller/project"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/jobservice/job"
......@@ -15,7 +16,6 @@ import (
"github.com/goharbor/harbor/src/lib/orm"
"github.com/goharbor/harbor/src/pkg/notification"
"github.com/goharbor/harbor/src/pkg/notifier/model"
notifyModel "github.com/goharbor/harbor/src/pkg/notifier/model"
"github.com/goharbor/harbor/src/replication"
rpModel "github.com/goharbor/harbor/src/replication/model"
)
......@@ -120,7 +120,7 @@ func constructReplicationPayload(event *event.ReplicationEvent) (*model.Payload,
}
hostname := strings.Split(extURL, ":")[0]
remoteRes := &model.ReplicationResource{
remoteRes := &ctlModel.ReplicationResource{
RegistryName: remoteRegistry.Name,
RegistryType: string(remoteRegistry.Type),
Endpoint: remoteRegistry.URL,
......@@ -131,18 +131,18 @@ func constructReplicationPayload(event *event.ReplicationEvent) (*model.Payload,
if err != nil {
log.Errorf("Error while reading external endpoint: %v", err)
}
localRes := &model.ReplicationResource{
localRes := &ctlModel.ReplicationResource{
RegistryType: string(rpModel.RegistryTypeHarbor),
Endpoint: ext,
Namespace: destNamespace,
}
payload := &notifyModel.Payload{
payload := &model.Payload{
Type: event.EventType,
OccurAt: event.OccurAt.Unix(),
Operator: string(execution.Trigger),
EventData: &model.EventData{
Replication: &model.Replication{
Replication: &ctlModel.Replication{
HarborHostname: hostname,
JobStatus: event.Status,
Description: rpPolicy.Description,
......@@ -174,20 +174,20 @@ func constructReplicationPayload(event *event.ReplicationEvent) (*model.Payload,
}
if event.Status == string(job.SuccessStatus) {
succeedArtifact := &model.ArtifactInfo{
succeedArtifact := &ctlModel.ArtifactInfo{
Type: task.ResourceType,
Status: task.Status,
NameAndTag: nameAndTag,
}
payload.EventData.Replication.SuccessfulArtifact = []*model.ArtifactInfo{succeedArtifact}
payload.EventData.Replication.SuccessfulArtifact = []*ctlModel.ArtifactInfo{succeedArtifact}
}
if event.Status == string(job.ErrorStatus) {
failedArtifact := &model.ArtifactInfo{
failedArtifact := &ctlModel.ArtifactInfo{
Type: task.ResourceType,
Status: task.Status,
NameAndTag: nameAndTag,
}
payload.EventData.Replication.FailedArtifact = []*model.ArtifactInfo{failedArtifact}
payload.EventData.Replication.FailedArtifact = []*ctlModel.ArtifactInfo{failedArtifact}
}
prj, err := project.Ctl.GetByName(orm.Context(), prjName, project.Metadata(true))
......
package artifact
import (
"fmt"
"strings"
"github.com/goharbor/harbor/src/controller/event"
"github.com/goharbor/harbor/src/controller/event/handler/util"
evtModel "github.com/goharbor/harbor/src/controller/event/model"
"github.com/goharbor/harbor/src/core/api"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/pkg/notification"
"github.com/goharbor/harbor/src/pkg/notifier/model"
"github.com/goharbor/harbor/src/pkg/retention"
)
// RetentionHandler preprocess tag retention event data
type RetentionHandler struct {
RetentionController func() retention.APIController
}
// DefaultRetentionControllerFunc ...
var DefaultRetentionControllerFunc = NewRetentionController
// NewRetentionController ...
func NewRetentionController() retention.APIController {
return api.GetRetentionController()
}
// Handle ...
func (r *RetentionHandler) Handle(value interface{}) error {
if !config.NotificationEnable() {
log.Debug("notification feature is not enabled")
return nil
}
trEvent, ok := value.(*event.RetentionEvent)
if !ok {
return errors.New("invalid tag retention event type")
}
if trEvent == nil {
return errors.New("nil tag retention event")
}
if len(trEvent.Deleted) == 0 {
log.Debugf("empty delete info of retention event")
return nil
}
payload, dryRun, project, err := r.constructRetentionPayload(trEvent)
if err != nil {
return err
}
// if dry run, do not trigger webhook
if dryRun {
log.Debugf("retention task %v is dry run", trEvent.TaskID)
return nil
}
policies, err := notification.PolicyMgr.GetRelatedPolices(project, trEvent.EventType)
if err != nil {
log.Errorf("failed to find policy for %s event: %v", trEvent.EventType, err)
return err
}
if len(policies) == 0 {
log.Debugf("cannot find policy for %s event: %v", trEvent.EventType, trEvent)
return nil
}
err = util.SendHookWithPolicies(policies, payload, trEvent.EventType)
if err != nil {
return err
}
return nil
}
// IsStateful ...
func (r *RetentionHandler) IsStateful() bool {
return false
}
func (r *RetentionHandler) constructRetentionPayload(event *event.RetentionEvent) (*model.Payload, bool, int64, error) {
task, err := r.RetentionController().GetRetentionExecTask(event.TaskID)
if err != nil {
log.Errorf("failed to get retention task %d: error: %v", event.TaskID, err)
return nil, false, 0, err
}
if task == nil {
return nil, false, 0, fmt.Errorf("task %d not found with retention event", event.TaskID)
}
execution, err := r.RetentionController().GetRetentionExec(task.ExecutionID)
if err != nil {
log.Errorf("failed to get retention execution %d: error: %v", task.ExecutionID, err)
return nil, false, 0, err
}
if execution == nil {
return nil, false, 0, fmt.Errorf("execution %d not found with retention event", task.ExecutionID)
}
if execution.DryRun {
return nil, true, 0, nil
}
md, err := r.RetentionController().GetRetention(execution.PolicyID)
if err != nil {
log.Errorf("failed to get tag retention policy %d: error: %v", execution.PolicyID, err)
return nil, false, 0, err
}
if md == nil {
return nil, false, 0, fmt.Errorf("policy %d not found with tag retention event", execution.PolicyID)
}
extURL, err := config.ExtURL()
if err != nil {
log.Errorf("Error while reading external endpoint URL: %v", err)
}
hostname := strings.Split(extURL, ":")[0]
payload := &model.Payload{
Type: event.EventType,
OccurAt: event.OccurAt.Unix(),
Operator: execution.Trigger,
EventData: &model.EventData{
Retention: &evtModel.Retention{
Total: task.Total,
Retained: task.Retained,
HarborHostname: hostname,
ProjectName: event.Deleted[0].Target.Namespace,
RetentionPolicyID: execution.PolicyID,
Status: event.Status,
RetentionRules: []*evtModel.RetentionRule{},
},
},
}
for _, v := range event.Deleted {
target := v.Target
deletedArtifact := &evtModel.ArtifactInfo{
Type: target.Kind,
Status: event.Status,
}
if len(target.Tags) != 0 {
deletedArtifact.NameAndTag = target.Repository + ":" + target.Tags[0]
}
payload.EventData.Retention.DeletedArtifact = []*evtModel.ArtifactInfo{deletedArtifact}
}
for _, v := range md.Rules {
retentionRule := &evtModel.RetentionRule{
Template: v.Template,
Parameters: v.Parameters,
TagSelectors: v.TagSelectors,
ScopeSelectors: v.ScopeSelectors,
}
payload.EventData.Retention.RetentionRules = append(payload.EventData.Retention.RetentionRules, retentionRule)
}
return payload, false, event.Deleted[0].Target.NamespaceID, nil
}
package artifact
import (
"testing"
"time"
"github.com/goharbor/harbor/src/pkg/retention"
"github.com/goharbor/harbor/src/controller/event"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/lib/selector"
"github.com/goharbor/harbor/src/pkg/notification"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRetentionHandler_Handle(t *testing.T) {
config.Init()
handler := &RetentionHandler{RetentionController: DefaultRetentionControllerFunc}
policyMgr := notification.PolicyMgr
retentionCtlFunc := handler.RetentionController
defer func() {
notification.PolicyMgr = policyMgr
handler.RetentionController = retentionCtlFunc
}()
notification.PolicyMgr = &fakedNotificationPolicyMgr{}
handler.RetentionController = retention.FakedRetentionControllerFunc
type args struct {
data interface{}
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "RetentionHandler Want Error 1",
args: args{
data: "",
},
wantErr: true,
},
{
name: "RetentionHandler 1",
args: args{
data: &event.RetentionEvent{
OccurAt: time.Now(),
Deleted: []*selector.Result{
{
Target: &selector.Candidate{
NamespaceID: 1,
Namespace: "project1",
Tags: []string{"v1"},
Labels: nil,
},
},
},
},
},
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := handler.Handle(tt.args.data)
if tt.wantErr {
require.NotNil(t, err, "Error: %s", err)
return
}
assert.Nil(t, err)
})
}
}
func TestRetentionHandler_IsStateful(t *testing.T) {
handler := &RetentionHandler{}
assert.False(t, handler.IsStateful())
}
package metadata
import (
"time"
event2 "github.com/goharbor/harbor/src/controller/event"
"github.com/goharbor/harbor/src/lib/selector"
"github.com/goharbor/harbor/src/pkg/notifier/event"
)
// RetentionMetaData defines tag retention related event data
type RetentionMetaData struct {
Total int
Retained int
Deleted []*selector.Result
Status string
TaskID int64
}
// Resolve tag retention metadata into tag retention event
func (r *RetentionMetaData) Resolve(evt *event.Event) error {
data := &event2.RetentionEvent{
EventType: event2.TopicTagRetention,
OccurAt: time.Now(),
Status: r.Status,
Deleted: r.Deleted,
TaskID: r.TaskID,
}
evt.Topic = event2.TopicTagRetention
evt.Data = data
return nil
}
package metadata
import (
"testing"
event2 "github.com/goharbor/harbor/src/controller/event"
"github.com/goharbor/harbor/src/pkg/notifier/event"
"github.com/stretchr/testify/suite"
)
type retentionEventTestSuite struct {
suite.Suite
}
func (r *retentionEventTestSuite) TestResolveOfDeleteRepositoryEventMetadata() {
e := &event.Event{}
metadata := &RetentionMetaData{
Total: 0,
Retained: 0,
Deleted: nil,
Status: "",
TaskID: 0,
}
err := metadata.Resolve(e)
r.Require().Nil(err)
r.Equal(event2.TopicTagRetention, e.Topic)
r.Require().NotNil(e.Data)
_, ok := e.Data.(*event2.RetentionEvent)
r.Require().True(ok)
}
func TestRetentionEventTestSuite(t *testing.T) {
suite.Run(t, &retentionEventTestSuite{})
}
package model
import "github.com/goharbor/harbor/src/pkg/retention/policy/rule"
// Replication describes replication infos
type Replication struct {
HarborHostname string `json:"harbor_hostname,omitempty"`
JobStatus string `json:"job_status,omitempty"`
Description string `json:"description,omitempty"`
ArtifactType string `json:"artifact_type,omitempty"`
AuthenticationType string `json:"authentication_type,omitempty"`
OverrideMode bool `json:"override_mode,omitempty"`
TriggerType string `json:"trigger_type,omitempty"`
PolicyCreator string `json:"policy_creator,omitempty"`
ExecutionTimestamp int64 `json:"execution_timestamp,omitempty"`
SrcResource *ReplicationResource `json:"src_resource,omitempty"`
DestResource *ReplicationResource `json:"dest_resource,omitempty"`
SuccessfulArtifact []*ArtifactInfo `json:"successful_artifact,omitempty"`
FailedArtifact []*ArtifactInfo `json:"failed_artifact,omitempty"`
}
// ArtifactInfo describe info of artifact
type ArtifactInfo struct {
Type string `json:"type"`
Status string `json:"status"`
NameAndTag string `json:"name_tag"`
FailReason string `json:"fail_reason,omitempty"`
}
// ReplicationResource describes replication resource info
type ReplicationResource struct {
RegistryName string `json:"registry_name,omitempty"`
RegistryType string `json:"registry_type"`
Endpoint string `json:"endpoint"`
Provider string `json:"provider,omitempty"`
Namespace string `json:"namespace,omitempty"`
}
// Retention describes tag retention infos
type Retention struct {
Total int `json:"total"`
Retained int `json:"retained"`
HarborHostname string `json:"harbor_hostname,omitempty"`
ProjectName string `json:"project_name,omitempty"`
RetentionPolicyID int64 `json:"retention_policy_id,omitempty"`
RetentionRules []*RetentionRule `json:"retention_rule,omitempty"`
Status string `json:"result,omitempty"`
DeletedArtifact []*ArtifactInfo `json:"deleted_artifact,omitempty"`
}
// RetentionRule describes tag retention rule
type RetentionRule struct {
// Template ID
Template string `json:"template,omitempty"`
// The parameters of this rule
Parameters map[string]rule.Parameter `json:"params,omitempty"`
// Selector attached to the rule for filtering tags
TagSelectors []*rule.Selector `json:"tag_selectors,omitempty" `
// Selector attached to the rule for filtering scope (e.g: repositories or namespaces)
ScopeSelectors map[string][]*rule.Selector `json:"scope_selectors,omitempty"`
}
......@@ -19,6 +19,7 @@ import (
"time"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/lib/selector"
"github.com/goharbor/harbor/src/pkg/artifact"
"github.com/goharbor/harbor/src/pkg/audit/model"
v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
......@@ -46,6 +47,7 @@ const (
TopicDeleteChart = "DELETE_CHART"
TopicReplication = "REPLICATION"
TopicArtifactLabeled = "ARTIFACT_LABELED"
TopicTagRetention = "TAG_RETENTION"
)
// CreateProjectEvent is the creating project event
......@@ -289,3 +291,12 @@ type ArtifactLabeledEvent struct {
OccurAt time.Time
Operator string
}
// RetentionEvent is tag retention related event data to publish
type RetentionEvent struct {
TaskID int64
EventType string
OccurAt time.Time
Status string
Deleted []*selector.Result
}
......@@ -384,6 +384,7 @@ func initSupportedEvents() map[string]struct{} {
event.TopicScanningFailed,
event.TopicScanningCompleted,
event.TopicReplication,
event.TopicTagRetention,
}
var supportedEventTypes = make(map[string]struct{})
......
......@@ -4,10 +4,11 @@ import (
"encoding/json"
"errors"
"fmt"
"github.com/goharbor/harbor/src/core/config"
"net/http"
"strconv"
"github.com/goharbor/harbor/src/core/config"
"github.com/goharbor/harbor/src/common/rbac"
"github.com/goharbor/harbor/src/core/promgr"
"github.com/goharbor/harbor/src/pkg/retention"
......
......@@ -18,15 +18,15 @@ import (
"encoding/json"
"time"
"github.com/goharbor/harbor/src/core/service/notifications"
"github.com/goharbor/harbor/src/common/job"
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/controller/event/metadata"
"github.com/goharbor/harbor/src/controller/scan"
"github.com/goharbor/harbor/src/core/service/notifications"
jjob "github.com/goharbor/harbor/src/jobservice/job"
"github.com/goharbor/harbor/src/lib/errors"
"github.com/goharbor/harbor/src/lib/log"
"github.com/goharbor/harbor/src/lib/selector"
"github.com/goharbor/harbor/src/pkg/notification"
"github.com/goharbor/harbor/src/pkg/notifier/event"
"github.com/goharbor/harbor/src/pkg/retention"
......@@ -188,8 +188,9 @@ func (h *Handler) HandleRetentionTask() {
// handle checkin
if h.checkIn != "" {
var retainObj struct {
Total int `json:"total"`
Retained int `json:"retained"`
Total int `json:"total"`
Retained int `json:"retained"`
Deleted []*selector.Result `json:"deleted"`
}
if err := json.Unmarshal([]byte(h.checkIn), &retainObj); err != nil {
log.Errorf("failed to resolve checkin of retention task %d: %v", taskID, err)
......@@ -205,6 +206,23 @@ func (h *Handler) HandleRetentionTask() {
h.SendInternalServerError(err)
return
}
e := &event.Event{}
metaData := &metadata.RetentionMetaData{
Total: retainObj.Total,
Retained: retainObj.Retained,
Deleted: retainObj.Deleted,
Status: "SUCCESS",
TaskID: taskID,
}
if err := e.Build(metaData); err == nil {
if err := e.Publish(); err != nil {
log.Error(errors.Wrap(err, "tag retention job hook handler: event publish"))
}
} else {
log.Error(errors.Wrap(err, "tag retention job hook handler: event publish"))
}
return
}
......
......@@ -2,6 +2,7 @@ package model
import (
"github.com/goharbor/harbor/src/common/models"
"github.com/goharbor/harbor/src/controller/event/model"
)
// HookEvent is hook related event data to publish
......@@ -22,10 +23,11 @@ type Payload struct {
// EventData of notification event payload
type EventData struct {
Resources []*Resource `json:"resources,omitempty"`
Repository *Repository `json:"repository,omitempty"`
Replication *Replication `json:"replication,omitempty"`
Custom map[string]string `json:"custom_attributes,omitempty"`
Resources []*Resource `json:"resources,omitempty"`
Repository *Repository `json:"repository,omitempty"`
Replication *model.Replication `json:"replication,omitempty"`
Retention *model.Retention `json:"retention,omitempty"`
Custom map[string]string `json:"custom_attributes,omitempty"`
}
// Resource describe infos of resource triggered notification
......@@ -44,37 +46,3 @@ type Repository struct {
RepoFullName string `json:"repo_full_name"`
RepoType string `json:"repo_type"`
}
// Replication describes replication infos
type Replication struct {
HarborHostname string `json:"harbor_hostname,omitempty"`
JobStatus string `json:"job_status,omitempty"`
Description string `json:"description,omitempty"`
ArtifactType string `json:"artifact_type,omitempty"`
AuthenticationType string `json:"authentication_type,omitempty"`
OverrideMode bool `json:"override_mode,omitempty"`
TriggerType string `json:"trigger_type,omitempty"`