Merge branch 'master' into add-2.1-upgrade-pipeline-in-nightly

api/v2.0/legacy_swagger.yaml

@@ -3679,6 +3679,10 @@ responses:
     description: 'Success'
   Created:
     description: 'Created'
+    headers:
+      Location:
+        type: string
+        description: The URL of the created resource
   BadRequest:
     description: 'Bad Request'
   Unauthorized:

src/portal/src/i18n/lang/zh-cn-lang.json

@@ -674,8 +674,8 @@
         "DELETION_SUMMARY_REPO": "确认删除镜像仓库 {{repoName}}?",
         "DELETION_TITLE_ARTIFACT": "删除镜像 Artifact 确认",
         "DELETION_SUMMARY_ARTIFACT": "确认删除镜像 Artifact {{param}}? 如果您删除此 Artifact，则这个 digest 的所有 Tag 也将被删除。",
-        "DELETION_TITLE_TAG": "删除镜像 Tag 确认",
-        "DELETION_SUMMARY_TAG": "确认删除镜像 Tag {{param}}? 如果您删除此 Tag，则这个 Tag 引用的同一个 digest 的所有其他 Tag 也将被删除。",
+        "DELETION_TITLE_TAG": "删除 Tag 确认",
+        "DELETION_SUMMARY_TAG": "确认删除 Tag {{param}}?",
         "DELETION_TITLE_TAG_DENIED": "已签名的镜像不能被删除",
         "DELETION_SUMMARY_TAG_DENIED": "要删除此镜像 Tag 必须首先从 Notary 中删除。\n请执行如下 Notary 命令删除:\n",
         "TAGS_NO_DELETE": "在只读模式下删除是被禁止的",

src/portal/src/i18n/lang/zh-tw-lang.json

@@ -670,8 +670,8 @@
     "DELETION_SUMMARY_REPO": "確認刪除鏡像倉庫{{repoName}}?",
     "DELETION_TITLE_ARTIFACT": "刪除鏡像Artifact 確認",
     "DELETION_SUMMARY_ARTIFACT": "確認刪除鏡像Artifact {{param}}? 如果您刪除此Artifact,則這個digest 的所有Tag 也將被刪除。",
-    "DELETION_TITLE_TAG": "刪除鏡像Tag 確認",
-    "DELETION_SUMMARY_TAG": "確認刪除鏡像Tag {{param}}? 如果您刪除此Tag,則這個Tag 引用的同一個digest 的所有其他Tag 也將被刪除。",
+    "DELETION_TITLE_TAG": "刪除Tag 確認",
+    "DELETION_SUMMARY_TAG": "確認刪除Tag {{param}}?",
     "DELETION_TITLE_TAG_DENIED": "已簽名的鏡像不能被刪除",
     "DELETION_SUMMARY_TAG_DENIED": "要刪除此鏡像Tag 必須首先從Notary 中刪除。\n請執行如下Notary 命令刪除:\n",
     "TAGS_NO_DELETE": "在只讀模式下刪除是被禁止的",

src/portal/src/lib/components/create-edit-endpoint/create-edit-endpoint.component.ts

@@ -38,6 +38,7 @@ const FAKE_PASSWORD = "rjGcfuRu";
 const FAKE_JSON_KEY = "No Change";
 const METADATA_URL = CURRENT_BASE_HREF + "/replication/adapterinfos";
 const HELM_HUB = "helm-hub";
+const FIXED_PATTERN_TYPE: string = "EndpointPatternTypeFix";
 @Component({
   selector: "hbr-create-edit-endpoint",
   templateUrl: "./create-edit-endpoint.component.html",
@@ -271,6 +272,9 @@ export class CreateEditEndpointComponent
       if (this.endpointList.length === 1) {
         this.target.url = this.endpointList[0].value;
       }
+      if (this.adapterInfo[selectValue].endpoint_pattern.endpoint_type === FIXED_PATTERN_TYPE) {
+        this.urlDisabled = true;
+      }
     } else {
       this.endpointList = [];
     }

src/replication/adapter/awsecr/adapter.go

@@ -16,15 +16,10 @@ package awsecr
 
 import (
 	"errors"
-	"net/http"
 	"regexp"
 
-	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
-	"github.com/aws/aws-sdk-go/aws/credentials"
-	"github.com/aws/aws-sdk-go/aws/session"
 	awsecrapi "github.com/aws/aws-sdk-go/service/ecr"
-	commonhttp "github.com/goharbor/harbor/src/common/http"
 	"github.com/goharbor/harbor/src/lib/log"
 	adp "github.com/goharbor/harbor/src/replication/adapter"
 	"github.com/goharbor/harbor/src/replication/adapter/native"
@@ -52,11 +47,16 @@ func newAdapter(registry *model.Registry) (*adapter, error) {
 	if err != nil {
 		return nil, err
 	}
-	authorizer := NewAuth(region, registry.Credential.AccessKey, registry.Credential.AccessSecret, registry.Insecure)
+	svc, err := getAwsSvc(
+		region, registry.Credential.AccessKey, registry.Credential.AccessSecret, registry.Insecure, nil)
+	if err != nil {
+		return nil, err
+	}
+	authorizer := NewAuth(registry.Credential.AccessKey, svc)
 	return &adapter{
 		registry: registry,
 		Adapter:  native.NewAdapterWithAuthorizer(registry, authorizer),
-		region:   region,
+		cacheSvc: svc,
 	}, nil
 }
 
@@ -88,9 +88,8 @@ var (
 
 type adapter struct {
 	*native.Adapter
-	registry      *model.Registry
-	region        string
-	forceEndpoint *string
+	registry *model.Registry
+	cacheSvc *awsecrapi.ECR
 }
 
 func (*adapter) Info() (info *model.RegistryInfo, err error) {
@@ -205,11 +204,6 @@ func getAdapterInfo() *model.AdapterPattern {
 
 // HealthCheck checks health status of a registry
 func (a *adapter) HealthCheck() (model.HealthStatus, error) {
-	if a.registry.Credential == nil ||
-		len(a.registry.Credential.AccessKey) == 0 || len(a.registry.Credential.AccessSecret) == 0 {
-		log.Errorf("no credential to ping registry %s", a.registry.URL)
-		return model.Unhealthy, nil
-	}
 	if err := a.Ping(); err != nil {
 		log.Errorf("failed to ping registry %s: %v", a.registry.URL, err)
 		return model.Unhealthy, nil
@@ -242,33 +236,7 @@ func (a *adapter) PrepareForPush(resources []*model.Resource) error {
 }
 
 func (a *adapter) createRepository(repository string) error {
-	if a.registry.Credential == nil ||
-		len(a.registry.Credential.AccessKey) == 0 || len(a.registry.Credential.AccessSecret) == 0 {
-		return errors.New("no credential ")
-	}
-	cred := credentials.NewStaticCredentials(
-		a.registry.Credential.AccessKey,
-		a.registry.Credential.AccessSecret,
-		"")
-	if a.region == "" {
-		return errors.New("no region parsed")
-	}
-
-	config := &aws.Config{
-		Credentials: cred,
-		Region:      &a.region,
-		HTTPClient: &http.Client{
-			Transport: commonhttp.GetHTTPTransportByInsecure(a.registry.Insecure),
-		},
-	}
-	if a.forceEndpoint != nil {
-		config.Endpoint = a.forceEndpoint
-	}
-	sess := session.Must(session.NewSession(config))
-
-	svc := awsecrapi.New(sess)
-
-	_, err := svc.CreateRepository(&awsecrapi.CreateRepositoryInput{
+	_, err := a.cacheSvc.CreateRepository(&awsecrapi.CreateRepositoryInput{
 		RepositoryName: &repository,
 	})
 	if err != nil {
@@ -284,40 +252,7 @@ func (a *adapter) createRepository(repository string) error {
 
 // DeleteManifest ...
 func (a *adapter) DeleteManifest(repository, reference string) error {
-	// AWS doesn't implement standard OCI delete manifest API, so use it's sdk.
-	if a.registry.Credential == nil ||
-		len(a.registry.Credential.AccessKey) == 0 || len(a.registry.Credential.AccessSecret) == 0 {
-		return errors.New("no credential ")
-	}
-	cred := credentials.NewStaticCredentials(
-		a.registry.Credential.AccessKey,
-		a.registry.Credential.AccessSecret,
-		"")
-	if a.region == "" {
-		return errors.New("no region parsed")
-	}
-
-	var tr *http.Transport
-	if a.registry.Insecure {
-		tr = commonhttp.GetHTTPTransport(commonhttp.InsecureTransport)
-	} else {
-		tr = commonhttp.GetHTTPTransport(commonhttp.SecureTransport)
-	}
-	config := &aws.Config{
-		Credentials: cred,
-		Region:      &a.region,
-		HTTPClient: &http.Client{
-			Transport: tr,
-		},
-	}
-	if a.forceEndpoint != nil {
-		config.Endpoint = a.forceEndpoint
-	}
-	sess := session.Must(session.NewSession(config))
-
-	svc := awsecrapi.New(sess)
-
-	_, err := svc.BatchDeleteImage(&awsecrapi.BatchDeleteImageInput{
+	_, err := a.cacheSvc.BatchDeleteImage(&awsecrapi.BatchDeleteImageInput{
 		RepositoryName: &repository,
 		ImageIds:       []*awsecrapi.ImageIdentifier{{ImageTag: &reference}},
 	})

src/replication/adapter/awsecr/adapter_test.go

@@ -3,6 +3,8 @@ package awsecr
 import (
 	"errors"
 	"fmt"
+	awsecrapi "github.com/aws/aws-sdk-go/service/ecr"
+	"github.com/stretchr/testify/require"
 	"io"
 	"io/ioutil"
 	"net/http"
@@ -149,17 +151,23 @@ func getMockAdapter(t *testing.T, hasCred, health bool) (*adapter, *httptest.Ser
 		Type: model.RegistryTypeAwsEcr,
 		URL:  server.URL,
 	}
+
+	var svc *awsecrapi.ECR
 	if hasCred {
 		registry.Credential = &model.Credential{
 			AccessKey:    "xxx",
 			AccessSecret: "ppp",
 		}
+		svc, _ = getAwsSvc(
+			"test-region", registry.Credential.AccessKey, registry.Credential.AccessSecret, registry.Insecure, &server.URL)
+	} else {
+		svc, _ = getAwsSvc(
+			"test-region", "", "", registry.Insecure, &server.URL)
 	}
 	return &adapter{
-		registry:      registry,
-		Adapter:       native.NewAdapter(registry),
-		region:        "test-region",
-		forceEndpoint: &server.URL,
+		registry: registry,
+		Adapter:  native.NewAdapter(registry),
+		cacheSvc: svc,
 	}, server
 }
 
@@ -180,7 +188,7 @@ func TestAdapter_HealthCheck(t *testing.T) {
 	status, err := a.HealthCheck()
 	assert.Nil(t, err)
 	assert.NotNil(t, status)
-	assert.EqualValues(t, model.Unhealthy, status)
+	assert.EqualValues(t, model.Healthy, status)
 
 	a, s = getMockAdapter(t, true, false)
 	defer s.Close()
@@ -260,16 +268,18 @@ func TestAwsAuthCredential_Modify(t *testing.T) {
 		},
 	)
 	defer server.Close()
-	a, _ := NewAuth("test-region", "xxx", "ppp", true).(*awsAuthCredential)
-	a.forceEndpoint = &server.URL
+	svc, err := getAwsSvc(
+		"test-region", "xxx", "ppp", true, &server.URL)
+	require.Nil(t, err)
+	a, _ := NewAuth("xxx", svc).(*awsAuthCredential)
 	req := httptest.NewRequest(http.MethodGet, "https://1234.dkr.ecr.test-region.amazonaws.com/v2/", nil)
-	err := a.Modify(req)
-	assert.Nil(t, err)
 	err = a.Modify(req)
-	assert.Nil(t, err)
+	require.Nil(t, err)
+	err = a.Modify(req)
+	require.Nil(t, err)
 	time.Sleep(time.Second)
 	err = a.Modify(req)
-	assert.Nil(t, err)
+	require.Nil(t, err)
 }
 
 var urlForBenchmark = []string{

src/replication/adapter/awsecr/auth.go

@@ -18,19 +18,19 @@ import (
 	"encoding/base64"
 	"errors"
 	"fmt"
-	"net/http"
-	"net/url"
-	"strings"
-	"time"
-
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
 	"github.com/aws/aws-sdk-go/aws/session"
 	awsecrapi "github.com/aws/aws-sdk-go/service/ecr"
 	commonhttp "github.com/goharbor/harbor/src/common/http"
 	"github.com/goharbor/harbor/src/common/http/modifier"
 	"github.com/goharbor/harbor/src/lib/log"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
 )
 
 // Credential ...
@@ -38,11 +38,8 @@ type Credential modifier.Modifier
 
 // Implements interface Credential
 type awsAuthCredential struct {
-	region        string
-	accessKey     string
-	accessSecret  string
-	insecure      bool
-	forceEndpoint *string
+	accessKey string
+	awssvc    *awsecrapi.ECR
 
 	cacheToken   *cacheToken
 	cacheExpired *time.Time
@@ -91,36 +88,44 @@ func (a *awsAuthCredential) Modify(req *http.Request) error {
 	return nil
 }
 
-func (a *awsAuthCredential) getAuthorization() (string, string, string, *time.Time, error) {
-	log.Infof("Aws Ecr getAuthorization %s", a.accessKey)
-	cred := credentials.NewStaticCredentials(
-		a.accessKey,
-		a.accessSecret,
-		"")
-
+func getAwsSvc(region, accessKey, accessSecret string, insecure bool, forceEndpoint *string) (*awsecrapi.ECR, error) {
+	sess, err := session.NewSession()
+	if err != nil {
+		return nil, err
+	}
+	var cred *credentials.Credentials
+	log.Debugf("Aws Ecr getAuthorization %s", accessKey)
+	if accessKey != "" {
+		cred = credentials.NewStaticCredentials(
+			accessKey,
+			accessSecret,
+			"")
+	} else {
+		cred = ec2rolecreds.NewCredentials(sess)
+	}
 	var tr *http.Transport
-	if a.insecure {
+	if insecure {
 		tr = commonhttp.GetHTTPTransport(commonhttp.InsecureTransport)
 	} else {
 		tr = commonhttp.GetHTTPTransport(commonhttp.SecureTransport)
 	}
 	config := &aws.Config{
 		Credentials: cred,
-		Region:      &a.region,
+		Region:      &region,
 		HTTPClient: &http.Client{
 			Transport: tr,
 		},
 	}
-	if a.forceEndpoint != nil {
-		config.Endpoint = a.forceEndpoint
-	}
-	sess, err := session.NewSession(config)
-	if err != nil {
-		return "", "", "", nil, err
+	if forceEndpoint != nil {
+		config.Endpoint = forceEndpoint
 	}
 
-	svc := awsecrapi.New(sess)
+	svc := awsecrapi.New(sess, config)
+	return svc, nil
+}
 
+func (a *awsAuthCredential) getAuthorization() (string, string, string, *time.Time, error) {
+	svc := a.awssvc
 	result, err := svc.GetAuthorizationToken(nil)
 	if err != nil {
 		if aerr, ok := err.(awserr.Error); ok {
@@ -161,11 +166,9 @@ func (a *awsAuthCredential) isTokenValid() bool {
 }
 
 // NewAuth new aws auth
-func NewAuth(region, accessKey, accessSecret string, insecure bool) Credential {
+func NewAuth(accessKey string, awssvc *awsecrapi.ECR) Credential {
 	return &awsAuthCredential{
-		region:       region,
-		accessKey:    accessKey,
-		accessSecret: accessSecret,
-		insecure:     insecure,
+		accessKey: accessKey,
+		awssvc:    awssvc,
 	}
 }

tests/resources/Harbor-Pages/Replication.robot

@@ -167,7 +167,7 @@ Delete Rule
 
 Select Rule
     [Arguments]  ${rule}
-    Retry Element Click  //clr-dg-row[contains(.,'${rule}')]//label
+    Retry Double Keywords When Error  Retry Element Click  //clr-dg-cell[contains(.,'${rule}')]  Retry Wait Element  ${replication_exec_id}
 
 Stop Jobs
     Retry Element Click  ${stop_jobs_button}

tests/resources/Helm-Util.robot

@@ -18,6 +18,9 @@ Library  OperatingSystem
 Library  Process
 
 *** Keywords ***
+Prepare Helm Cert
+    Wait Unitl Command Success  cp harbor_ca.crt /ca/server.crt
+
 Helm Repo Add
     [Arguments]  ${harbor_url}  ${user}  ${pwd}  ${project_name}=library  ${helm_repo_name}=myrepo
     ${rc}  ${output}=  Run And Return Rc And Output  helm repo remove ${project_name}
@@ -35,4 +38,4 @@ Helm Chart Push
     [Arguments]  ${ip}  ${user}  ${pwd}  ${chart_file}  ${archive}  ${project}  ${repo_name}  ${verion}
     ${rc}  ${output}=  Run And Return Rc And Output  ./tests/robot-cases/Group0-Util/helm_push_chart.sh ${ip} ${user} ${pwd} ${chart_file} ${archive} ${project} ${repo_name} ${verion}
     Log  ${output}
-    Should Be Equal As Integers  ${rc}  0
\ No newline at end of file
+    Should Be Equal As Integers  ${rc}  0

tests/resources/Nightly-Util.robot

@@ -40,6 +40,20 @@ CA Setup
     Log To Console  Prepare Docker Cert ...
     Prepare Docker Cert  ${ip}
 
+Nightly Test Setup For Nightly
+    [Arguments]  ${ip}  ${HARBOR_PASSWORD}  ${ip1}==${EMPTY}
+    Run Keyword If  '${ip1}' != '${EMPTY}'  CA setup For Nightly  ${ip1}  ${HARBOR_PASSWORD}  /ca/ca1.crt
+    Run Keyword If  '${ip1}' != '${EMPTY}'  Run  rm -rf ./harbor_ca.crt
+    Run Keyword  CA setup For Nightly  ${ip}  ${HARBOR_PASSWORD}
+    Run Keyword  Start Docker Daemon Locally
+
+CA Setup For Nightly
+    [Arguments]  ${ip}  ${HARBOR_PASSWORD}  ${cert}=/ca/ca.crt
+    Run  cp ${cert} harbor_ca.crt
+    Generate Certificate Authority For Chrome  ${HARBOR_PASSWORD}
+    Prepare Docker Cert  ${ip}
+    Prepare Helm Cert
+
 Collect Nightly Logs
     [Arguments]  ${ip}  ${SSH_PWD}  ${ip1}==${EMPTY}
     Run Keyword  Collect Logs  ${ip}  ${SSH_PWD}

tests/robot-cases/Group1-Nightly/Common.robot

@@ -619,7 +619,7 @@ Test Case - Push CNAB Bundle and Display
     Create An New Project And Go Into Project  test${d}
 
     ${target}=  Set Variable  ${ip}/test${d}/cnab${d}:cnab_tag${d}
-    CNAB Push Bundle  ${ip}  user010  Test1@34  ${target}  ./tests/robot-cases/Group0-Util/bundle.json
+    Retry Keyword N Times When Error  5  CNAB Push Bundle  ${ip}  user010  Test1@34  ${target}  ./tests/robot-cases/Group0-Util/bundle.json
 
     Go Into Project  test${d}
     Wait Until Page Contains  test${d}/cnab${d}

tests/robot-cases/Group1-Nightly/Setup_Nightly.robot

+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+*** Settings ***
+Documentation  Harbor BATs
+Resource  ../../resources/Util.robot
+Default Tags  Nightly
+
+*** Test Cases ***
+Test Suites Setup For UI Test
+    Nightly Test Setup For Nightly  ${ip}  ${HARBOR_PASSWORD}  ${ip1}
+    Setup API Test
+
+Test Case - Get Harbor Version
+#Just get harbor version and log it
+    Get Harbor Version
\ No newline at end of file

tests/robot-cases/Group3-Upgrade/prepare.py

@@ -544,12 +544,9 @@ class HarborAPI:
         image_b = "busybox"
         repo_name_a, tag_a = push_image_to_project(project, args.endpoint, 'admin', 'Harbor12345', image_a, "latest")
         repo_name_b, tag_b = push_image_to_project(project, args.endpoint, 'admin', 'Harbor12345', image_b, "latest")
-
-        #4. Push an index(IA) to Harbor by docker manifest CLI successfully;
         manifests = [args.endpoint+"/"+repo_name_a+":"+tag_a, args.endpoint+"/"+repo_name_b+":"+tag_b]
         index = args.endpoint+"/"+project+"/"+name+":"+tag
-        docker_manifest_push_to_harbor(index, manifests, args.endpoint, 'admin', 'Harbor12345', cfg_file = args.libpath + "/update_docker_cfg.sh")
-
+        docker_manifest_push_to_harbor(index, manifests, args.endpoint, 'admin', 'Harbor12345', cfg_file = args.libpath + "/update_docker_cfg.sh"
 
 def request(url, method, user = None, userp = None, **kwargs):
     if user is None: