Recovery nightly setup

1. Recovery nightly setup;
2. Add retry for replication rule selection.
3. Add retry for CNAB test.
4. Rerun keyword of push CNAB
Signed-off-by: danfengliu <danfengl@vmware.com>

tests/resources/Harbor-Pages/Replication.robot

@@ -167,7 +167,7 @@ Delete Rule
 
 Select Rule
     [Arguments]  ${rule}
-    Retry Element Click  //clr-dg-row[contains(.,'${rule}')]//label
+    Retry Double Keywords When Error  Retry Element Click  //clr-dg-cell[contains(.,'${rule}')]  Retry Wait Element  ${replication_exec_id}
 
 Stop Jobs
     Retry Element Click  ${stop_jobs_button}

tests/resources/Helm-Util.robot

@@ -18,18 +18,21 @@ Library  OperatingSystem
 Library  Process
 
 *** Keywords ***
+Prepare Helm Cert
+    Wait Unitl Command Success  cp harbor_ca.crt /ca/server.crt
+
 Helm Repo Add
     [Arguments]  ${harbor_url}  ${user}  ${pwd}  ${project_name}=library  ${helm_repo_name}=myrepo
     ${rc}  ${output}=  Run And Return Rc And Output  helm repo remove ${project_name}
     Log To Console  ${output}
-    Wait Unitl Command Success  helm repo add --username=${user} --password=${pwd} ${helm_repo_name} ${harbor_url}/chartrepo/${project_name}
+    Wait Unitl Command Success  helm repo add --ca-file /ca/server.crt --username=${user} --password=${pwd} ${helm_repo_name} ${harbor_url}/chartrepo/${project_name}
 
 Helm Repo Push
     [Arguments]  ${user}  ${pwd}  ${chart_filename}  ${helm_repo_name}=myrepo  ${helm_cmd}=helm
     ${current_dir}=  Run  pwd
     Run  cd ${current_dir}
     Run  wget ${harbor_chart_file_url}
-    Wait Unitl Command Success  ${helm_cmd} push --username=${user} --password=${pwd} ${chart_filename} ${helm_repo_name}
+    Wait Unitl Command Success  ${helm_cmd} push --ca-file=/ca/server.crt --username=${user} --password=${pwd} ${chart_filename} ${helm_repo_name}
 
 Helm Chart Push
     [Arguments]  ${ip}  ${user}  ${pwd}  ${chart_file}  ${archive}  ${project}  ${repo_name}  ${verion}

tests/resources/Nightly-Util.robot

@@ -40,6 +40,20 @@ CA Setup
     Log To Console  Prepare Docker Cert ...
     Prepare Docker Cert  ${ip}
 
+Nightly Test Setup For Nightly
+    [Arguments]  ${ip}  ${HARBOR_PASSWORD}  ${ip1}==${EMPTY}
+    Run Keyword If  '${ip1}' != '${EMPTY}'  CA setup For Nightly  ${ip1}  ${HARBOR_PASSWORD}  /ca/ca1.crt
+    Run Keyword If  '${ip1}' != '${EMPTY}'  Run  rm -rf ./harbor_ca.crt
+    Run Keyword  CA setup For Nightly  ${ip}  ${HARBOR_PASSWORD}
+    Run Keyword  Start Docker Daemon Locally
+
+CA Setup For Nightly
+    [Arguments]  ${ip}  ${HARBOR_PASSWORD}  ${cert}=/ca/ca.crt
+    Run  cp ${cert} harbor_ca.crt
+    Generate Certificate Authority For Chrome  ${HARBOR_PASSWORD}
+    Prepare Docker Cert  ${ip}
+    Prepare Helm Cert
+
 Collect Nightly Logs
     [Arguments]  ${ip}  ${SSH_PWD}  ${ip1}==${EMPTY}
     Run Keyword  Collect Logs  ${ip}  ${SSH_PWD}

tests/robot-cases/Group1-Nightly/Common.robot

@@ -619,7 +619,7 @@ Test Case - Push CNAB Bundle and Display
     Create An New Project And Go Into Project  test${d}
 
     ${target}=  Set Variable  ${ip}/test${d}/cnab${d}:cnab_tag${d}
-    CNAB Push Bundle  ${ip}  user010  Test1@34  ${target}  ./tests/robot-cases/Group0-Util/bundle.json
+    Retry Keyword N Times When Error  5  CNAB Push Bundle  ${ip}  user010  Test1@34  ${target}  ./tests/robot-cases/Group0-Util/bundle.json
 
     Go Into Project  test${d}
     Wait Until Page Contains  test${d}/cnab${d}

tests/robot-cases/Group1-Nightly/Setup_Nightly.robot

+// Copyright Project Harbor Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+*** Settings ***
+Documentation  Harbor BATs
+Resource  ../../resources/Util.robot
+Default Tags  Nightly
+
+*** Test Cases ***
+Test Suites Setup For UI Test
+    Nightly Test Setup For Nightly  ${ip}  ${HARBOR_PASSWORD}  ${ip1}
+    Setup API Test
+
+Test Case - Get Harbor Version
+#Just get harbor version and log it
+    Get Harbor Version
\ No newline at end of file

tests/robot-cases/Group3-Upgrade/prepare.py

@@ -403,9 +403,6 @@ class HarborAPI:
                 pass
         open(target, 'wb').write(ca_content.encode('utf-8'))
 
-    @get_feature_branch
-    def push_artifact(self, project, **kwargs):
-
 def request(url, method, user = None, userp = None, **kwargs):
     if user is None:
         user = "admin"