Commit 5c3fd9fc authored by kunw's avatar kunw
Browse files

Merge latest updates.

parent a80008d0
......@@ -31,6 +31,7 @@ env:
before_install:
- sudo ./tests/hostcfg.sh
- sudo ./tests/generateCerts.sh
- sudo ./make/prepare
install:
......@@ -92,12 +93,12 @@ script:
- goveralls -coverprofile=profile.cov -service=travis-ci
- docker-compose -f make/docker-compose.test.yml down
- sudo make/prepare
- sudo rm -rf /data/config/*
- docker-compose -f make/dev/docker-compose.yml up -d
- sudo rm -rf /data/config/*
- ls /data/cert
- sudo make install GOBUILDIMAGE=golang:1.7.3 COMPILETAG=compile_golangimage CLARITYIMAGE=danieljt/harbor-clarity-base:0.8.4 NOTARYFLAG=true
- docker ps
- go run tests/startuptest.go http://localhost/
- go run tests/startuptest.go https://localhost/
- go run tests/userlogintest.go -name ${HARBOR_ADMIN} -passwd ${HARBOR_ADMIN_PASSWD}
# - sudo ./tests/testprepare.sh
......
......@@ -79,6 +79,13 @@ REGISTRYSERVER=
REGISTRYPROJECTNAME=vmware
DEVFLAG=true
NOTARYFLAG=false
REGISTRYVERSION=2.6.0
NGINXVERSION=1.11.5
PHOTONVERSION=1.0
NOTARYVERSION=server-0.5.0-fix
NOTARYSIGNERVERSION=signer-0.5.0
MARIADBVERSION=10.1.10
HTTPPROXY=
#clarity parameters
CLARITYIMAGE=danieljt/harbor-clarity-base[:tag]
......@@ -206,7 +213,11 @@ compile_jobservice:
compile_clarity:
@echo "compiling binary for clarity ui..."
@$(DOCKERCMD) run --rm -v $(UIPATH)/static/new-ui:$(CLARITYSEEDPATH)/dist -v $(UINGPATH)/src:$(CLARITYSEEDPATH)/src -v $(UINGPATH)/src/app:$(CLARITYSEEDPATH)/src/app $(CLARITYIMAGE) $(SHELL) $(CLARITYBUILDSCRIPT)
@if [ "$(HTTPPROXY)" != "" ] ; then \
$(DOCKERCMD) run --rm -v $(UIPATH)/static:$(CLARITYSEEDPATH)/dist -v $(UINGPATH)/src:$(CLARITYSEEDPATH)/src $(CLARITYIMAGE) $(SHELL) $(CLARITYBUILDSCRIPT) -p $(HTTPPROXY); \
else \
$(DOCKERCMD) run --rm -v $(UIPATH)/static:$(CLARITYSEEDPATH)/dist -v $(UINGPATH)/src:$(CLARITYSEEDPATH)/src $(CLARITYIMAGE) $(SHELL) $(CLARITYBUILDSCRIPT); \
fi
@echo "Done."
compile_normal: compile_clarity compile_adminserver compile_ui compile_jobservice
......@@ -290,13 +301,13 @@ package_offline: compile build modify_composefile
@cp NOTICE $(HARBORPKG)/NOTICE
@echo "pulling nginx and registry..."
@$(DOCKERPULL) registry:2.5.1
@$(DOCKERPULL) nginx:1.11.5
@$(DOCKERPULL) registry:$(REGISTRYVERSION)
@$(DOCKERPULL) nginx:$(NGINXVERSION)
@if [ "$(NOTARYFLAG)" = "true" ] ; then \
echo "pulling notary and mariadb..."; \
$(DOCKERPULL) jiangd/notary:server-0.5.0-fix; \
$(DOCKERPULL) notary:signer-0.5.0; \
$(DOCKERPULL) mariadb:10.1.10; \
$(DOCKERPULL) jiangd/notary:$(NOTARYVERSION); \
$(DOCKERPULL) notary:$(NOTARYSIGNERVERSION); \
$(DOCKERPULL) mariadb:$(MARIADBVERSION); \
fi
@echo "saving harbor docker image"
......@@ -307,8 +318,8 @@ package_offline: compile build modify_composefile
$(DOCKERIMAGENAME_LOG):$(VERSIONTAG) \
$(DOCKERIMAGENAME_DB):$(VERSIONTAG) \
$(DOCKERIMAGENAME_JOBSERVICE):$(VERSIONTAG) \
nginx:1.11.5 registry:2.5.1 photon:1.0 \
jiangd/notary:server-0.5.0-fix notary:signer-0.5.0 mariadb:10.1.10; \
nginx:$(NGINXVERSION) registry:$(REGISTRYVERSION) photon:$(PHOTONVERSION) \
jiangd/notary:$(NOTARYVERSION) notary:$(NOTARYSIGNERVERSION) mariadb:$(MARIADBVERSION); \
else \
$(DOCKERSAVE) -o $(HARBORPKG)/$(DOCKERIMGFILE).$(VERSIONTAG).tgz \
$(DOCKERIMAGENAME_ADMINSERVER):$(VERSIONTAG) \
......@@ -316,7 +327,7 @@ package_offline: compile build modify_composefile
$(DOCKERIMAGENAME_LOG):$(VERSIONTAG) \
$(DOCKERIMAGENAME_DB):$(VERSIONTAG) \
$(DOCKERIMAGENAME_JOBSERVICE):$(VERSIONTAG) \
nginx:1.11.5 registry:2.5.1 photon:1.0 ; \
nginx:$(NGINXVERSION) registry:$(REGISTRYVERSION) photon:$(PHOTONVERSION) ; \
fi
@if [ "$(NOTARYFLAG)" = "true" ] ; then \
......@@ -324,14 +335,14 @@ package_offline: compile build modify_composefile
$(HARBORPKG)/common/templates $(HARBORPKG)/$(DOCKERIMGFILE).$(VERSIONTAG).tgz \
$(HARBORPKG)/prepare $(HARBORPKG)/NOTICE \
$(HARBORPKG)/LICENSE $(HARBORPKG)/install.sh \
$(HARBORPKG)/harbor.cfg $(HARBORPKG)/$(DOCKERCOMPOSEFILENAME) ; \
$(HARBORPKG)/harbor.cfg $(HARBORPKG)/$(DOCKERCOMPOSEFILENAME) \
$(HARBORPKG)/$(DOCKERCOMPOSENOTARYFILENAME) ; \
else \
$(TARCMD) -zcvf harbor-offline-installer-$(VERSIONTAG).tgz \
$(HARBORPKG)/common/templates $(HARBORPKG)/$(DOCKERIMGFILE).$(VERSIONTAG).tgz \
$(HARBORPKG)/prepare $(HARBORPKG)/NOTICE \
$(HARBORPKG)/LICENSE $(HARBORPKG)/install.sh \
$(HARBORPKG)/harbor.cfg $(HARBORPKG)/$(DOCKERCOMPOSEFILENAME) \
$(HARBORPKG)/$(DOCKERCOMPOSENOTARYFILENAME) ; \
$(HARBORPKG)/harbor.cfg $(HARBORPKG)/$(DOCKERCOMPOSEFILENAME) ; \
fi
@rm -rf $(HARBORPKG)
......@@ -400,7 +411,7 @@ cleanimage:
- $(DOCKERRMIMAGE) -f $(DOCKERIMAGENAME_DB):$(VERSIONTAG)
- $(DOCKERRMIMAGE) -f $(DOCKERIMAGENAME_JOBSERVICE):$(VERSIONTAG)
- $(DOCKERRMIMAGE) -f $(DOCKERIMAGENAME_LOG):$(VERSIONTAG)
# - $(DOCKERRMIMAGE) -f registry:2.5.1
# - $(DOCKERRMIMAGE) -f registry:$(REGISTRYVERSION)
# - $(DOCKERRMIMAGE) -f nginx:1.11.5
cleandockercomposefile:
......
......@@ -1529,6 +1529,22 @@ paths:
description: User does not have permission of admin role.
500:
description: Unexpected internal errors.
/configurations/reset:
post:
summary: Reset system configurations.
description: |
Reset system configurations from environment variables. Can only be accessed by admin user.
tags:
- Products
responses:
200:
description: Reset system configurations successfully.
401:
description: User need to log in first.
403:
description: User does not have permission of admin role.
500:
description: Unexpected internal errors.
/email/ping:
post:
summary: Test connection and authentication with email server.
......@@ -2009,6 +2025,9 @@ definitions:
self_registration:
type: boolean
description: Indicate whether the Harbor instance enable user to register himself.
has_ca_root:
type: boolean
description: Indicate whether there is a ca root cert file ready for download in the file system.
SystemInfo:
type: object
properties:
......@@ -2082,7 +2101,7 @@ definitions:
type: string
description: The host of email server.
email_port:
type: string
type: integer
description: The port of email server.
email_username:
type: string
......@@ -2091,7 +2110,7 @@ definitions:
type: string
description: The password of email server.
email_ssl:
type: string
type: boolean
description: Use ssl/tls or not.
email_identity:
type: string
......@@ -2146,4 +2165,4 @@ definitions:
description: The creation time of repository.
update_time:
type: string
description: The update time of repository.
\ No newline at end of file
description: The update time of repository.
......@@ -191,14 +191,14 @@ Run the below commands on the host which Harbor is deployed on to preview what f
```sh
$ docker-compose stop
$ docker run -it --name gc --rm --volumes-from registry registry:2.5.1 garbage-collect --dry-run /etc/registry/config.yml
$ docker run -it --name gc --rm --volumes-from registry registry:2.6.0 garbage-collect --dry-run /etc/registry/config.yml
```
**NOTE:** The above option "--dry-run" will print the progress without removing any data.
Verify the result of the above test, then use the below commands to perform garbage collection and restart Harbor.
```sh
$ docker run -it --name gc --rm --volumes-from registry registry:2.5.1 garbage-collect /etc/registry/config.yml
$ docker run -it --name gc --rm --volumes-from registry registry:2.6.0 garbage-collect /etc/registry/config.yml
$ docker-compose start
```
......
......@@ -37,3 +37,4 @@ USE_COMPRESSED_JS=$use_compressed_js
GODEBUG=netdns=cgo
ADMIRAL_URL=$admiral_url
WITH_NOTARY=$with_notary
RESET=false
......@@ -8,7 +8,7 @@ events {
http {
tcp_nodelay on;
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/conf.d/*.upstream.conf;
# this is necessary for us to be able to disable request buffering in all cases
proxy_http_version 1.1;
......@@ -42,6 +42,8 @@ http {
# required to avoid HTTP 411: see Issue #1486 (https://github.com/docker/docker/issues/1486)
chunked_transfer_encoding on;
include /etc/nginx/conf.d/*.location.conf;
location / {
proxy_pass http://ui/;
proxy_set_header Host $$http_host;
......@@ -62,19 +64,6 @@ http {
return 404;
}
location /notary/v2/ {
proxy_pass http://notary-server/v2/;
proxy_set_header Host $$http_host;
proxy_set_header X-Real-IP $$remote_addr;
proxy_set_header X-Forwarded-For $$proxy_add_x_forwarded_for;
# When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings.
proxy_set_header X-Forwarded-Proto $$scheme;
proxy_buffering off;
proxy_request_buffering off;
}
location /v2/ {
proxy_pass http://registry/v2/;
proxy_set_header Host $$http_host;
......
location /notary/v2/ {
proxy_pass http://notary-server/v2/;
proxy_set_header Host $http_host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
# When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings.
proxy_set_header X-Forwarded-Proto $scheme;
proxy_buffering off;
proxy_request_buffering off;
}
......@@ -10,7 +10,7 @@ services:
ports:
- 1514:514
registry:
image: library/registry:2.5.1
image: library/registry:2.6.0
restart: always
volumes:
- /data/registry:/storage
......
FROM node:7.5.0
COPY angular-cli.json /
COPY index.html /
COPY entrypoint.sh /
RUN mkdir -p /clarity-seed
RUN npm install -g @angular/cli && \
chmod u+x entrypoint.sh
COPY src/ui_ng/package.json /clarity-seed
COPY src/ui_ng/tslint.json /clarity-seed
COPY src/ui_ng/typings.json /clarity-seed
COPY src/ui_ng/yarn.lock /clarity-seed
COPY make/dev/nodeclarity/angular-cli.json /clarity-seed
COPY make/dev/nodeclarity/entrypoint.sh /
VOLUME ["/clarity-seed", "/clarity-seed/dist"]
WORKDIR /clarity-seed
ENTRYPOINT ["/entrypoint.sh"]
RUN npm install -g @angular/cli && \
npm install && \
chmod u+x /entrypoint.sh
VOLUME ["/clarity-seed", "/clarity-seed/dist"]
\ No newline at end of file
#!/bin/bash
set -e
cd /clarity-seed
rm -rf dist/*
cp /angular-cli.json /clarity-seed
npm_proxy=
while getopts p: option
do
case "${option}"
in
p) npm_proxy=${OPTARG};;
esac
done
if [ ! -z "$npm_proxy" -a "$npm_proxy" != " " ]; then
npm config set proxy $npm_proxy
fi
npm install
ng build
cp /index.html dist/index.html
cp -r ./src/i18n/ dist/
......@@ -11,7 +11,7 @@ services:
networks:
- harbor
registry:
image: registry:2.5.1
image: registry:2.6.0
container_name: registry
restart: always
volumes:
......
......@@ -147,7 +147,10 @@ token_expiration = rcp.get("configuration", "token_expiration")
verify_remote_cert = rcp.get("configuration", "verify_remote_cert")
proj_cre_restriction = rcp.get("configuration", "project_creation_restriction")
secretkey_path = rcp.get("configuration", "secretkey_path")
admiral_url = rcp.get("configuration", "admiral_url")
if rcp.has_option("configuration", "admiral_url"):
admiral_url = rcp.get("configuration", "admiral_url")
else:
admiral_url = ""
secret_key = get_secret_key(secretkey_path)
########
......@@ -313,20 +316,21 @@ if args.notary_mode:
shutil.rmtree(os.path.join(notary_config_dir, "mysql-initdb.d"))
shutil.copytree(os.path.join(notary_temp_dir, "mysql-initdb.d"), os.path.join(notary_config_dir, "mysql-initdb.d"))
#TODO:generate certs?
print ("Copying certs for notary signer")
print("Copying certs for notary signer")
shutil.copy2(os.path.join(notary_temp_dir, "notary-signer.crt"), notary_config_dir)
shutil.copy2(os.path.join(notary_temp_dir, "notary-signer.key"), notary_config_dir)
shutil.copy2(os.path.join(notary_temp_dir, "root-ca.crt"), notary_config_dir)
shutil.copy2(os.path.join(registry_config_dir, "root.crt"), notary_config_dir)
print ("Copying notary signer configuration file")
print("Copying notary signer configuration file")
shutil.copy2(os.path.join(notary_temp_dir, "signer-config.json"), notary_config_dir)
render(os.path.join(notary_temp_dir, "server-config.json"),
os.path.join(notary_config_dir, "server-config.json"),
token_endpoint=ui_url)
print ("Copying nginx configuration file for notary")
shutil.copy2(os.path.join(templates_dir, "nginx", "nginx.notary.conf"), nginx_conf_d)
print("Copying nginx configuration file for notary")
shutil.copy2(os.path.join(templates_dir, "nginx", "notary.upstream.conf"), nginx_conf_d)
shutil.copy2(os.path.join(templates_dir, "nginx", "notary.location.conf"), nginx_conf_d)
default_alias = ''.join(random.choice(string.ascii_letters) for i in range(8))
render(os.path.join(notary_temp_dir, "signer_env"), os.path.join(notary_config_dir, "signer_env"), alias = default_alias)
......
......@@ -16,6 +16,7 @@
package api
import (
"encoding/json"
"net/http"
)
......@@ -32,3 +33,17 @@ func handleUnauthorized(w http.ResponseWriter) {
http.Error(w, http.StatusText(http.StatusUnauthorized),
http.StatusUnauthorized)
}
// response status code will be written automatically if there is an error
func writeJSON(w http.ResponseWriter, v interface{}) error {
b, err := json.Marshal(v)
if err != nil {
handleInternalServerError(w)
return err
}
if _, err = w.Write(b); err != nil {
return err
}
return nil
}
......@@ -19,40 +19,13 @@ import (
"encoding/json"
"io/ioutil"
"net/http"
"os"
cfg "github.com/vmware/harbor/src/adminserver/systemcfg"
"github.com/vmware/harbor/src/common/utils/log"
)
func isAuthenticated(r *http.Request) (bool, error) {
uiSecret := os.Getenv("UI_SECRET")
jobserviceSecret := os.Getenv("JOBSERVICE_SECRET")
c, err := r.Cookie("secret")
if err != nil {
if err == http.ErrNoCookie {
return false, nil
}
return false, err
}
return c != nil && (c.Value == uiSecret ||
c.Value == jobserviceSecret), nil
}
// ListCfgs lists configurations
func ListCfgs(w http.ResponseWriter, r *http.Request) {
authenticated, err := isAuthenticated(r)
if err != nil {
log.Errorf("failed to check whether the request is authenticated or not: %v", err)
handleInternalServerError(w)
return
}
if !authenticated {
handleUnauthorized(w)
return
}
cfg, err := cfg.GetSystemCfg()
if err != nil {
log.Errorf("failed to get system configurations: %v", err)
......@@ -60,31 +33,14 @@ func ListCfgs(w http.ResponseWriter, r *http.Request) {
return
}
b, err := json.MarshalIndent(cfg, "", " ")
if err != nil {
log.Errorf("failed to marshal configurations: %v", err)
handleInternalServerError(w)
return
}
if _, err = w.Write(b); err != nil {
if err = writeJSON(w, cfg); err != nil {
log.Errorf("failed to write response: %v", err)
return
}
}
// UpdateCfgs updates configurations
func UpdateCfgs(w http.ResponseWriter, r *http.Request) {
authenticated, err := isAuthenticated(r)
if err != nil {
log.Errorf("failed to check whether the request is authenticated or not: %v", err)
handleInternalServerError(w)
return
}
if !authenticated {
handleUnauthorized(w)
return
}
b, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Errorf("failed to read request body: %v", err)
......@@ -104,3 +60,12 @@ func UpdateCfgs(w http.ResponseWriter, r *http.Request) {
return
}
}
// ResetCfgs resets configurations from environment variables
func ResetCfgs(w http.ResponseWriter, r *http.Request) {
if err := cfg.Reset(); err != nil {
log.Errorf("failed to reset system configurations: %v", err)
handleInternalServerError(w)
return
}
}
......@@ -43,7 +43,7 @@ func TestConfigAPI(t *testing.T) {
secret := "secret"
envs := map[string]string{
"AUTH_MODE": comcfg.DBAuth,
"JSON_CFG_STORE_PATH": configPath,
"KEY_PATH": secretKeyPath,
"UI_SECRET": secret,
......@@ -79,19 +79,12 @@ func TestConfigAPI(t *testing.T) {
return
}
w := httptest.NewRecorder()
ListCfgs(w, r)
if w.Code != http.StatusUnauthorized {
t.Errorf("unexpected status code: %d != %d", w.Code, http.StatusUnauthorized)
return
}
r.AddCookie(&http.Cookie{
Name: "secret",
Value: secret,
})
w = httptest.NewRecorder()
w := httptest.NewRecorder()
ListCfgs(w, r)
if w.Code != http.StatusOK {
t.Errorf("unexpected status code: %d != %d", w.Code, http.StatusOK)
......@@ -164,7 +157,55 @@ func TestConfigAPI(t *testing.T) {
mode := m[comcfg.AUTHMode].(string)
if mode != comcfg.LDAPAuth {
t.Errorf("unexpected ldap scope: %s != %s", mode, comcfg.LDAPAuth)
t.Errorf("unexpected auth mode: %s != %s", mode, comcfg.LDAPAuth)
return
}
// reset configurations
w = httptest.NewRecorder()
r, err = http.NewRequest("POST", "", nil)
if err != nil {
t.Errorf("failed to create request: %v", err)
return
}
r.AddCookie(&http.Cookie{
Name: "secret",
Value: secret,
})
ResetCfgs(w, r)
if w.Code != http.StatusOK {
t.Errorf("unexpected status code: %d != %d", w.Code, http.StatusOK)
return
}
// confirm the reset
r, err = http.NewRequest("GET", "", nil)
if err != nil {
t.Errorf("failed to create request: %v", err)
return
}
r.AddCookie(&http.Cookie{
Name: "secret",
Value: secret,
})
w = httptest.NewRecorder()
ListCfgs(w, r)
if w.Code != http.StatusOK {
t.Errorf("unexpected status code: %d != %d", w.Code, http.StatusOK)
return
}
m, err = parse(w.Body)
if err != nil {
t.Errorf("failed to parse response body: %v", err)
return
}
mode = m[comcfg.AUTHMode].(string)
if mode != comcfg.DBAuth {
t.Errorf("unexpected auth mode: %s != %s", mode, comcfg.LDAPAuth)
return
}
}
......
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package api
import (
"net/http"
"github.com/vmware/harbor/src/adminserver/systeminfo/imagestorage"
"github.com/vmware/harbor/src/common/utils/log"
)
// Capacity handles /api/systeminfo/capacity and returns system capacity
func Capacity(w http.ResponseWriter, r *http.Request) {
capacity, err := imagestorage.GlobalDriver.Cap()
if err != nil {
log.Errorf("failed to get capacity: %v", err)
handleInternalServerError(w)
return
}
if err = writeJSON(w, capacity); err != nil {
log.Errorf("failed to write response: %v", err)
return
}
}
/*
Copyright (c) 2016 VMware, Inc. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package api
import (
"encoding/json"
"errors"