Commit 8857e89e authored by Ziming Zhang's avatar Ziming Zhang Committed by Ziming
Browse files

feature(redis) support redis sentinel


Signed-off-by: default avatarZiming Zhang <zziming@vmware.com>
parent d891e023
......@@ -116,6 +116,8 @@ jobs:
echo "::set-env name=GOPATH::$(go env GOPATH):$GITHUB_WORKSPACE"
echo "::add-path::$(go env GOPATH)/bin"
echo "::set-env name=TOKEN_PRIVATE_KEY_PATH::${GITHUB_WORKSPACE}/src/github.com/goharbor/harbor/tests/private_key.pem"
IP=`hostname -I | awk '{print $1}'`
echo "::set-env name=IP::$IP"
shell: bash
- name: before_install
run: |
......@@ -129,20 +131,18 @@ jobs:
curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose
chmod +x docker-compose
sudo mv docker-compose /usr/local/bin
IP=`hostname -I | awk '{print $1}'`
echo '{"insecure-registries" : ["'$IP':5000"]}' | sudo tee /etc/docker/daemon.json
echo "::set-env name=IP::$IP"
sudo cp ./tests/harbor_ca.crt /usr/local/share/ca-certificates/
sudo update-ca-certificates
sudo service docker restart
wget https://get.helm.sh/helm-v2.14.1-linux-386.tar.gz && tar zxvf helm-v2.14.1-linux-386.tar.gz
sudo mv linux-386/helm /usr/local/bin/helm2
helm2 init --client-only
helm2 plugin install https://github.com/chartmuseum/helm-push
helm2 plugin list | grep push || helm2 plugin install https://github.com/chartmuseum/helm-push
wget https://get.helm.sh/helm-v3.1.1-linux-386.tar.gz && tar zxvf helm-v3.1.1-linux-386.tar.gz
sudo mv linux-386/helm /usr/local/bin/helm3
helm3 plugin install https://github.com/chartmuseum/helm-push
mkdir -p $CNAB_PATH && cd $CNAB_PATH && git clone https://github.com/cnabio/cnab-to-oci.git
helm3 plugin list | grep push || helm3 plugin install https://github.com/chartmuseum/helm-push
rm -rf $CNAB_PATH;mkdir -p $CNAB_PATH && cd $CNAB_PATH && git clone https://github.com/cnabio/cnab-to-oci.git
cd cnab-to-oci && git checkout v0.3.0-beta4
go list
make build
......
......@@ -98,16 +98,19 @@ PKGVERSIONTAG=dev
PREPARE_VERSION_NAME=versions
#versions
REGISTRYVERSION=v2.7.1-patch-2819-2553
REGISTRYVERSION=v2.7.1-patch-2819-2553-redis
NOTARYVERSION=v0.6.1
CLAIRVERSION=v2.1.4
NOTARYMIGRATEVERSION=v3.5.4
CLAIRADAPTERVERSION=v1.0.2
CLAIRADAPTERVERSION=v1.1.0-rc1
TRIVYVERSION=v0.9.1
TRIVYADAPTERVERSION=v0.12.0
TRIVYADAPTERVERSION=v0.13.0
# version of chartmuseum
CHARTMUSEUMVERSION=v0.12.0
CHARTMUSEUMVERSION=v0.12.0-redis
# version of chartmuseum for pulling the source code
CHARTMUSEUM_SRC_TAG=v0.12.0
# version of registry for pulling the source code
REGISTRY_SRC_TAG=v2.7.1
......@@ -375,7 +378,7 @@ build:
-e TRIVYVERSION=$(TRIVYVERSION) -e TRIVYADAPTERVERSION=$(TRIVYADAPTERVERSION) \
-e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRADAPTERVERSION=$(CLAIRADAPTERVERSION) -e VERSIONTAG=$(VERSIONTAG) \
-e BUILDBIN=$(BUILDBIN) \
-e CHARTMUSEUMVERSION=$(CHARTMUSEUMVERSION) -e DOCKERIMAGENAME_CHART_SERVER=$(DOCKERIMAGENAME_CHART_SERVER) \
-e CHARTMUSEUMVERSION=$(CHARTMUSEUMVERSION) -e CHARTMUSEUM_SRC_TAG=$(CHARTMUSEUM_SRC_TAG) -e DOCKERIMAGENAME_CHART_SERVER=$(DOCKERIMAGENAME_CHART_SERVER) \
-e NPM_REGISTRY=$(NPM_REGISTRY) -e BASEIMAGETAG=$(BASEIMAGETAG) -e BASEIMAGENAMESPACE=$(BASEIMAGENAMESPACE) \
-e CLAIRURL=$(CLAIRURL) -e CHARTURL=$(CHARTURL) -e NORARYURL=$(NORARYURL) -e REGISTRYURL=$(REGISTRYURL) -e CLAIR_ADAPTER_DOWNLOAD_URL=$(CLAIR_ADAPTER_DOWNLOAD_URL) \
-e TRIVY_DOWNLOAD_URL=$(TRIVY_DOWNLOAD_URL) -e TRIVY_ADAPTER_DOWNLOAD_URL=$(TRIVY_ADAPTER_DOWNLOAD_URL)
......@@ -383,7 +386,7 @@ build:
build_base_docker:
@for name in chartserver clair clair-adapter trivy-adapter core db jobservice log nginx notary-server notary-signer portal prepare redis registry registryctl; do \
echo $$name ; \
$(DOCKERBUILD) --pull -f $(MAKEFILEPATH_PHOTON)/$$name/Dockerfile.base -t $(BASEIMAGENAMESPACE)/harbor-$$name-base:$(BASEIMAGETAG) --label base-build-date=$(date +"%Y%m%d") . && \
$(DOCKERBUILD) --pull --no-cache -f $(MAKEFILEPATH_PHOTON)/$$name/Dockerfile.base -t $(BASEIMAGENAMESPACE)/harbor-$$name-base:$(BASEIMAGETAG) --label base-build-date=$(date +"%Y%m%d") . && \
$(PUSHSCRIPTPATH)/$(PUSHSCRIPTNAME) $(BASEIMAGENAMESPACE)/harbor-$$name-base:$(BASEIMAGETAG) $(REGISTRYUSER) $(REGISTRYPASSWORD) || exit 1; \
done
......
......@@ -171,9 +171,14 @@ _version: 2.0.0
# Uncomment external_redis if using external Redis server
# external_redis:
# host: redis
# port: 6379
# # support redis, redis+sentinel
# # host for redis: <host_redis>:<port_redis>
# # host for redis+sentinel:
# # <host_sentinel1>:<port_sentinel1>,<host_sentinel2>:<port_sentinel2>,<host_sentinel3>:<port_sentinel3>
# host: redis:6379
# password:
# # sentinel_master_set must be set to support redis+sentinel
# #sentinel_master_set:
# # db_index 0 is for core, it's unchangeable
# registry_db_index: 1
# jobservice_db_index: 2
......
......@@ -192,7 +192,7 @@ _build_chart_server:
rm -rf $(DOCKERFILEPATH_CHART_SERVER)/binary && mkdir -p $(DOCKERFILEPATH_CHART_SERVER)/binary && \
$(call _get_binary, $(CHARTURL), $(DOCKERFILEPATH_CHART_SERVER)/binary/chartm); \
else \
cd $(DOCKERFILEPATH_CHART_SERVER) && $(DOCKERFILEPATH_CHART_SERVER)/builder $(GOBUILDIMAGE) $(CHART_SERVER_CODE_BASE) $(CHARTMUSEUMVERSION) $(CHART_SERVER_MAIN_PATH) $(CHART_SERVER_BIN_NAME) && cd - ; \
cd $(DOCKERFILEPATH_CHART_SERVER) && $(DOCKERFILEPATH_CHART_SERVER)/builder $(GOBUILDIMAGE) $(CHART_SERVER_CODE_BASE) $(CHARTMUSEUM_SRC_TAG) $(CHART_SERVER_MAIN_PATH) $(CHART_SERVER_BIN_NAME) && cd - ; \
fi ; \
echo "building chartmuseum container for photon..." ; \
$(DOCKERBUILD) --build-arg harbor_base_image_version=$(BASEIMAGETAG) --build-arg harbor_base_namespace=$(BASEIMAGENAMESPACE) -f $(DOCKERFILEPATH_CHART_SERVER)/$(DOCKERFILENAME_CHART_SERVER) -t $(DOCKERIMAGENAME_CHART_SERVER):$(VERSIONTAG) . ; \
......
......@@ -26,6 +26,7 @@ cur=$PWD
mkdir -p binary
rm -rf binary/$BIN_NAME || true
cp compile.sh binary/
cp *.patch binary/
docker run -it --rm -v $cur/binary:/go/bin --name golang_code_builder $GOLANG_IMAGE /bin/bash /go/bin/compile.sh $GIT_PATH $CODE_VERSION $MAIN_GO_PATH $BIN_NAME
......
......@@ -26,6 +26,11 @@ set -e
cd $SRC_PATH
git checkout tags/$VERSION -b $VERSION
#Patch
for p in $(ls /go/bin/*.patch); do
git apply $p || exit /b 1
done
#Compile
cd $SRC_PATH/$MAIN_GO_PATH && go build -a -o $BIN_NAME
mv $BIN_NAME /go/bin/
diff --git a/cmd/chartmuseum/main.go b/cmd/chartmuseum/main.go
index e2d8ec0..116b1d4 100644
--- a/cmd/chartmuseum/main.go
+++ b/cmd/chartmuseum/main.go
@@ -264,6 +264,8 @@ func storeFromConfig(conf *config.Config) cache.Store {
switch cacheFlag {
case "redis":
store = redisCacheFromConfig(conf)
+ case "redis_sentinel":
+ store = redisSentinelCacheFromConfig(conf)
default:
crash("Unsupported cache store: ", cacheFlag)
}
@@ -280,6 +282,16 @@ func redisCacheFromConfig(conf *config.Config) cache.Store {
))
}
+func redisSentinelCacheFromConfig(conf *config.Config) cache.Store {
+ crashIfConfigMissingVars(conf, []string{"cache.redis.addr", "cache.redis.mastername"})
+ return cache.Store(cache.NewRedisSentinelStore(
+ conf.GetString("cache.redis.mastername"),
+ strings.Split(conf.GetString("cache.redis.addr"), ","),
+ conf.GetString("cache.redis.password"),
+ conf.GetInt("cache.redis.db"),
+ ))
+}
+
func crashIfConfigMissingVars(conf *config.Config, vars []string) {
missing := []string{}
for _, v := range vars {
diff --git a/pkg/cache/redis_sentinel.go b/pkg/cache/redis_sentinel.go
new file mode 100644
index 0000000..0c73427
--- /dev/null
+++ b/pkg/cache/redis_sentinel.go
@@ -0,0 +1,18 @@
+package cache
+
+import (
+ "github.com/go-redis/redis"
+)
+
+// NewRedisStore creates a new RedisStore
+func NewRedisSentinelStore(masterName string, sentinelAddrs []string, password string, db int) *RedisStore {
+ store := &RedisStore{}
+ redisClientOptions := &redis.FailoverOptions{
+ MasterName: masterName,
+ SentinelAddrs: sentinelAddrs,
+ Password: password,
+ DB: db,
+ }
+ store.Client = redis.NewFailoverClient(redisClientOptions)
+ return store
+}
diff --git a/pkg/config/vars.go b/pkg/config/vars.go
index 2b30ec4..603eebc 100644
--- a/pkg/config/vars.go
+++ b/pkg/config/vars.go
@@ -237,10 +237,19 @@ var configVars = map[string]configVar{
Default: "",
CLIFlag: cli.StringFlag{
Name: "cache-redis-addr",
- Usage: "address of Redis service (host:port)",
+ Usage: "address of Redis service (host:port), addresses of Redis+Sentinel service (host1:port1,host2:port2)",
EnvVar: "CACHE_REDIS_ADDR",
},
},
+ "cache.redis.mastername": {
+ Type: stringType,
+ Default: "",
+ CLIFlag: cli.StringFlag{
+ Name: "cache-redis-mastername",
+ Usage: "address of Redis+Sentinel mastername",
+ EnvVar: "CACHE_REDIS_MASTERNAME",
+ },
+ },
"cache.redis.password": {
Type: stringType,
Default: "",
......@@ -306,9 +306,14 @@ external_database:
{% if external_redis is defined %}
external_redis:
host: {{ external_redis.host }}
port: {{ external_redis.port }}
# support redis, redis+sentinel
# host for redis: <host_redis>:<port_redis>
# host for redis+sentinel:
# <host_sentinel1>:<port_sentinel1>,<host_sentinel2>:<port_sentinel2>,<host_sentinel3>:<port_sentinel3>
host: {{ external_redis.host }}:{{ external_redis.port }}
password: {{ external_redis.password }}
# sentinel_master_set must be set to support redis+sentinel
#sentinel_master_set:
# db_index 0 is for core, it's unchangeable
registry_db_index: {{ external_redis.registry_db_index }}
jobservice_db_index: {{ external_redis.jobservice_db_index }}
......@@ -319,9 +324,14 @@ external_redis:
{% else %}
# Umcomments external_redis if using external Redis server
# external_redis:
# host: redis
# port: 6379
# # support redis, redis+sentinel
# # host for redis: <host_redis>:<port_redis>
# # host for redis+sentinel:
# # <host_sentinel1>:<port_sentinel1>,<host_sentinel2>:<port_sentinel2>,<host_sentinel3>:<port_sentinel3>
# host: redis:6379
# password:
# # sentinel_master_set must be set to support redis+sentinel
# #sentinel_master_set:
# # db_index 0 is for core, it's unchangeable
# registry_db_index: 1
# jobservice_db_index: 2
......
......@@ -11,9 +11,16 @@ PORT=9999
# Only support redis now. If redis is setup, then enable cache
CACHE={{cache_store}}
{% if cache_redis_mastername %}
CACHE_REDIS_ADDR={{cache_redis_addr}}
CACHE_REDIS_MASTERNAME={{cache_redis_mastername}}
CACHE_REDIS_PASSWORD={{cache_redis_password}}
CACHE_REDIS_DB={{cache_redis_db_index}}
{% else %}
CACHE_REDIS_ADDR={{cache_redis_addr}}
CACHE_REDIS_PASSWORD={{cache_redis_password}}
CACHE_REDIS_DB={{cache_redis_db_index}}
{% endif %}
# Credential for internal communication
BASIC_AUTH_USER=chart_controller
......
CONFIG_PATH=/etc/core/app.conf
UAA_CA_ROOT=/etc/core/certificates/uaa_ca.pem
_REDIS_URL={{redis_host}}:{{redis_port}},100,{{redis_password}},0,{{redis_idle_timeout_seconds}}
_REDIS_URL_CORE={{redis_url_core}}
SYNC_QUOTA=true
CHART_CACHE_DRIVER={{chart_cache_driver}}
_REDIS_URL_REG={{redis_url_reg}}
......
......@@ -17,7 +17,13 @@ storage:
disable: true
{% endif %}
redis:
addr: {{redis_host}}:{{redis_port}}
{% if sentinel_master_set %}
# sentinel hosts with comma
addr: {{redis_host}}
sentinelMasterSet: {{sentinel_master_set}}
{% else %}
addr: {{redis_host}}
{% endif %}
password: {{redis_password}}
db: {{redis_db_index_reg}}
http:
......
SCANNER_LOG_LEVEL={{log_level}}
SCANNER_REDIS_URL={{trivy_redis_url}}
SCANNER_STORE_REDIS_URL={{trivy_redis_url}}
SCANNER_STORE_REDIS_NAMESPACE=harbor.scanner.trivy:store
SCANNER_JOB_QUEUE_REDIS_URL={{trivy_redis_url}}
......
import os, shutil
import os
from urllib.parse import urlsplit
from g import templates_dir, config_dir, data_dir, DEFAULT_UID, DEFAULT_GID
from .jinja import render_jinja
from .misc import prepare_dir
......@@ -12,12 +13,29 @@ chart_museum_env = os.path.join(config_dir, "chartserver", "env")
chart_museum_data_dir = os.path.join(data_dir, 'chart_storage')
def prepare_chartmuseum(config_dict):
redis_host = config_dict['redis_host']
redis_port = config_dict['redis_port']
redis_password = config_dict['redis_password']
redis_db_index_chart = config_dict['redis_db_index_chart']
def parse_redis(redis_url_chart):
u = urlsplit(redis_url_chart)
if not u.scheme or u.scheme == 'redis':
return {
'cache_store': 'redis',
'cache_redis_addr': u.netloc.split('@')[-1],
'cache_redis_password': u.password or '',
'cache_redis_db_index': u.path and int(u.path[1:]) or 0,
}
elif u.scheme == 'redis+sentinel':
return {
'cache_store': 'redis_sentinel',
'cache_redis_mastername': u.path.split('/')[1],
'cache_redis_addr': u.netloc.split('@')[-1],
'cache_redis_password': u.password or '',
'cache_redis_db_index': len(u.path.split('/')) == 3 and int(u.path.split('/')[2]) or 0,
}
else:
raise Exception('bad redis url for chart:' + redis_url_chart)
def prepare_chartmuseum(config_dict):
storage_provider_name = config_dict['storage_provider_name']
storage_provider_config_map = config_dict['storage_provider_config']
......@@ -25,10 +43,7 @@ def prepare_chartmuseum(config_dict):
prepare_dir(chart_museum_config_dir)
# process redis info
cache_store = "redis"
cache_redis_password = redis_password
cache_redis_addr = "{}:{}".format(redis_host, redis_port)
cache_redis_db_index = redis_db_index_chart
cache_redis_ops = parse_redis(config_dict['redis_url_chart'])
# process storage info
......@@ -85,8 +100,10 @@ def prepare_chartmuseum(config_dict):
storage_provider_config_options.append("STORAGE_ALIBABA_BUCKET=%s" % bucket )
storage_provider_config_options.append("STORAGE_ALIBABA_ENDPOINT=%s" % endpoint )
storage_provider_config_options.append("STORAGE_ALIBABA_PREFIX=%s" % ( storage_provider_config_map.get("rootdirectory") or '') )
storage_provider_config_options.append("ALIBABA_CLOUD_ACCESS_KEY_ID=%s" % ( storage_provider_config_map.get("accesskeyid") or '') )
storage_provider_config_options.append("ALIBABA_CLOUD_ACCESS_KEY_SECRET=%s" % ( storage_provider_config_map.get("accesskeysecret") or '') )
storage_provider_config_options.append(
"ALIBABA_CLOUD_ACCESS_KEY_ID=%s" % (storage_provider_config_map.get("accesskeyid") or ''))
storage_provider_config_options.append(
"ALIBABA_CLOUD_ACCESS_KEY_SECRET=%s" % (storage_provider_config_map.get("accesskeysecret") or ''))
else:
# use local file system
storage_provider_config_options.append("STORAGE_LOCAL_ROOTDIR=/chart_storage")
......@@ -95,15 +112,11 @@ def prepare_chartmuseum(config_dict):
all_storage_provider_configs = ('\n').join(storage_provider_config_options)
render_jinja(
chart_museum_env_temp,
chart_museum_env,
cache_store=cache_store,
cache_redis_addr=cache_redis_addr,
cache_redis_password=cache_redis_password,
cache_redis_db_index=cache_redis_db_index,
core_secret=config_dict['core_secret'],
storage_driver=storage_driver,
all_storage_driver_configs=all_storage_provider_configs,
public_url=config_dict['public_url'],
chart_absolute_url=config_dict['chart_absolute_url'],
internal_tls=config_dict['internal_tls'])
\ No newline at end of file
chart_museum_env_temp,
chart_museum_env,
storage_driver=storage_driver,
all_storage_driver_configs=all_storage_provider_configs,
public_url=config_dict['public_url'],
chart_absolute_url=config_dict['chart_absolute_url'],
internal_tls=config_dict['internal_tls'],
**cache_redis_ops)
import logging
import os
import yaml
import logging
from models import InternalTLS
from urllib.parse import urlencode
from g import versions_file_path, host_root_dir, DEFAULT_UID, INTERNAL_NO_PROXY_DN
from models import InternalTLS
from utils.misc import generate_random_string, owner_can_read, other_can_read
default_db_max_idle_conns = 2 # NOTE: https://golang.org/pkg/database/sql/#DB.SetMaxIdleConns
......@@ -73,17 +73,6 @@ def validate(conf: dict, **kwargs):
if uid != DEFAULT_UID and not other_can_read(st_mode):
raise Exception(err_msg)
# Redis validate
redis_host = conf.get("redis_host")
if redis_host is None or len(redis_host) < 1:
raise Exception(
"Error: redis_host in harbor.yml needs to point to an endpoint of Redis server or cluster.")
redis_port = conf.get("redis_port")
if redis_host is None or (redis_port < 1 or redis_port > 65535):
raise Exception(
"Error: redis_port in harbor.yml needs to point to the port of Redis server or cluster.")
# TODO:
# If user enable trust cert dir, need check if the files in this dir is readable.
......@@ -372,24 +361,38 @@ def parse_yaml_config(config_file_path, with_notary, with_clair, with_trivy, wit
def get_redis_url(db, redis=None):
"""Returns redis url with format `redis://[arbitrary_username:password@]ipaddress:port/database_index`
"""Returns redis url with format `redis://[arbitrary_username:password@]ipaddress:port/database_index?idle_timeout_seconds=30`
>>> get_redis_url(1)
'redis://redis:6379/1'
>>> get_redis_url(1, {'host': 'localhost', 'password': 'password'})
>>> get_redis_url(1, {'host': 'localhost:6379', 'password': 'password'})
'redis://anonymous:password@localhost:6379/1'
>>> get_redis_url(1, {'host':'host1:26379,host2:26379', 'sentinel_master_set':'mymaster', 'password':'password1'})
'redis+sentinel://anonymous:password@host1:26379,host2:26379/mymaster/1'
>>> get_redis_url(1, {'host':'host1:26379,host2:26379', 'sentinel_master_set':'mymaster', 'password':'password1','idle_timeout_seconds':30})
'redis+sentinel://anonymous:password@host1:26379,host2:26379/mymaster/1?idle_timeout_seconds=30'
"""
kwargs = {
'host': 'redis',
'port': 6379,
'host': 'redis:6379',
'password': '',
}
kwargs.update(redis or {})
kwargs['db'] = db
kwargs['scheme'] = kwargs.get('sentinel_master_set', None) and 'redis+sentinel' or 'redis'
kwargs['db_part'] = db and ("/%s" % db) or ""
kwargs['sentinel_part'] = kwargs.get('sentinel_master_set', None) and ("/" + kwargs['sentinel_master_set']) or ''
kwargs['password_part'] = kwargs.get('password', None) and (':%s@' % kwargs['password']) or ''
if kwargs['password']:
return "redis://anonymous:{password}@{host}:{port}/{db}".format(**kwargs)
return "redis://{host}:{port}/{db}".format(**kwargs)
return "{scheme}://{password_part}{host}{sentinel_part}{db_part}".format(**kwargs) + get_redis_url_param(kwargs)
def get_redis_url_param(redis=None):
params = {}
if redis and 'idle_timeout_seconds' in redis:
params['idle_timeout_seconds'] = redis['idle_timeout_seconds']
if params:
return "?" + urlencode(params)
return ""
def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True):
......@@ -437,8 +440,7 @@ def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True):
# internal redis config as the default
redis = {
'host': 'redis',
'port': 6379,
'host': 'redis:6379',
'password': '',
'registry_db_index': 1,
'jobservice_db_index': 2,
......@@ -451,23 +453,15 @@ def get_redis_configs(external_redis=None, with_clair=True, with_trivy=True):
# overwriting existing keys by external_redis
redis.update({key: value for (key, value) in external_redis.items() if value})
configs['redis_host'] = redis['host']
configs['redis_port'] = redis['port']
configs['redis_password'] = redis['password']
configs['redis_db_index_reg'] = redis['registry_db_index']
configs['redis_db_index_js'] = redis['jobservice_db_index']
configs['redis_db_index_chart'] = redis['chartmuseum_db_index']
configs['redis_idle_timeout_seconds'] = redis['idle_timeout_seconds']
configs['redis_url_js'] = get_redis_url(configs['redis_db_index_js'], redis)
configs['redis_url_reg'] = get_redis_url(configs['redis_db_index_reg'], redis)
configs['redis_url_core'] = get_redis_url(0, redis)
configs['redis_url_chart'] = get_redis_url(redis['chartmuseum_db_index'], redis)
configs['redis_url_js'] = get_redis_url(redis['jobservice_db_index'], redis)
configs['redis_url_reg'] = get_redis_url(redis['registry_db_index'], redis)
if with_clair:
configs['redis_db_index_clair'] = redis['clair_db_index']
configs['redis_url_clair'] = get_redis_url(configs['redis_db_index_clair'], redis)
configs['redis_url_clair'] = get_redis_url(redis['clair_db_index'], redis)
if with_trivy:
configs['redis_db_index_trivy'] = redis['trivy_db_index']
configs['trivy_redis_url'] = get_redis_url(configs['redis_db_index_trivy'], redis)
configs['trivy_redis_url'] = get_redis_url(redis['trivy_db_index'], redis)
return configs
import shutil, os
import os
import shutil
from g import config_dir, templates_dir, data_dir, DEFAULT_GID, DEFAULT_UID
from utils.misc import prepare_dir, generate_random_string
from utils.jinja import render_jinja
from utils.misc import prepare_dir, generate_random_string
core_config_dir = os.path.join(config_dir, "core", "certificates")
core_env_template_path = os.path.join(templates_dir, "core", "env.jinja")
......@@ -19,7 +19,7 @@ def prepare_core(config_dict, with_notary, with_clair, with_trivy, with_chartmus
# Render Core
# set cache for chart repo server
# default set 'memory' mode, if redis is configured then set to 'redis'
if len(config_dict['redis_host']) > 0:
if len(config_dict['redis_url_core']) > 0:
chart_cache_driver = "redis"
else:
chart_cache_driver = "memory"
......
import os, string, sys
import os
import secrets
import string
import sys
from pathlib import Path
from functools import wraps
from g import DEFAULT_UID, DEFAULT_GID, host_root_dir
# To meet security requirement
# By default it will change file mode to 0600, and make the owner of the file to 10000:10000
def mark_file(path, mode=0o600, uid=DEFAULT_UID, gid=DEFAULT_GID):
......@@ -52,22 +54,6 @@ def validate(conf, **kwargs):
raise Exception(
"Error: no provider configurations are provided for provider %s" % storage_provider_name)
# Redis validate
redis_host = conf.get("configuration", "redis_host")
if redis_host is None or len(redis_host) < 1:
raise Exception(
"Error: redis_host in harbor.yml needs to point to an endpoint of Redis server or cluster.")
redis_port = conf.get("configuration", "redis_port")
if len(redis_port) < 1:
raise Exception(
"Error: redis_port in harbor.yml needs to point to the port of Redis server or cluster.")
redis_db_index = conf.get("configuration", "redis_db_index").strip()
if len(redis_db_index.split(",")) != 3:
raise Exception(
"Error invalid value for redis_db_index: %s. please set it as 1,2,3" % redis_db_index)
def validate_crt_subj(dirty_subj):
subj_list = [item for item in dirty_subj.strip().split("/") \
if len(item.split("=")) == 2 and len(item.split("=")[1]) > 0]
......
import os, copy, subprocess
import copy
import os
import subprocess
from g import config_dir, templates_dir, DEFAULT_GID, DEFAULT_UID, data_dir
from utils.misc import prepare_dir
from urllib.parse import urlsplit
from utils.jinja import render_jinja
from utils.misc import prepare_dir
registry_config_dir = os.path.join(config_dir, "registry")
registry_config_template_path = os.path.join(templates_dir, "registry", "config.yml.jinja")
......@@ -26,8 +27,11 @@ def prepare_registry(config_dict):
gen_passwd_file(config_dict)
storage_provider_info = get_storage_provider_info(
config_dict['storage_provider_name'],
config_dict['storage_provider_config'])
config_dict['storage_provider_name'],
config_dict['storage_provider_config'])
# process redis info
redis_ops = parse_redis(config_dict['redis_url_reg'])
render_jinja(
registry_config_template_path,
......@@ -36,8 +40,26 @@ def prepare_registry(config_dict):
gid=DEFAULT_GID,
level=levels_map[config_dict['log_level']],
storage_provider_info=storage_provider_info,
**config_dict)
**config_dict, **redis_ops)
def parse_redis(redis_url):
u = urlsplit(redis_url)
if not u.scheme or u.scheme == 'redis':
return {
'redis_host': u.netloc.split('@')[-1],
'redis_password': u.password or '',
'redis_db_index_reg': u.path and int(u.path[1:]) or 0,
}
elif u.scheme == 'redis+sentinel':
return {
'sentinel_master_set': u.path.split('/')[1],
'redis_host': u.netloc.split('@')[-1],
'redis_password': u.password or '',
'redis_db_index_reg': len(u.path.split('/')) == 3 and int(u.path.split('/')[2]) or 0,
}
else: