Enhance: Support multi downversion in migration

1. Change down version to list to accept multi verstion value
2. Update search function use BFS to find migration path
2. Add test case
Signed-off-by: DQ <dengq@vmware.com>

make/photon/prepare/commands/migrate.py

@@ -14,6 +14,9 @@ from migrations import accept_versions
 def migrate(input_, output, target):
     """
     migrate command will migrate config file style to specific version
+    :input_: is the path of the original config file
+    :output: is the destination path of config file, the generated configs will storage in it
+    :target: is the the target version of config file will upgrade to
     """
     if target not in accept_versions:
         click.echo('target version {} not supported'.format(target))

make/photon/prepare/migrations/version_1_10_0/__init__.py

@@ -3,7 +3,7 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined
 from utils.migration import read_conf
 
 revision = '1.10.0'
-down_revision = '1.9.0'
+down_revisions = ['1.9.0']
 
 def migrate(input_cfg, output_cfg):
     config_dict = read_conf(input_cfg)

make/photon/prepare/migrations/version_1_9_0/__init__.py

@@ -3,14 +3,14 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined
 from utils.migration import read_conf
 
 revision = '1.9.0'
-down_revision = None
+down_revisions = []
 
 def migrate(input_cfg, output_cfg):
     config_dict = read_conf(input_cfg)
 
-    this_dir = os.path.dirname(__file__)
+    current_dir = os.path.dirname(__file__)
     tpl = Environment(
-        loader=FileSystemLoader(this_dir),
+        loader=FileSystemLoader(current_dir),
         undefined=StrictUndefined,
         trim_blocks=True,
         lstrip_blocks=True

make/photon/prepare/migrations/version_1_9_0/harbor.yml.jinja

@@ -48,7 +48,7 @@ harbor_admin_password: {{ harbor_admin_password }}
 # Harbor DB configuration
 database:
   # The password for the root user of Harbor DB. Change this before any production use.
-  password: {{ database.password}}
+  password: {{ database.password }}
   # The maximum number of connections in the idle connection pool. If it <=0, no idle connections are retained.
   max_idle_conns: 50
   # The maximum number of open connections to the database. If it <= 0, then there is no limit on the number of open connections.

make/photon/prepare/migrations/version_2_0_0/__init__.py

@@ -3,7 +3,7 @@ from jinja2 import Environment, FileSystemLoader, StrictUndefined
 from utils.migration import read_conf
 
 revision = '2.0.0'
-down_revision = '1.10.0'
+down_revisions = ['1.10.0']
 
 def migrate(input_cfg, output_cfg):
     config_dict = read_conf(input_cfg)

make/photon/prepare/tests/migrations/utils_test.py

 import pytest
 import importlib
 
-from utils.migration import search
+from utils.migration import search, MigratioNotFound
 
 class mockModule:
-    def __init__(self, revision, down_revision):
+    def __init__(self, revision: str, down_revisions: list):
         self.revision = revision
-        self.down_revision = down_revision
+        self.down_revisions = down_revisions
 
 def mock_import_module_loop(module_path: str):
-    loop_modules = {
-    'migration.versions.1_9_0': mockModule('1.9.0', None),
-    'migration.versions.1_10_0': mockModule('1.10.0', '2.0.0'),
-    'migration.versions.2_0_0': mockModule('2.0.0', '1.10.0')
+    modules = {
+    'migrations.version_1_9_0': mockModule('1.9.0', []),
+    'migrations.version_1_10_0': mockModule('1.10.0', ['2.0.0']),
+    'migrations.version_2_0_0': mockModule('2.0.0', ['1.10.0'])
     }
-    return loop_modules[module_path]
+    return modules[module_path]
 
 def mock_import_module_mission(module_path: str):
-    loop_modules = {
-    'migration.versions.1_9_0': mockModule('1.9.0', None),
-    'migration.versions.1_10_0': mockModule('1.10.0', None),
-    'migration.versions.2_0_0': mockModule('2.0.0', '1.10.0')
+    modules = {
+    'migrations.version_1_9_0': mockModule('1.9.0', []),
+    'migrations.version_1_10_0': mockModule('1.10.0', []),
+    'migrations.version_2_0_0': mockModule('2.0.0', ['1.10.0'])
     }
-    return loop_modules[module_path]
+    return modules[module_path]
 
 def mock_import_module_success(module_path: str):
-    loop_modules = {
-    'migration.versions.1_9_0': mockModule('1.9.0', None),
-    'migration.versions.1_10_0': mockModule('1.10.0', '1.9.0'),
-    'migration.versions.2_0_0': mockModule('2.0.0', '1.10.0')
+    modules = {
+    'migrations.version_1_9_0': mockModule('1.9.0', []),
+    'migrations.version_1_10_0': mockModule('1.10.0', ['1.9.0']),
+    'migrations.version_2_0_0': mockModule('2.0.0', ['1.10.0'])
     }
-    return loop_modules[module_path]
+    return modules[module_path]
+
+def mock_import_module_success_multi_downversion(module_path: str):
+    modules = {
+    'migrations.version_1_9_0': mockModule('1.9.0', []),
+    'migrations.version_1_10_0': mockModule('1.10.0', ['1.9.0']),
+    'migrations.version_1_10_1': mockModule('1.10.1', ['1.9.0']),
+    'migrations.version_1_10_2': mockModule('1.10.2', ['1.9.0']),
+    'migrations.version_2_0_0': mockModule('2.0.0', ['1.10.0', '1.10.1', '1.10.2'])
+    }
+    return modules[module_path]
 
 @pytest.fixture
 def mock_import_module_with_loop(monkeypatch):
@@ -44,15 +54,25 @@ def mock_import_module_with_mission(monkeypatch):
 def mock_import_module_with_success(monkeypatch):
     monkeypatch.setattr(importlib, "import_module", mock_import_module_success)
 
+@pytest.fixture
+def mock_import_module_with_success_multi_downversion(monkeypatch):
+    monkeypatch.setattr(importlib, "import_module", mock_import_module_success_multi_downversion)
+
 def test_search_loop(mock_import_module_with_loop):
     with pytest.raises(Exception):
         search('1.9.0', '2.0.0')
 
 def test_search_mission(mock_import_module_with_mission):
-    with pytest.raises(Exception):
+    with pytest.raises(MigratioNotFound):
         search('1.9.0', '2.0.0')
 
-def test_search_success():
+def test_search_success(mock_import_module_with_success):
     migration_path = search('1.9.0', '2.0.0')
     assert migration_path[0].revision == '1.10.0'
     assert migration_path[1].revision == '2.0.0'
+
+def test_search_success_multi_downversion(mock_import_module_with_success_multi_downversion):
+    migration_path = search('1.9.0', '2.0.0')
+    print(migration_path)
+    assert migration_path[0].revision == '1.10.2'
+    assert migration_path[1].revision == '2.0.0'

make/photon/prepare/utils/migration.py

@@ -4,7 +4,24 @@ import importlib
 import os
 from collections import deque
 
-from migrations import MIGRATION_BASE_DIR
+class MigratioNotFound(Exception): ...
+
+class MigrationVersion:
+    '''
+    The version used to migration
+
+    Arttribute:
+        name(str): version name like `1.0.0`
+        module: the python module object for a specific migration which contains migrate info, codes and templates
+        down_versions(list): previous versions that can migrated to this version
+    '''
+    def __init__(self, version: str):
+        self.name = version
+        self.module = importlib.import_module("migrations.version_{}".format(version.replace(".","_")))
+
+    @property
+    def down_versions(self):
+        return self.module.down_revisions
 
 def read_conf(path):
     with open(path) as f:
@@ -15,29 +32,35 @@ def read_conf(path):
             exit(-1)
     return d
 
-def _to_module_path(ver):
-    return "migrations.version_{}".format(ver.replace(".","_"))
-
-def search(input_ver: str, target_ver: str) -> deque :
+def search(input_version: str, target_version: str) -> list :
     """
-    Search accept a input version and the target version.
-    Returns the module of migrations in the upgrade path
+    Find the migration path by BFS
+    Args:
+        input_version(str): The version migration start from
+        target_version(str): The target version migrated to
+    Returns:
+        list: the module of migrations in the upgrade path
     """
-    upgrade_path, visited = deque(), set()
-    while True:
-        module_path = _to_module_path(target_ver)
-        visited.add(target_ver)  # mark current version for loop finding
-        if os.path.isdir(os.path.join(MIGRATION_BASE_DIR, 'version_{}'.format(target_ver.replace(".","_")))):
-            module = importlib.import_module(module_path)
-            if module.revision == input_ver:    # migration path found
-                break
-            elif module.down_revision is None:  # migration path not found
-                raise Exception('no migration path found')
-            else:
-                upgrade_path.appendleft(module)
-                target_ver = module.down_revision
-                if target_ver in visited: # version visited before, loop found
-                    raise Exception('find a loop caused by {} on migration path'.format(target_ver))
-        else:
-            raise Exception('{} not dir'.format(os.path.join(MIGRATION_BASE_DIR, 'versions', target_ver.replace(".","_"))))
-    return upgrade_path
+    upgrade_path = []
+    next_version, visited, q = {}, set(), deque()
+    q.append(target_version)
+    found = False
+    while q: # BFS to find a valid path
+        version = MigrationVersion(q.popleft())
+        visited.add(version.name)
+        if version.name == input_version:
+            found = True
+            break # break loop cause migration path found
+        for v in version.down_versions:
+            next_version[v] = version.name
+            if v not in (visited.union(q)):
+                q.append(v)
+
+    if not found:
+        raise MigratioNotFound('no migration path found to target version')
+
+    current_version = MigrationVersion(input_version)
+    while current_version.name != target_version:
+        current_version = MigrationVersion(next_version[current_version.name])
+        upgrade_path.append(current_version)
+    return list(map(lambda x: x.module, upgrade_path))
\ No newline at end of file