Split `id_child.mat`

Integrating work done by Mevludin Isic. We add the possibility to
split `id_child` because it takes more than 2 GiB for the global
domain and thus we cannot save the whole variable to Matlab v6
format. We split `id_child` along the time dimension. As the Matlab
code for this is the same for anticylonic and cyclonic eddies, we
extract it from the script `overlap.m` to a new script
`id_child.m`.

Convert_Matlab/id_child.m

+% This is a Matlab script. This script converts to Matlab v6 format
+% some output of TOEddies: what is necessary for the script
+% overlap_v6.py.
+
+clear
+disp('Loading Association_eddies_max...')
+tic
+load('Association_eddies_max.mat', 'id_child')
+toc
+disp('Saving id_child...')
+n_dates = size(id_child, 2)
+n_files = 10
+chunck = ceil(n_dates / n_files)
+last_date = 0
+tic
+
+for i = 0:n_files - 2
+    first_date = last_date + 1;
+    last_date = last_date + chunck
+    part_id_child = id_child(:, first_date:last_date);
+    destination = strcat('id_child_', int2str(i));
+    save(destination, 'part_id_child', '-v6')
+end
+
+first_date = last_date + 1;
+part_id_child = id_child(:, first_date:end);
+destination = strcat('id_child_', int2str(n_files - 1));
+save(destination, 'part_id_child', '-v6')
+toc
+exit

Convert_Matlab/overlap.m

 % This is a Matlab script. This script converts to Matlab v6 format
-% some output of TOEddies: what is necessary for the script
+% some output of TOEddies: part of what is necessary for the script
 % overlap_v6.py.
 
-clear
-disp('Loading Association_eddies_Anti_max...')
-tic
-load('Association_eddies_Anti_max.mat', 'id_child')
-toc
-disp('Saving id_child...')
-tic
-save('id_child_anti', 'id_child', '-v6')
-toc
-
-clear
-disp('Loading Association_eddies_Cyclo_max...')
-tic
-load('Association_eddies_Cyclo_max.mat', 'id_child')
-toc
-disp('Saving id_child...')
-tic
-save('id_child_cyclo', 'id_child', '-v6')
-toc
-
-clear
 disp('Processing Parameters_Anticyclonic_Eddies...')
 load('Parameters_Anticyclonic_Eddies.mat', 'Nanti', 'date_num')
 N_eddies = Nanti;

Convert_Matlab/overlap_v6.py

 #!/usr/bin/env python3
 
-"""This script reads the eddy graph from Matlab v6 files id_child.mat
-    and N_eddies.mat, for a given orientation and writes the graph to
-    a new file in edgelist format.
+"""This script reads the eddy graph from Matlab v6 files
+    id_child_*.mat and N_eddies.mat, for a given orientation and
+    writes the graph to a new file in edgelist format.
 
 """
 
@@ -11,6 +11,7 @@ import csv
 import sys
 import json
 import datetime
+import numpy as np
 
 def to_days_1950(date_num):
     """Convert Matlab serial date number to number of days since Jan 1st
@@ -26,7 +27,26 @@ def adjust_n(n_Matlab):
     return n_Matlab + (k - k_Matlab) * e_overestim
 
 if __name__ == "__main__":
-    id_child = sio.loadmat("id_child.mat", squeeze_me=True)["id_child"]
+    n_files = input("Number of files = ? ")
+    n_files = int(n_files)
+    id_child_list = []
+
+    for i in range(n_files):
+        print(f"Loading id_child_{i}.mat...")
+        part_id_child \
+            = sio.loadmat(f"id_child_{i}.mat", squeeze_me=True)["part_id_child"]
+        # part_id_child is a numpy array with shape (e_overestim, chunk)
+        # where chunk is lower than or equal to the number of dates in the
+        # original Matlab files.
+
+        id_child_list.append(part_id_child)
+
+    if n_files >= 2:
+        id_child = np.concatenate(id_child_list, axis = 1)
+        del id_child_list, part_id_child # can be a lot of memory
+    else:
+        id_child = part_id_child
+
     n_eddies = sio.loadmat("N_eddies.mat", squeeze_me = True)["N_eddies"]\
                   .astype(int, casting = "safe", copy = False)
     date_num = sio.loadmat("date_num.mat", squeeze_me = True)["date_num"]