#!/usr/bin/python3
# -*- coding: utf-8 -*-

#  Copyright © 2012-2017  B. Clausius <barcc@gmx.de>
#
#  This program is free software: you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation, either version 3 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program.  If not, see <http://www.gnu.org/licenses/>.


import sys, os
sys.path.insert(0, '.')
import pickletools
from multiprocessing import Pool, cpu_count
from collections import defaultdict
from bisect import bisect_left
from array import array
from itertools import chain
from math import sin, cos, pi
from contextlib import suppress

from pybiklib.debug import DEBUG_MSG, DEBUG_INDEXONLY, DEBUG_MODELFAST, DEBUG_MAXSIZE5
from pybiklib.utils import epsilon, filebyteorder, get_texcoords_range
# executable script needs absolute imports
from buildlib.geom import roundeps, Vector
from buildlib.modeldef import modeldefs


minsize = 1
maxsize = 10
if DEBUG_MAXSIZE5:  maxsize = 5

dumps = None

def make_dumps(reproducible):
    global dumps
    import io
    import pickle
    if not reproducible:
        dumps = pickle.dumps
        return
    try:
        class _Pickler (pickle._Pickler):
            def save_dict(self, obj):
                if self.bin:
                    self.write(pickle.EMPTY_DICT)
                else:   # proto 0 -- can't use EMPTY_DICT
                    self.write(pickle.MARK + pickle.DICT)
                self.memoize(obj)
                self._batch_setitems(sorted(obj.items(), key=lambda v: str(v)))
                
            pickle._Pickler.dispatch[dict] = save_dict
            
        def _dumps(obj, protocol=None, *, fix_imports=True):
            f = io.BytesIO()
            try:
                _Pickler(f, protocol, fix_imports=fix_imports).dump(obj)
                return f.getvalue()
            except Exception:
                print('warning: using undocumented interface in module pickle failed, models will not be reproducible:')
                sys.excepthook(*sys.exc_info())
                return pickle.dumps(obj, protocol, fix_imports=fix_imports)
        _dumps({1:2, 3:4})
    except Exception:
        print('warning: using undocumented interface in module pickle failed, models will not be reproducible:')
        sys.excepthook(*sys.exc_info())
        _dumps = pickle.dumps
    dumps = _dumps
        
def equal_vector_fuzzy(v1, v2):
    for v1k, v2k in zip(v1, v2):
        if abs(v1k - v2k) > epsilon:
            return False
    return True
    
    
class VectorData:
    __slots__ = 'vectors origs indices sorted cnts dups toindex'.split()
    
    def __init__(self):
        self.vectors = []
        self.origs = {}
        self.indices = []
        self.sorted = []
        self.cnts = 0
        self.dups = 0
        self.toindex = self.toindex_fast if DEBUG_MODELFAST else self.toindex_dedup
        
    def toindex_dedup(self, ovector):
        with suppress(KeyError):
            return self.origs[ovector]
        vector = ovector.rounded()
        vector = [(0. if v == 0. else v) for v in vector]  # 0.0 == -0.0
        self.cnts += 1
        si = bisect_left(self.sorted, vector)
        i = len(self.vectors)
        if si < i and self.sorted[si] == vector:
            self.dups += 1
            i = self.indices[si]
        else:
            self.vectors.append(vector)
            self.sorted.insert(si, vector)
            self.indices.insert(si, i)
        self.origs[ovector] = i
        return i
        
    def toindex_fast(self, ovector):
        with suppress(KeyError):
            return self.origs[ovector]
        vector = ovector.rounded()
        i = len(self.vectors)
        self.vectors.append(vector)
        self.origs[ovector] = i
        return i
        
        
class ModelFactory:
    mtype_attributes = (
            'name', 'mformat', 'sizenames', 'defaultsize',
            'symmetries', 'axes', 'symbols', 'symbolsI', 'faces', 'facekeys',
            'normal_rotation_symbols', 'rotation_symbols', 'rotation_matrices',
            'face_permutations', 'default_rotation', 'reversepick', 'slicesmode')
        
    def __init__(self, modeldef):
        self.modeldef = modeldef()
        self.create_rotations()
        
    def __getattr__(self, attrname):
        return getattr(self.modeldef, attrname)
        
    def _getattr(self, attrname):
        value = getattr(self, attrname)
        if attrname == 'axes':
            return tuple(tuple(v) for v in value)
        return value
        
    def getattrs(self):
        return {attr: self._getattr(attr) for attr in self.mtype_attributes}
        
    @staticmethod
    def _matrix_equal(m1, m2):
        for line1, line2 in zip(m1, m2):
            for value1, value2 in zip(line1, line2):
                if abs(value1 - value2) > epsilon:
                    return False
        return True
        
    @staticmethod
    def _mult_matrix_vector3(matrix, vector):
        return Vector(sum(matrix[i][k]*vector[k] for k in range(3)) for i in range(3))
        
    @staticmethod
    def _mult_matrix(matrix1, matrix2):
        return [[sum(matrix1[i][k]*matrix2[k][j] for k in range(4)) for j in range(4)] for i in range(4)]
        
    @staticmethod
    def _create_rotation(axis, angle):
        angle = angle / 180. * pi
        sa = sin(angle)
        ca = cos(angle)
        e_ca = 1 - ca
        n1 = axis[0]
        n2 = axis[1]
        n3 = axis[2]
        m = [
            [n1*n1*e_ca + ca,    n1*n2*e_ca - n3*sa, n1*n3*e_ca + n2*sa, 0.],
            [n2*n1*e_ca + n3*sa, n2*n2*e_ca + ca,    n2*n3*e_ca - n1*sa, 0.],
            [n3*n1*e_ca - n2*sa, n3*n2*e_ca + n1*sa, n3*n3*e_ca + ca,    0.],
            [0.,    0.,     0.,     1.],
        ]
        #XXX: try to keep the matrix clean
        for y, line in enumerate(m):
            for x, value in enumerate(line):
                if abs(value) < epsilon:
                    m[y][x] = 0.
        return m
        
    def _create_permutation(self, matrix):
        permutation = {}
        for sym1, symI1, axis1 in zip(self.symbols, self.symbolsI, self.axes):
            for sym2, symI2, axis2 in zip(self.symbols, self.symbolsI, self.axes):
                axisR = self._mult_matrix_vector3(matrix, axis1)
                if axisR.equalfuzzy(axis2):
                    permutation[sym2] = sym1
                    permutation[symI2] = symI1
                elif axisR.inversfuzzy(axis2):
                    permutation[symI2] = sym1
                    permutation[sym2] = symI1
        return permutation
        
    def create_rotations(self):
        prim = []
        for axis, sym, symI, symmetry in zip(self.axes, self.symbols, self.symbolsI, self.symmetries):
            angle = 360. / symmetry
            # for some models (towers, bricks) rotations are equal to the invers rotations,
            # but the symbols are different
            prim.append((sym, self._create_rotation(axis, angle)))
            prim.append((symI, self._create_rotation(axis, -angle)))
        self.normal_rotation_symbols = {'': ''}
        transform = [['', [[1.,0.,0.,0.],[0.,1.,0.,0.],[0.,0.,1.,0.],[0.,0.,0.,1.]]]]
        for sp, p in prim:
            transform.append([sp, p])
        for sm, m in transform:
            for sp, p in prim:
                n = self._mult_matrix(m, p)
                sn = sm + sp
                for st, t in transform:
                    if self._matrix_equal(t, n):
                        self.normal_rotation_symbols[sn] = st
                        break
                else:
                    self.normal_rotation_symbols[sn] = sn
                    transform.append([sn, n])
        self.rotation_symbols = [s for s, m in transform]
        self.rotation_matrices = [m for s, m in transform]
        self.face_permutations = {s: self._create_permutation(m) for s, m in transform}
        
    def get_rotated_position(self, cells):
        rotated_position = {}
        centers = [c.center() for c in cells]
        for b, center in enumerate(centers):
            for sym, rotation in zip(self.rotation_symbols, self.rotation_matrices):
                coords = self._mult_matrix_vector3(rotation, center)
                for p, center2 in enumerate(centers):
                    if equal_vector_fuzzy(center2, coords):
                        if sym not in rotated_position:
                            rotated_position[sym] = [0] * len(cells)
                        rotated_position[sym][p] = b
                        break
                else:
                    assert False, 'not a permutation'
        return rotated_position
        
    def get_data(self, sizes, vectordata):
        polys = self.create_cells(sizes)
        
        data_cells_visible_faces = [[f.id for f in cell.faces if f.type == 'face'] for cell in polys.cells]
        polys_verts = [v.point for v in polys.verts]
        data_texranges_mosaic = [list(get_texcoords_range(polys_verts, self.normals[sym])) for sym in self.faces]
        data_rotated_position = self.get_rotated_position(polys.cells)
        data_cell_indices = [cell.indices for cell in polys.cells]
        data_cell_centers = [vectordata.toindex(cell.center()) for cell in polys.cells]
        data_pick_polygons = list(self.gl_pick_polygons(polys.cells, vectordata))
        blocksdata = self.gl_block_data(polys, vectordata)
        
        return {
                'rotated_position': data_rotated_position,
                'cell_indices': data_cell_indices,
                'cell_centers': data_cell_centers,
                'cells_visible_faces': data_cells_visible_faces,
                'texranges_mosaic': data_texranges_mosaic,
                'blocksdata': blocksdata,
                'facesdata': data_pick_polygons,
            }
        
    def gl_block_data(self, polys, vectordata):
        polys_faces = polys.faces
        def gl_block_data_cell(cell):
            def is_visible(vert):
                for hv in vert.halfverts:
                    he = hv.halfedge
                    hf = he.halfface
                    if hv is he.halfverts[1] and hf.hdim is cell and not hf.face.type.endswith('_removed'):
                        return True
                return False
            def gen_cedges(cell_halffaces):
                edgesdone = []
                for hf in cell_halffaces:
                    for he1 in hf.halfedges:
                        if he1.edge not in edgesdone:
                            he3 = he1.other
                            hf3_face = he3.halfface.face
                            edgesdone.append(he1.edge)
                            if not (hf.face.type.endswith('_removed') and hf3_face.type.endswith('_removed')):
                                v1_point = vectordata.toindex(he1.verts[1].point)
                                v3_point = vectordata.toindex(he3.verts[1].point)
                                yield polys_faces.index(hf.face), polys_faces.index(hf3_face), v1_point, v3_point
            def gen_vhalfface(halfface, vert):
                for hv in vert.halfverts:
                    he = hv.halfedge
                    hf = he.halfface
                    if hv is he.halfverts[1] and hf is halfface:
                        break
                else:
                    assert False
                while True:
                    assert hv.vert is vert
                    yield polys_faces.index(hf.face)
                    he = hv.halfedge.other
                    hv = he.halfverts[0].other
                    assert hv.vert is vert
                    hf = he.halfface
                    if hf is halfface:
                        break
            def gen_cverts(cell_halffaces):
                vertsdone = []
                for hf in cell_halffaces:
                    for he1 in hf.halfedges:
                        he3 = he1.other
                        hf3 = he3.halfface
                        v1 = he1.verts[1]
                        v3 = he3.verts[1]
                        if v1 not in vertsdone:
                            vertsdone.append(v1)
                            if is_visible(v1):
                                vertdata = [f for f in gen_vhalfface(hf, v1)]
                                yield vectordata.toindex(v1.point), vertdata
                        if v3 not in vertsdone:
                            vertsdone.append(v3)
                            if is_visible(v3):
                                vertdata = [f for f in gen_vhalfface(hf3, v3)]
                                yield vectordata.toindex(v3.point), vertdata
            def gen_cfaces(cell_halffaces):
                c_faces_label = []
                c_faces_black = []
                for hf in cell_halffaces:
                    verts = [vectordata.toindex(v.point) for v in hf.verts]
                    if hf.face.type == 'face':
                        c_faces_label.append((polys_faces.index(hf.face), self.faces.index(hf.face.id), verts))
                    elif hf.face.type == 'cut':
                        c_faces_black.append((polys_faces.index(hf.face), verts))
                    else:
                        assert hf.face.type.endswith('_removed')
                return c_faces_label, c_faces_black
            c_faces_label, c_faces_black = list(gen_cfaces(cell.halffaces))
            c_edges = list(gen_cedges(cell.halffaces))
            c_verts = list(gen_cverts(cell.halffaces))
            return c_faces_label, c_faces_black, c_edges, c_verts
        return [gl_block_data_cell(cell) for cell in polys.cells]
        
    def gl_pick_polygons(self, cells, vectordata):
        black_faces = []
        for cellidx, cell in enumerate(cells):
            cell_facesdata = []
            for halfface in cell.halffaces:
                if halfface.face.type == 'face':
                    symbol = halfface.face.id
                    face = self.faces.index(symbol)
                    visible_face_indices = [i for i, he in enumerate(halfface.halfedges)
                                          if he.other.halfface.face.type == 'face']
                else:
                    if halfface.face in black_faces:
                        continue
                    black_faces.append(halfface.face)
                    face = halfface.face.id
                edges_iverts = [vectordata.toindex(he.verts[0].point) for he in halfface.halfedges]
                if halfface.face.type != 'face':
                    cell_facesdata.append([face, None, edges_iverts])
                    continue
                if len(visible_face_indices) == 0:
                    picktype = 0
                elif len(visible_face_indices) == 1:
                    if len(edges_iverts) == 3:
                        picktype = 1
                    elif len(edges_iverts) == 4:
                        picktype = 2
                    elif len(edges_iverts) == 6:
                        picktype = 4
                    else:
                        assert False, len(edges_iverts)
                    i = visible_face_indices[0]
                    edges_iverts = edges_iverts[i:] + edges_iverts[:i]
                elif len(visible_face_indices) == 2:
                    assert 3 <= len(edges_iverts) <= 5, len(edges_iverts)
                    i = visible_face_indices[0]
                    if len(edges_iverts) < 5:
                        picktype = 3
                    else:
                        picktype = 5
                    if i+1 == visible_face_indices[1]:
                        edges_iverts = edges_iverts[i:] + edges_iverts[:i]
                    elif visible_face_indices == [0, len(edges_iverts)-1]:
                        i = visible_face_indices[1]
                        edges_iverts = edges_iverts[i:] + edges_iverts[:i]
                    else:
                        picktype = -1
                else:
                    picktype = -1
                cell_facesdata.append([face, picktype, edges_iverts])
            yield cell_facesdata
                

class Dedup:
    def __init__(self):
        self.dedup_data = defaultdict(list)
        self.cnt_dups = 0
        self.cnt_values = 0
        
    def _float(self, value):
        self.cnt_values += 1
        dddlt = self.dedup_data['f']
        value = roundeps(value)
        if value == 0.0:  # 0.0 == -0.0
            value = 0.0  # for reproducible build
        didx = bisect_left(dddlt, value)
        if didx < len(dddlt) and dddlt[didx] == value:
            self.cnt_dups += 1
            return dddlt[didx], True, 'f'
        else:
            dddlt.insert(didx, value)
            return value, False, 'f'
            
    def _recursion(self, value, iterable):
        dedup = True
        vtypes = []
        for k, v in iterable:
            v, d, t = self.dedup(v)
            dedup = dedup and d
            vtypes.append(t)
            value[k] = v
        return dedup, '('+''.join(vtypes)+')'
        
    def _replace(self, value, dedup, vtype):
        self.cnt_values += 1
        dddlt = self.dedup_data[vtype]
        try:
            return self._replace_bisect(dddlt, value, dedup, vtype)
        except TypeError:
            return self._replace_index(dddlt, value, dedup, vtype)
            
    def _replace_index(self, dddlt, value, dedup, vtype):
        if not dedup:
            #assert value not in dddlt
            dddlt.append(value)
            return value, False, vtype
        try:
            didx = dddlt.index(value)
        except ValueError:
            dddlt.append(value)
            return value, False, vtype
        else:
            dvalue = dddlt[didx]
            self.cnt_dups += 1
            return dvalue, True, vtype
        
    def _replace_bisect(self, dddlt, value, unused_dedup, vtype):
        didx = bisect_left(dddlt, value)
        try:
            dvalue = dddlt[didx]
        except IndexError:
            dddlt.append(value)
            return value, False, vtype
        if value == dvalue:
            self.cnt_dups += 1
            return dvalue, True, vtype
        else:
            dddlt.insert(didx, value)
            return value, False, vtype
            
    def dedup(self, value):
        if type(value) is dict:
            value = {(sys.intern(k) if type(k) is str else k):v for k,v in value.items()}
            dedup, vtype = self._recursion(value, value.items())
            return self._replace(value, dedup, 'd' + vtype)
        elif type(value) is list:
            dedup, vtype = self._recursion(value, enumerate(value))
            return self._replace(tuple(value), dedup, 'a' + vtype)
        elif type(value) is tuple:
            value = list(value)
            dedup, vtype = self._recursion(value, enumerate(value))
            return self._replace(tuple(value), dedup, 'a' + vtype)
        elif type(value) is float:
            return self._float(value)
        elif type(value) is int:
            return value, True, 'i'
        elif type(value) is str:
            return sys.intern(value), True, 's'
        elif type(value) is bool:
            return value, True, 'b'
        elif value is None:
            return value, True, 'n'
        else:
            assert False, type(value)
            
def pool_functions(parallel, reproducible):
    pool = None
    if reproducible:
        print('warning: using undocumented interface in module pickle for reproducible build')
    try:
        if parallel > 1:
            pool = Pool(processes=parallel, initializer=make_dumps, initargs=[reproducible])
    except OSError as e:
        print('process pool not available ({}):'.format(e))
        print('  deactivating multiprocessing')
    sys.stdout.flush() # when Pool(…) fails this line is 
    if pool is not None:
        return pool.imap_unordered
    else:
        make_dumps(reproducible)
        return map
        
#XXX: speedup parallel builds, longer jobs first
fileorder = 'b111 d000000 b101 b000 b010 p111111 p101111 b100 t1111 b110 p010000 b011 p000000 t0000 b001 t1011 t0100'.split()

def get_datafilename(Factory, sizes):
    if DEBUG_MODELFAST:
        return 'f{:02}{}{}'.format(modeldefs.index(Factory)+1, Factory.fileclass, ''.join(str(s-1) for s in sizes))
    else:
        sizes = Factory.fileclass + ''.join(str((s-1)%2) for s in sizes)
        try:
            priority = fileorder.index(sizes) + 1
        except ValueError:
            priority = 99
        return 'd{:02}'.format(priority) + sizes
        
def enum_modelfiles(Factory):
    def tuples(maxlen, part=()):
        if maxlen == len(part):
            yield part
        else:
            for i in range(minsize, maxsize+1):
                yield from tuples(maxlen, part+(i,))
    for size in tuples(len(Factory.sizenames)):
        norm_size, sizes = Factory.norm_sizes(size)
        if sizes is None:
            continue
        filename = get_datafilename(Factory, sizes)
        yield filename, sizes, size, norm_size
            
def pool_enum_modelfiles(dirname, testfunc):
    ignored = []
    modelfiles = []
    for Factory in modeldefs:
        for filename, *unused in enum_modelfiles(Factory):
            filename = os.path.join(dirname, filename)
            if filename in modelfiles or filename in ignored:
                continue
            if testfunc(filename):
                modelfiles.append(filename)
            else:
                ignored.append(filename)
    if modelfiles:
        for filename in sorted(ignored):
            print('skipping', filename)
    return sorted(modelfiles)
    
def pool_create_modelfiledata(path):
    filename = os.path.basename(path)
    savedata = {}
    vectordata = VectorData()
    for Factory in modeldefs:
        sizes_list = []
        factory = None
        for _filename, sizes, *unused in enum_modelfiles(Factory):
            if _filename != filename:
                continue
            if sizes in sizes_list:
                continue
            sizes_list.append(sizes)
            if factory is None:
                factory = ModelFactory(Factory)
            savedata.setdefault(factory.type, {})[sizes] = factory.get_data(sizes, vectordata)
    return savedata, vectordata.vectors, vectordata.cnts, vectordata.dups
    
def format_seconds(seconds):
    if seconds > 60:
        return '{:.0f}m {:.2f}s'.format(*divmod(seconds, 60))
    else:
        return '{:.2f}s'.format(seconds)
    
def pool_check_compare_modelfiledata(check_savedata, check_vectors, savedata, vectors):
    if savedata == check_savedata and vectors == check_vectors:
        return 'check: pass full'
        
    for mtype, check_value_mtype in list(check_savedata.items()):
        if mtype not in savedata:
            del check_savedata[mtype]
        else:
            value_mtype = savedata[mtype]
            for sizes, check_value_sizes in list(check_value_mtype.items()):
                if sizes not in value_mtype:
                    del check_value_mtype[sizes]
                else:
                    value_sizes = value_mtype[sizes]
                    for attr in 'block_polygons', 'pick_polygons':  # older versions may have this attrs
                        with suppress(KeyError):
                            del check_value_sizes[attr]
                    for attr in 'facesdata', 'blocksdata', 'normals':
                        with suppress(KeyError):
                            del check_value_sizes[attr]
                        del value_sizes[attr]
    for mtype, value_mtype in list(savedata.items()):
        if mtype not in check_savedata:
            del savedata[mtype]
        else:
            for sizes in list(value_mtype.keys()):
                if sizes not in check_savedata[mtype]:
                    del value_mtype[sizes]
    if savedata == check_savedata:
        return 'check: pass weak'
    return 'check: data different'
    
def write_diff(filename, check_savedata, savedata):
    from difflib import unified_diff
    from pprint import pformat
    columns = int(os.environ.get('COLUMNS') or '80') - 1
    diff = unified_diff(pformat(check_savedata, width=columns).splitlines(),
                        pformat(savedata, width=columns).splitlines(),
                        fromfile=filename, tofile='<generated by {}>'.format(__file__),
                        n=3, lineterm='')
    filename += '.diff'
    isempty = True
    with open(filename, 'wt') as file:
        for line in diff:
            print(line, file=file)
            isempty = False
    if isempty and os.path.exists(filename):
        os.remove(filename)
            
def pool_create_modelfile(filename, pickle_protocol, check):
    import time
    seconds = time.process_time()
    
    if check:
        import pickle
        import pybiklib.model
        try:
            with open(filename, 'rb') as datafile:
                check_savedata = pickle.load(datafile)
                check_vectors = pybiklib.model.Model.read_vectors(datafile)
        except FileNotFoundError:
            check_savedata = {}
            check_vectors = []
        check = (check_savedata, check_vectors)
    else:
        check = None
        
    savedata, vectors, cnt_vectors, dup_vectors = pool_create_modelfiledata(filename)
    vals = dups = ''
    if not DEBUG_MODELFAST:
        dedup = Dedup()
        savedata = dedup.dedup(savedata)[0]
        if DEBUG_MSG:
            vals = '\n  vals: %6s  vec: %6s' % (dedup.cnt_values, cnt_vectors)
            dups = '\n  dups: %6s       %6s' % (dedup.cnt_dups, dup_vectors)
    
    bsavedata = dumps(savedata, pickle_protocol)
    datasize = ['{:.1f} kb'.format(len(bsavedata) / 1000), '---']
    vectors = array('f', chain.from_iterable(vectors))
    vectorssize = '{:.1f} kb'.format(len(vectors) * vectors.itemsize / 1000)
    if not DEBUG_MODELFAST:
        bsavedata = pickletools.optimize(bsavedata)
        datasize[1] = '{:.1f} kb'.format(len(bsavedata) / 1000)
    seconds = time.process_time() - seconds
    
    if check is None:
        vlen = array('I', [len(vectors)])
        if sys.byteorder != filebyteorder:
            vlen.byteswap()
            vectors.byteswap()
        with open(filename, 'wb') as datafile:
            datafile.write(bsavedata)
            vlen.tofile(datafile)
            vectors.tofile(datafile)
        message = 'generated'
    else:
        message = pool_check_compare_modelfiledata(check_savedata, check_vectors, savedata, vectors)
        write_diff(filename, check_savedata, savedata)
    return filename, seconds, '{} {:{}} ({:>9}, {:>9}), {:>9}, {:7} vectors {:>8}{}{}'.format(
                    message, filename, len(os.path.dirname(filename))+11,
                    datasize[0], datasize[1], format_seconds(seconds), len(vectors) / 3, vectorssize,
                    vals, dups)
    
def pool_create_indexdata():
    savedata_type = {}
    savedata_types = []
    savedata_size = {}
    savedata_sizes = {}
    savedata_facenames = []
    savedata = {'type': savedata_type, 'types': savedata_types,
                'normsize': savedata_size, 'sizes': savedata_sizes,
                'facenames': savedata_facenames,
               }
    facekeys = []
    for Factory in modeldefs:
        factory = ModelFactory(Factory)
        savedata_type[factory.type] = factory.getattrs()
        savedata_types.append(factory.type)
        savedata_size_type = {}
        savedata_sizes_type = {}
        savedata_size[factory.type] = savedata_size_type
        savedata_sizes[factory.type] = savedata_sizes_type
        for facekey, facename in zip(factory.facekeys, factory.facenames):
            if facekey not in facekeys:
                facekeys.append(facekey)
                savedata_facenames.append((facekey, facename))
        for filename, sizes, size, norm_size in enum_modelfiles(Factory):
            savedata_size_type[size] = norm_size
            if norm_size in savedata_sizes_type:
                assert [sizes, filename] == savedata_sizes_type[norm_size]
            else:
                savedata_sizes_type[norm_size] = [sizes, filename]
        size_range = ((min(vals), max(vals)) for vals in zip(*savedata_size_type.keys()))
        defaultsize = savedata_type[factory.type]['defaultsize']
        defaultsize = tuple(max(smin, min(s, smax)) for s, (smin, smax) in zip(defaultsize, size_range))
        savedata_type[factory.type]['defaultsize'] = defaultsize
    return savedata
    
def pool_check_compare_indexdata(check, savedata):
    if check == savedata:
        return 'check: pass full'
        
    def remove_mtype(data, mtype):
        del data['normsize'][mtype]
        del data['sizes'][mtype]
        del data['type'][mtype]
    for mtype in check['types']:
        if mtype not in savedata['types']:
            remove_mtype(check, mtype)
    for mtype in savedata['types']:
        if mtype not in check['types']:
            remove_mtype(savedata, mtype)
    for attr in 'normsize','sizes',:
        for mtype, check_value in check[attr].items():
            savedata_value = savedata[attr][mtype]
            for key in list(check_value.keys()):
                if key not in savedata_value:
                    del check_value[key]
            
    if check == savedata:
        return 'check: pass weak'
    return 'check: data different'
        
def pool_create_indexfile(filename, pickle_protocol, check):
    import time
    seconds = time.process_time()
    
    if check:
        import pybiklib.model
        pybiklib.model.Model.load_index()
        check = pybiklib.model.Model.cache_index
    else:
        check = None
    savedata = pool_create_indexdata()
    vals = dups = ''
    if not DEBUG_MODELFAST:
        dedup = Dedup()
        savedata = dedup.dedup(savedata)[0]
        if DEBUG_MSG:
            vals = '\n  vals: %6s' % dedup.cnt_values
            dups = '\n  dups: %6s' % dedup.cnt_dups
    bsavedata = dumps(savedata, pickle_protocol)
    datasize = ['{:.1f} kb'.format(len(bsavedata) / 1000), '---']
    if not DEBUG_MODELFAST:
        bsavedata = pickletools.optimize(bsavedata)
        datasize[1] = '{:.1f} kb'.format(len(bsavedata) / 1000)
    seconds = time.process_time() - seconds
    
    if check is None:
        with open(filename, 'wb') as datafile:
            datafile.write(bsavedata)
        message = 'generated'
    else:
        message = pool_check_compare_indexdata(check, savedata)
        write_diff(filename, check, savedata)
    return filename, seconds, '{} {:{}} ({:>9}, {:>9}), {:>9}{}{}'.format(
                    message, filename, len(os.path.dirname(filename))+11,
                    datasize[0], datasize[1], format_seconds(seconds),
                    vals, dups)
    
def pool_run(args):
    func, *args = args
    return func(*args)
    
def get_indexfilename(dirname):
    return os.path.join(dirname, 'f00index' if DEBUG_MODELFAST else 'd00index')
    
def create_modeldata(dirname, testfunc=None, parallel=1, pickle_protocol=-1,
                              reproducible=False, check=False):
    # prepare jobs
    if testfunc is None:
        testfunc = lambda arg: True
    if DEBUG_INDEXONLY:
        modelfiles = []
    else:
        modelfiles = pool_enum_modelfiles(dirname, testfunc)
    jobs = [(pool_create_modelfile, m, pickle_protocol, check) for m in modelfiles]
    indexfilename = get_indexfilename(dirname)
    jobs.append((pool_create_indexfile, indexfilename, pickle_protocol, check))
    # run jobs
    if parallel is True:
        parallel = cpu_count()
    if not parallel or parallel < 1:
        parallel = 1
    realparallel = min(parallel, len(jobs))
    print('using {} / {} processes'.format(realparallel, parallel))
    imap_model = pool_functions(realparallel, reproducible)
    result = []
    for filename, seconds, lines in imap_model(pool_run, jobs):
        result.append((filename, seconds))
        print(lines)
        sys.stdout.flush()
    if DEBUG_MSG and not DEBUG_MODELFAST:
        result.sort(key=lambda fs: fs[1], reverse=True)
        if result != sorted(result, key=lambda fs: fs[0]):
            print('warning: reorder jobs to optimize parallel build')
            for filename, seconds in result:
                print(' ', os.path.basename(filename), format_seconds(seconds))
    
    
if __name__ == '__main__':
    minsize = 1
    maxsize = 5
    create_modeldata('data/models',
                     check=True,
                    )