#----------------------------------------------------------------------------
# SOL, seek optimized layout tool.
# based on sol package from AFK
#
# Copyright 2007-2010 Electronic Arts Ltd.
# Script by Andy Younger, ayounger@ea.com
# Modified for generation 4 consoles by Timotius Margo, tmargo@ea.com
#----------------------------------------------------------------------------
import os
import re
import shutil
import sys
import time
import struct
import hashlib


class DataStream:
    def __init__(self, file, extradata=None):
        self.fp = file
        self.extradata = extradata
        self.offset = 0
        self.size = os.fstat(file.fileno()).st_size
        self.extradatalen = 0
        if (extradata != None):
            self.extradatalen = len(extradata)
            self.size += self.extradatalen

    def Read(self, bytes):
        if self.extradatalen > 0:
            if (self.offset < self.extradatalen):
                if ((self.offset + bytes) < self.extradatalen):
                    data = self.extradata[self.offset:bytes]
                    self.offset += bytes
                    return data
                else:
                    data = self.extradata[self.offset:(self.extradatalen-self.offset)]
                    data = data + self.fp.read(bytes-(self.extradatalen-self.offset))
                    self.offset += bytes
                    return data
            else:
                data = self.fp.read(bytes)
                return data
        else:
            data = self.fp.read(bytes)
            return data

        return ""

    def Write(self, file):
        while True:
            data = self.Read(8192)
            if data != "":
                file.write(data)
            else:
                return

    def Len(self):
        return self.size

    def Reset(self):
        self.offset = 0
        self.fp.seek(0)

    def IsEncrypted(self):
        return (self.extradatalen > 0)


#----------------------------------------------------------------------------
def Human(v):
    if (v < 1024):
        return v
    if (v < 1024 * 1024):
        return str( (v + 1023) / 1024) + "K"
    return str( (v + 1023) / (1024 * 1024)) + "M"

#----------------------------------------------------------------------------
def ProgressBar( got, total, width=54, char=chr(124) ):
    out = " [%-" + str(width) + "s] %s/%s (%d%%)\r"
    progress = out % (char * int(width*got/total), Human(got), Human(total), (100*got/total) )
    sys.stdout.write( progress )

#----------------------------------------------------------------------------
def Align(offset, alignment):
    return (offset + (alignment-1)) & ~(alignment-1)

#----------------------------------------------------------------------------
def AlignBytes16(str):
    a = len(str) & 15
    if (a == 0):
        return ""
    return '\0' * (16-a)

#----------------------------------------------------------------------------
def DJB2(str):
    h = 5381L
    m = 0xFFFFFFFFFFFFFFFFL
    for c in str:
        h = (((h * 33)&m) + ord(c)) & m
    return h & m

#----------------------------------------------------------------------------
def RefpackSize(data):
    if (len(data) <= 6):
        return -1
    size = -1
    if (ord(data[0]) == 0x10 and ord(data[1]) == 0xFB):
        size = (ord(data[2])<<16) + (ord(data[3])<<8)
        size += ord(data[4])
    elif (ord(data[0]) == 0x90 and ord(data[1]) == 0xFB):
        size = (ord(data[2])<<24) + (ord(data[3])<<16)
        size += (ord(data[4])<<8) + (ord(data[5]))
    return size

#----------------------------------------------------------------------------
def cmp(a, b):
    v = a - b
    if (v < 0):
        return -1
    if (v >= 1):
        return 1
    return 0

#----------------------------------------------------------------------------
def fastbigOffsetSort(a, b):
    v = cmp(a[4], b[4]) # offset
    if (v == 0):
        return cmp(a[3], b[3])   # size
    return v

#----------------------------------------------------------------------------
def fastbigHashSort(a, b):
    return cmp(a[0], b[0])

#----------------------------------------------------------------------------
# Build a 'new' format fast big file without the need for bigfile.exe
#
def BuildBigF(name, bigHeadName, alignment, filename_fullpath_size_tuple, file_reader):
#    filename_fullpath_size_tuple = filename_fullpath_size_tuple[:2]
    bigSize = 0
    paths = []
    fileCount = len(filename_fullpath_size_tuple)

    # first path
    totalSize = 0
    dirs = []
    maxDirSize = 0
    maxFnSize = 0
    ftab = []
    for filename, fullpath, size in filename_fullpath_size_tuple:
        filename = filename.lower()
        filename = filename.replace("\\", "/")
        dir, fn = os.path.split(filename)
        dir = dir.replace("\\", "/")
        if not dir in dirs:
            maxDirSize = max(maxDirSize, len(dir))
            dirs.append(dir)
        maxFnSize = max(maxFnSize, len(fn))
        ftab.append([ DJB2(filename), filename, fullpath, size, totalSize ])
        totalSize += size
        totalSize = Align(totalSize, alignment)

    alignmentBitCount = 0
    a = alignment
    while a:
        alignmentBitCount += 1
        a >>= 1
    alignmentBitCount -= 1
    assert( 1<< alignmentBitCount == alignment )

    # Header
    baseHeaderSize = Align(36, 16)
    baseHeaderSize += Align(fileCount * 20, 16)  # file hash and offset
    baseHeaderSize += Align(fileCount, 16)       # compression
    nameHeaderSize = Align((maxFnSize+3) * fileCount, 16)
    nameHeaderSize += Align((maxDirSize+1) * len(dirs), 16)

    firstFileOffset = Align(baseHeaderSize + nameHeaderSize, alignment)
    totalSize += firstFileOffset

    header = "EB"
    header += struct.pack(">H", 3)  # version
    header += struct.pack(">I", len(filename_fullpath_size_tuple))
    header += struct.pack(">H", 0x16 + 0x1) # sorted by 64bit hash
    header += struct.pack(">BB", alignmentBitCount, 0)
    header += struct.pack(">II", baseHeaderSize, nameHeaderSize)
    header += struct.pack(">BBH", maxFnSize + 3, maxDirSize+1, len(dirs))
    header += struct.pack(">Q", firstFileOffset)
    header += struct.pack(">I", 0)  # FAT size
    header += AlignBytes16(header)
    assert(len(header) == 48)

    # Hash of file + offset, and compression data, sorted by hash
    ftab.sort( fastbigHashSort )
    for hashv, fname, path, size, offset in ftab:
        fileoffset = (offset + firstFileOffset)
        assert( fileoffset & (alignment-1) == 0 )
        header += struct.pack(">III", fileoffset >> alignmentBitCount, 0, size)
        header += struct.pack(">Q", hashv)
    header += AlignBytes16(header)
    for i in xrange(fileCount):
        header += struct.pack(">B", 0)
    header += AlignBytes16(header)
    assert(baseHeaderSize == len(header))

    # filename & dir tables
    for hashv, fname, path, size, offset in ftab:
        dir, fn = os.path.split( fname.lower() )
        dir = dir.replace("\\", "/")
        hl = struct.pack(">H", dirs.index(dir))
        hl += fn + ("\0" * (maxFnSize + 1 - len(fn)))
        assert(len(hl) == maxFnSize+3)
        header += hl
    header += AlignBytes16(header)
    for d in dirs:
        dl = d + ("\0" * (maxDirSize + 1 - len(d)))
        assert(len(dl) == maxDirSize+1)
        header += dl
    header += AlignBytes16(header)
    assert(nameHeaderSize + baseHeaderSize == len(header))

    # Now write out header and all files
    blank = '\0' * alignment

    fp = open(name, "wb", 64 * 1024 * 1024)
    fp.write(header)
    ftab.sort( fastbigOffsetSort )
    lastpos = curpos = fp.tell()

    refpackSize = {}
    for hashv, fname, path, size, foffset in ftab:
        offset = foffset + firstFileOffset

        datastream = file_reader.GetContents(fname, path)
        assert(datastream.Len() == size)

        data = datastream.Read(16)
        datastream.Reset()

        usize = RefpackSize(data)
        if (usize != -1):
            refpackSize[ hashv ] = usize

        delta = offset - curpos
        fp.write( blank[:delta] )
        assert(fp.tell() == offset)
        datastream.Write(fp)

        curpos += datastream.Len() + delta
        if (curpos - lastpos >= (totalSize/100)):
            ProgressBar(curpos, totalSize)
            lastpos = curpos

    ProgressBar(totalSize, totalSize)
    print ""
    fp.close()

    # Build big header file, note, no catch for refpack compression info
    if (bigHeadName == ""):
        return

    fp = open(bigHeadName, "wb", 1 * 1024 * 1024)
    bhSize = (4*4) + ((5*4) * fileCount)
    header = "ViV4"
    header += struct.pack("<I", bhSize)
    header += struct.pack(">I", fileCount)
    header += struct.pack(">I", bhSize)
    ftab.sort( fastbigHashSort )
    for hashv, filename, path, size, foffset in ftab:
        usize = 0
        if (refpackSize.has_key(hashv)):
            usize = refpackSize[ hashv ]
        hash1,hash2 = (hashv>>32) & 0xFFFFFFFF, hashv & 0xFFFFFFFF
        foffset += firstFileOffset
        header += struct.pack(">IIIII", foffset, size, usize, hash1, hash2)
    fp.write(header)
    assert(fp.tell() == bhSize)
    fp.close()

#----------------------------------------------------------------------------
# Build a big4 without the need from glump, hopefully get better IO
# times as can control buffering.
#
def BuildBig4(fnBig, fnBigHead, alignment, filename_fullpath_size_tuple, file_reader):

    # 1st pass, work out first file offset
    firstFileOffset = 16
    offsets = []
    for filename, path, size in filename_fullpath_size_tuple:
        firstFileOffset += 8    # size + offset
        firstFileOffset += len(filename) + 1
    firstFileOffset = Align(firstFileOffset, alignment)

    # 2nd pass, work out all file offsets, and build
    # FAT
    offset = firstFileOffset
    endoffset = offset
    offsetArray = []
    fat = ""
    for filename, path, size in filename_fullpath_size_tuple:
        offset = Align(offset, alignment)
        fat += struct.pack(">II", offset, size)
        filename = filename.lower().replace("\\", "/")
        fat += filename + "\0"
        offsetArray.append([path, filename, size, offset])
        offset = offset + size

    # write out header + FAT
    fp = open(fnBig, "wb", 64 * 1024 * 1024)
    header = "BIG4"
    header += struct.pack("<I", offset)         # length
    header += struct.pack(">I", len(filename_fullpath_size_tuple))
    header += struct.pack(">I", 16 + len(fat))  # header length
    fp.write(header)
    fp.write(fat)

    refpackSize = {}

    # write out files
    blank = "\0" * alignment
    curpos = fp.tell()
    lastpos = curpos
    for path, filename, size, foffset in offsetArray:
        delta = foffset - curpos
        assert(delta >= 0)
        fp.write( blank[:delta] )
        assert(fp.tell() == foffset)

        datastream = file_reader.GetContents(filename, path)
        assert(datastream.Len() == size)
        data = datastream.Read(16)
        datastream.Reset()

        usize = RefpackSize(data)
        if (usize != -1):
            refpackSize[ filename ] = usize

        datastream.Write(fp)
        curpos += datastream.Len() + delta
        assert( curpos == fp.tell() )
        if (curpos - lastpos >= (offset/90)):
            ProgressBar(curpos, offset)
            lastpos = curpos

    ProgressBar(offset, offset)
    print ""
    fp.close()

    if (fnBigHead == ""):
        return

    # Build big header file, note, no catch for refpack compression info
    fp = open(fnBigHead, "wb", 1 * 1024 * 1024)
    bhSize = (4*4) + ((5*4) * len(offsetArray))
    header = "ViV4"
    header += struct.pack("<I", bhSize)
    header += struct.pack(">I", len(offsetArray))
    header += struct.pack(">I", bhSize)
    for path, filename, size, foffset in offsetArray:
        usize = 0
        if (refpackSize.has_key(filename)):
            usize = refpackSize[filename]
        h = DJB2(filename)
        hash1,hash2 = ((h>>32)&0xFFFFFFFF, h&0xFFFFFFFF)
        header += struct.pack(">IIIII", foffset, size, usize, hash1, hash2)
    fp.write(header)
    assert(fp.tell() == bhSize)
    fp.close()

#----------------------------------------------------------------------------
def BuildBigFiles(sorted_files, dst_path, src_path, compressed_path, bigFormat, bighead, file_reader):
    dst_path = FixPath(dst_path)
    src_path = FixPath(src_path)
    compressed_path = FixPath(compressed_path)

    MkDir(dst_path)
    bigcount = 0
    rebuild_count = 0
    files = []
    bigfile = 0
    sorted_index = 0
    logfile = open(os.path.join(os.path.dirname(dst_path), "sol.log"), "at")
    checkpointstring = "bigfile_checkpoint"
    for i in sorted_files:
        flush = False
        if (i[1].lower().find(checkpointstring) == 0):
            bigfilename = i[1][len(checkpointstring):]
            if (bigfilename[0] == "_"):
                bigfilename = bigfilename[1:].lower()
            flush = True
        else:
            files.append(i)

        if (flush):
            filebigdata = []
            deps = []
            needsbuilt = False
            compressionFailure = False
            bigname = dst_path + "%s.big" % (bigfilename)
            bhname = dst_path + "%s.bh" % (bigfilename)
            logfile.write(bigfilename + ":\n-" + "-" * len(bigfilename) + "\n")
            if (not bighead):
                bhname = ""
            for f in files:
                file_path = compressed_path + f[1]
                unpacked_size = GetSize(src_path + f[1])
                packed_size = GetSize( file_path )
                file_size = packed_size
                if (packed_size != 0):
                    # need at least 5% compression
                    if (unpacked_size * 0.95 <= packed_size):
                        file_path = src_path + f[1]
                        file_size = unpacked_size

                    # Check here to catch case where file failed to compress so is stale
                    # in this case we have no choice but to swap back to the uncompressed file. This has been seen occuring
                    # when people add bad filenames (including braces for one) in filenames, which then breaks the compression
                    # gmake step
                    if NeedsBuilt(src_path+f[1], file_path):
                        file_path = src_path + f[1]
                        file_size = unpacked_size
                        compressionFailure = True
                        print "Compression failure: %s!" % (file_path)

                else:
                    file_path = src_path + f[1]
                    file_size = unpacked_size

                if (compressionFailure or NeedsBuilt(file_path, bigname)):
                    deps.append(file_path)
                    needsbuilt = True
                filebigdata.append( [f[1], file_path, file_reader.GetSize(f[1], file_path)] )
                logfile.write(f[1] + "\n")

            if (bhname != "" and NeedsBuilt(bigname, bhname)):  # bh file does not exist, so flag full build
                needsbuilt = True
                if (len(deps) == 0):                            # if len of deps is 0 then add it to tell user whats going on
                    deps.append(bhname)

            if (needsbuilt):
                maxFilesToDisplay = 16
                numDeps = len(deps)
                if (numDeps):
                    print "Rebuilding %s due to %d file/s of %d changing" % (bigname, numDeps, len(filebigdata))
                    for i in range( min(maxFilesToDisplay, numDeps) ):
                        print ">>%d/%d %s" % (i+1, numDeps, deps[i])
                    if (numDeps > maxFilesToDisplay):
                        print ">>.........."

                if (bigFormat == "bigf"):
                    BuildBigF(bigname, bhname, 64, filebigdata, file_reader)
                else:
                    BuildBig4(bigname, bhname, 64, filebigdata, file_reader)
                rebuild_count += 1

            files = []
            logfile.write("\n")

        sorted_index += 1

    logfile.close()
    return rebuild_count

#----------------------------------------------------------------------------
def FixPath(path):
    path = path.replace("\\", "/")
    if (path[-1] != "/"):
        path += "/"
    return path

#----------------------------------------------------------------------------
def Load(fname):
    file = []
    try:
        fp = open(fname, "rt")
    except:
        return file
    line = fp.readline()
    while (line != ""):
        line = line.lstrip().rstrip()
        if (line != ""):
            file.append(line)
        line = fp.readline()
    fp.close()

    if (len(file) == 1):
        # suspicious maybe it was unix line delimited..
        line = file[0]
        file = line.split('\r')
    return file

#----------------------------------------------------------------------------
def LoadRE(fname):
    strings = Load(fname)
    res = []
    for s in strings:
        comment = s.find("#")
        if (comment != -1):
            s = s[:comment].rstrip()
        if (lowercase):
            s = s.lower()
        if (len(s)):
            simple = ""
            if (s[0] == '^' and s[-1] == '$'):
                simple = s[1:-1]
                # wildcard = simple.find("*")
                if (simple.find("*") != -1 or simple.find("?") != -1 or simple.find("+") != -1):
                    simple = ""
                else:
                    simple = simple.replace("\\.", ".")
            res.append( [re.compile(s, re.IGNORECASE), simple] )
    return res

#----------------------------------------------------------------------------
stat_cache = {}

def ClearStatCache():
    stat_cache.clear()

#----------------------------------------------------------------------------
def FindSourceFiles(path):
    flist = []
    path = FixPath(path)
    pathlen = len(path)
    for root, dirs, files in os.walk(path):
        root = FixPath(root)
        for f in files:
            p = root + f
            p = p.replace("\\", "/")
            if (lowercase):
                p = p.lower()
            if (p.find(" ") != -1):
                continue # ignore fnames with spaces in them!
            flist.append( [p, p[pathlen:] ] )
            stat_cache[p] = os.stat(p); # set up stat cache
    return flist

#----------------------------------------------------------------------------
def MkDir(path):
    try:
        os.makedirs(path)
    except OSError:
        return

#----------------------------------------------------------------------------
def GetSize(fname):
    if (stat_cache.has_key(fname)):
        return stat_cache[fname].st_size
    try:
        st = os.stat(fname)
        stat_cache[fname] = st;
        size = st.st_size
    except OSError:
        return 0
    return size

#----------------------------------------------------------------------------
gIgnoreRE = []

def IgnoreTimeStampRE(a):
    gIgnoreRE.append( re.compile( a ) )

def NeedsBuilt(a, b):
    try:
        if (stat_cache.has_key(a)):
            src_stat = stat_cache[a]
        else:
            src_stat = os.stat(a)
            stat_cache[a] = src_stat
        if (stat_cache.has_key(b)):
            dst_stat = stat_cache[b]
        else:
            dst_stat = os.stat(b)
            stat_cache[b] = dst_stat

    except:
        return True
    if (src_stat.st_mtime > dst_stat.st_mtime):
#       print "--->> deps %s newer than %s <<---" % (a, b)
        for re in gIgnoreRE:
            if (re.search( a ) != None):
#               print "--->> ignored! due to regex <<---"
                return False

        stat_cache.pop(b)       # delete entry in stat_cache as we presume its going to be dirtied
        return True
    return False

#----------------------------------------------------------------------------
def StripFiles(log, regex):
    found = []
    newlog = []
    index = 0
    for l in log:
        finds = 0
        for j in regex:
            if (j[0].search(l[1],0) != None and finds == 0):
                found.append(l)
                finds = 1
        index = index +1
        if (finds == 0):
            newlog.append(l)
    return found, newlog

#----------------------------------------------------------------------------
# Create streaming install big files layout by filtering files to either
# launch(data) or extra checkpoints based on regular expression lists. E.g.
#
# filelist   : [ a, b, c, CP1, d, e, CP2 ]
# launchregex: [ a, e, g ]
# extraregex : [ c, d, e, f ]
#
# new filtered list: [ a, b, CP1, c, CP1Extra, e, CP2, d, CP2Extra ]
#
def FilterStreamingInstallFiles(filelist, launchregex, extraregex):
    extra = []
    newlist = []
    cpstring = "BigFile_Checkpoint_"
    extrastring = "_Extra"

    for f in filelist:
        # Insert extra files to new list if exist and append corresponding checkpoint
        if (f[1].find(cpstring) == 0):
            newlist.append(f)

            if (len(extra) > 0):
                newlist.extend(extra)
                newlist.append([f[0] + extrastring, f[1] + extrastring, f[2]])
                extra[:] = []

            continue

        # Append files that match launch regex to the new list
        for l in launchregex:
            if (l[0].search(f[1], 0) != None):
                newlist.append(f)
                break

        # Append files that match extra regex to the extra list
        else:
            for e in extraregex:
                if (e[0].search(f[1], 0) != None):
                    extra.append(f)
                    break

            # Append unmatch files to the new list
            else:
                newlist.append(f)

    return newlist

#----------------------------------------------------------------------------
def prioritySort(a, b):
    return a[2] - b[2]

#----------------------------------------------------------------------------
def SortFiles(log, regex):
    # create map for fast lookup
    fname_map = {}
    maxNum = 999999999
    for i in log:
        fname_map[i[1]] = maxNum

    # Set priority of each filename (pass1 simple non-regex)
    priority = 0
    nMatched = 0
    for i in regex:
        priority += 1
        if (i[1] != ""):
            if (fname_map.has_key(i[1])):
                if (fname_map[ i[1] ] == maxNum):
                    fname_map[ i[1] ] = priority
                    nMatched += 1

    # Set priority of each filename (pass2 regex)
    priority = 0
    for i in regex:
        priority += 1
        if (i[1] == ""):
            for j in log:
                if (fname_map[ j[1] ] > priority and i[0].search(j[1],0) != None):
                    fname_map[ j[1] ] = priority
                    continue

    # Then create sorted log from this
    sorted = []
    for i in log:
        priority = fname_map[ i[1] ]
        sorted.append( [ i[0], i[1], priority ] )
    sorted.sort( prioritySort )
    return sorted

#----------------------------------------------------------------------------
def FilterPatchList(log, fname):
    entries = Load(fname)
    sortedList = []
    missingFilesList = []
    counter = 0
    for e in entries:
        comment = e.find("#")
        if (comment != -1):
            e = e[:comment].rstrip()
        if (lowercase):
            e = e.lower()
        if (len(e)):
            counter += 1
            if (e.lower().find("bigfile_checkpoint") != -1):
                sortedList.append( ["", e, counter] )
            else:
                for l in log:
                    if (e.lower() == l[1].lower()):
                        sortedList.append( [l[0], e, counter] )
                        break
                else:
                    missingFilesList.append(e)
    return sortedList, missingFilesList

#----------------------------------------------------------------------------
def BuildCompressionMakefile(files, tmp_path, dst_path, comp_path, comp_opt, minSize, batchSize):
    if (comp_path == ""):
        return 0

    tmp_path = FixPath(tmp_path)
    dst_path = FixPath(dst_path)

    MkDir(dst_path)
    fp = open(tmp_path + "Makefile.pack", "wb")
    mf = []
    mf.append("# autogenerated makefile")
    mf.append("COMP=@%s" % comp_path.replace("\\", "/"))
    mf.append("COMPBATCH=@%s" % comp_path.replace("\\", "/"))
    mf.append("COMPOPT=%s" %  comp_opt)

    bChunkPack = (comp_path.find("chunkpack.exe")) != -1

    remote_targets = []
    build = []
    local_files = []
    count = 0
    for f in files:
        source = f[0]
        target = dst_path + f[1]

        if (NeedsBuilt(source, target)):
            ssize = GetSize(source)
            if (ssize):
                count = count + 1
                if (ssize >= minSize):
                    remote_targets.append(target)
                    build.append("%s: %s" % (target, source))
                    if (bChunkPack):
                        build.append("\t$(COMP) $(COMPOPT) %s %s" %(source, target))
                    else:
                        build.append("\t$(COMP) $(COMPOPT) %s=%s" %(source, target))
                    MkDir( os.path.dirname(target) )

                else:
                    local_files.append([ target, source ])

    nlocalfiles = len(local_files)
    i = 0
    local_targets = []
    while (i < nlocalfiles):
        targetName = "batchTarget%d" % i
        srcfiles = ""
        fnames = ""
        j = 0
        while (j < batchSize and i+j < nlocalfiles):
            target = local_files[i+j][0]
            srcfiles += " " + local_files[i+j][1]
            fnames += " %s=%s" % (local_files[i+j][1], target)
            j += 1
            MkDir( os.path.dirname(target) )
        if (bChunkPack):
            fnames = fnames.replace("=", " ")

        local_targets.append(targetName);
        build.append(".PHONY : %s" % targetName)
        build.append("%s: %s" % (targetName, srcfiles))
        build.append("\t$(COMPBATCH) $(COMPOPT) %s" %fnames)
        i = i + j

    targets = []
    li = 0
    ri = 0

    # mix together both types of targets, hopefully doing this will make it play nicer with HD / network bandwitdh
    distribution = 1
    if (len(local_targets)):
        distribution = (len(remote_targets)+len(local_targets)) / len(local_targets)
    while (li+ri < len(local_targets) + len(remote_targets)):
        if (li < len(local_targets)):
            targets.append(local_targets[li])
            li += 1
        j = 0
        while (j < distribution):
            if (ri < len(remote_targets)):
                targets.append(remote_targets[ri])
                ri += 1
            j += 1

    mf.append("all: \\")
    for i in targets:
        mf.append("\t%s \\" % i)
    mf.append("")
    for l in build:
        mf.append(l)

    for l in mf:
        fp.write(l + "\r\n")
    fp.close()
    return count

#----------------------------------------------------------------------------
def FilesEqual(a, b):
    a = Load(a)
    b = Load(b)
    if (len(a) != len(b)):
        return False
    i = 0
    nlines = len(a)
    while (i < nlines):
        if (a[i] != b[i]):
            return False
        i = i + 1
    return True

#----------------------------------------------------------------------------
def CopyExcludedFiles(excluded, dst_path, src_path, file_reader):
    dst_path = FixPath(dst_path)
    src_path = FixPath(src_path)
    count = 0

    # Copy over files that were excluded from bigfiles
    for e in excluded:
        spath = src_path + e[1]
        dpath = dst_path + e[1]
        size = GetSize(spath)
        if (size and NeedsBuilt(spath, dpath)):
            dir = dpath.rfind("/")
            if (dir != -1):
                MkDir( dpath[:dir] )
                print "\t%s" % e[1]

                file_reader.Copy(e[1], spath, dpath)
                count += 1
    return count

#----------------------------------------------------------------------------
def ReconcileCompressedFiles(compress_files, compressed_path, src_path):
    src_path = FixPath(src_path)
    file_map = {}
    for i in compress_files:
        file_map[ i[1] ] = i[0]

    files_in_compression_folder = FindSourceFiles(compressed_path)
    for i in files_in_compression_folder:
        if not file_map.has_key( i[1] ):
            # deal with file in pack folder
            src = src_path + i[1]
            print "\tdeleting %s" % (i[0])
            try:
                os.remove( i[0] )
            except:
                print "\t\tCan't delete %s, is it write protected? please manually delete" % i[0]
            if stat_cache.has_key(i[0]):
                del stat_cache[ i[0] ]

            # deal with file in runtime folder
            if stat_cache.has_key(src):
                del stat_cache[ src ]
                print "\tTouching %s" % (src)
                try:
                    os.utime( src, None )               # do unix style touch on file
                    stat_cache[ src] = os.stat( src )   # get new revised timestamp
                except:
                    print "\t\tCan't touch %s, please investigate why, is it being accessed by another process?" % src

#----------------------------------------------------------------------------
def MakeXGEProfile(fname):
    fp = open(fname, "wb")
    crlf = "\r\n"
    fp.write('<?xml version="1.0" encoding="UTF-8" standalone="no" ?>' + crlf)
    fp.write('<Profile FormatVersion="1">' + crlf)
    fp.write('<Tools>' + crlf)
    fp.write('<Tool Filename="gnumake" AllowIntercept="true" SuccessExitCodes="0" DeriveCaptionFrom="firstparam" />' + crlf)
    fp.write('<Tool Filename="vsimake" AllowIntercept="true" SuccessExitCodes="0" DeriveCaptionFrom="firstparam" />' + crlf)
    fp.write('<Tool Filename="huff" AllowRemote="true" DeriveCaptionFrom="lastparam" />' + crlf)
    fp.write('<Tool Filename="chunkpack" AllowRemote="true" DeriveCaptionFrom="lastparam" />' + crlf)
    fp.write('</Tools>' + crlf)
    fp.write('</Profile>' + crlf)
    fp.close()

#----------------------------------------------------------------------------
def LoadEncryptRE(fname):
    key = ""
    keyid = ""

    strings = Load(fname)
    res = []
    for s in strings:
        if s.startswith("#!key "):
            key = s[6:].strip()
        elif s.startswith("#!keyid "):
            keyid = s[8:].strip()
        else:
            comment = s.find("#")
            if (comment != -1):
                s = s[:comment].rstrip()
            if(lowercase):
                s = s.lower()
            if (len(s)):
                if len(key) != 32:
                    raise Exception("You should specify a 16-byte hex string for a key before adding encryption regexes")
                res.append( (re.compile(s, re.IGNORECASE), key, keyid) )
    return res

#----------------------------------------------------------------------------
# A helper to handle size/content retrieval for both encrypted and plain files
class FileReader:
    def __init__(self, encryptor, encrypted_file, temp_path):
        self.encrypted_files = LoadEncryptRE(encrypted_file)
        self.encryptor = encryptor
        self.temp_path = temp_path

    def _GetContents(self, path):
        return open(path, "rb")

    def _GetEncryptedSize(self, path):
        size = GetSize(path)
        # Add 48-byte header and pad to 16-byte block size; for some reason openssl.exe always rounds the size up
        # (even if the size is divisible by 16)
        return 48 + (size + 16) / 16 * 16

    def _GetEncryptedContents(self, path, key, keyid):
        temp = self.temp_path + 'encrypt.res'
        size = GetSize(path)

        checkSum = ""
        with open(path, "rb") as fp:
            checkSum = hashlib.sha256(fp.read(32)).digest()

        if os.system("%s enc -aes-128-cbc -in %s -out %s -K %s -iv %s" % (self.encryptor, path, temp, key, key)) != 0:
            raise Exception("Error encrypting file %s to %s" % (path, temp))

        contents = self._GetContents(temp)
        header = "EASF" + struct.pack(">I", size) + keyid.ljust(8)[0:8] + checkSum

        return DataStream(contents, header)

    def _GetEncryptionInfo(self, name):
        for (r, key, keyid) in self.encrypted_files:
            if r.search(name, 0) != None:
                return [key, keyid]

        return None

    def GetSize(self, name, path):
        ei = self._GetEncryptionInfo(name)

        return self._GetEncryptedSize(path) if ei != None else GetSize(path)

    def GetContents(self, name, path):
        ei = self._GetEncryptionInfo(name)

        return self._GetEncryptedContents(path, ei[0], ei[1]) if ei != None else DataStream(self._GetContents(path))

    def Copy(self, name, src_path, dst_path):
        ei = self._GetEncryptionInfo(name)

        if ei != None:
            contents = self._GetEncryptedContents(path, ei[0], ei[1])
            with open(dst_path, "wb") as fp:
                fp.write(contents)
        else:
            shutil.copyfile(src_path, dst_path)

#----------------------------------------------------------------------------
def usagemsg():
    print "\t-make=[gnumake path]"
    print "\t-xgconsole=[xgconsole path]"
    print ""
    print "\t-compressor=[huff.exe/chunkpack.exe path]"
    print "\t-bigformat=[big4|bigf]"
    print "\t-bighead"
    print ""
    print "\t-prefix=[big file prefix name, default is data]"
    print "\t-src=[full path to srcdata]"
    print "\t-dst=[full path to destination data directory (where bigfiles will reside)]"
    print "\t-pack=[full path to directory to pack files into]"
    print "\t-temp=[temporary path for makefile generation / response files etc - make unique per platform for caching purposes]"
    print "\t-ignoretimestamp=[regex] ignore timestamp differences on any file matching this regex"
    print ""
    print "\t-sort=[full path to sort.sol file]"
    print "\t-loose=[full path to loose.sol file]"
    print "\t-ignore=[full path to ignore.sol file]"
    print "\t-uncompressed=[full path to uncompressed.sol file]"
    print "\t-launch=[optional full path to launch.sol file]"
    print "\t-extra=[optional full path to extra.sol file]"
    print ""
    print "\t-remotejobs=[numberremotejobs] number of remote XGE cpu's to run on, default 12"
    print "\t-localjobs=[numberlocaljobs] number of local cpu's to run on (if not using XGE), default 2"
    print "\t-lowercase=[true/false] convert filenames to lowercase or leave as is"
    print ""

def usage():
    usagemsg()
    sys.exit(-1)

def die(msg):
    usagemsg()
    print "Error: %s" % msg
    sys.exit(-1)

#----------------------------------------------------------------------------
print ""
print "============================================================================"
print "SOL (Seek optimized layout)"
print "============================================================================"
print ""

data_prefix = "data"
src_path = ""
dest_path = ""
compressed_path = ""
tmp_path = ""

sort_file = ""
loose_file = ""
ignore_file = ""
uncompressed_file = ""
encrypt_file = ""
launch_file = ""
extra_file = ""
patch_file = ""

compressor_path = ""
compressor_opt = ""
huff_opt = "-tj -b 2048"
chunkpack_opt = "-refpack"
chunkpack_opt_pc64 = " -zlib -zlibchunksize 180"
chunkpack_opt_kettle = " -zlib -zlibchunksize 180"
chunkpack_opt_capilano = " -zlib -zlibchunksize 180"
encryptor_path = ""

bigFormat = "big4"
bigHead = False

xgconsole_path = ""
make_path = "" # gnumake.exe"

remote_jobs = 12
local_jobs = 2

lowercase = False

# Parse command line
for opt in sys.argv[1:]:
    if (opt[0] == "-"):
        equals = opt.find("=")
        arg = ""
        if (equals != -1):
            arg = opt[equals+1:]
            opt = opt[:equals]

        if (opt == "-compressor"):
            if (arg == ""):
                usage()
            compressor_path = arg
            compressor_opt = huff_opt
            if (arg.find("chunkpack.exe") != -1):
                compressor_opt = chunkpack_opt
            elif (arg.find("huff.exe") == -1):
                print "Unknown compression utility!"
                usage();
        elif (opt == "-bigformat"):
            if (arg != "big4" and arg != "bigf"):
                print "Unknown bigformat!"
                usage()
            bigFormat = arg
        elif (opt == "-platform"):
            platform = arg
        elif (opt == "-bighead"):
            bigHead = True
        elif (opt == "-xgconsole"):
            if (arg == ""):
                usage()
            xgconsole_path = arg
        elif (opt == "-sort"):
            if (arg == ""):
                usage()
            sort_file = arg
        elif (opt == "-loose"):
            if (arg == ""):
                usage()
            loose_file = arg
        elif (opt == "-ignore"):
            if (arg == ""):
                usage()
            ignore_file = arg
        elif (opt == "-uncompressed"):
            if (arg == ""):
                usage()
            uncompressed_file = arg
        elif (opt == "-patch"):
            if (arg == ""):
                usage()
            patch_file = arg
        elif (opt == "-launch"):
            if (arg == ""):
                usage()
            launch_file = arg
        elif (opt == "-extra"):
            if (arg == ""):
                usage()
            extra_file = arg
        elif (opt == "-make"):
            if (arg == ""):
                usage()
            make_path = arg
        elif (opt == "-remotejobs"):
            if (arg == ""):
                usage()
            remote_jobs = int(arg)
        elif (opt == "-localjobs"):
            if (arg == ""):
                usage()
            local_jobs = int(arg)
        elif (opt == "-src"):
            if (arg == ""):
                usage()
            src_path = arg
        elif (opt == "-prefix"):
            data_prefix = arg
        elif (opt == "-dst"):
            if (arg == ""):
                usage()
            dest_path = arg
        elif (opt == "-pack"):
            if (arg == ""):
                usage()
            compressed_path = arg
        elif (opt == "-tmp"):
            if (arg == ""):
                usage()
            tmp_path = arg
        elif (opt == "-ignoretimestamp"):
            if (arg == ""):
                usage()
            IgnoreTimeStampRE( arg )
        elif (opt == "-lowercase"):
            if(arg == ""):
                lowercase = False
            lowercase = bool(arg)
        elif (opt == "-encryptor"):
            encryptor_path = arg
        elif (opt == "-encrypt"):
            encrypt_file = arg
        else:
            print "command %s unknown" % opt
            usage()

if (make_path == ""):
    die("make path not set!")
if (sort_file == "" and patch_file == "" or sort_file != "" and patch_file != "" ):
    die("either sort file or patch file needs to be specified")
if (loose_file == ""):
    die("loose file not specified")
if (ignore_file == ""):
    die("ignore file not specified")
if (uncompressed_file == ""):
    die("uncompressed file not specified")
if (launch_file != "" and extra_file == ""):
    die("extra file needs to be specified along with %s launch file" % launch_file)
if (launch_file == "" and extra_file != ""):
    die("launch file needs to be specified along with %s extra file" % extra_file)
if (src_path == ""):
    die("src path not specified")
if (dest_path == ""):
    die("dst path not specified")
if (compressed_path == ""):
    die("pack path not specified")
if (tmp_path == ""):
    die("tmp path not set")

#----------------------------------------------------------------------------
startTime = time.time()

#----------------------------------------------------------------------------
ClearStatCache()

print "Creating temporary path"
tmp_path = FixPath(tmp_path)
MkDir(tmp_path)

print "Loading/Compiling regexpression files"
loose_re = LoadRE(loose_file)
ignore_re = LoadRE(ignore_file)
uncompressed_re = LoadRE(uncompressed_file)
file_reader = FileReader(encryptor_path, encrypt_file, tmp_path)

print "Cleaning out existing checkpoint files"
files = os.listdir(src_path)
for f in files:
    p = FixPath(src_path) + f
    if (p.lower().find("bigfile_checkpoint") != -1):
        print "Deleting: " + p
        os.remove(p)

print "Creating checkpoint files"
if (sort_file != ""):
    sort_re = LoadRE(sort_file)
    for i in sort_re:
        if (i[1].lower().find("bigfile_checkpoint") == 0):
            print "\tFound checkpoint: %s" % i[1]
            f = FixPath(src_path) + i[1]
            if (NeedsBuilt(f, f)):
                print "\t" + f
                fp = open(f, "wb")
                fp.close();

print "Scanning directory %s" % src_path
files = FindSourceFiles(src_path)
print "\t%d files found" % len(files)

print "Applying ignore rules"
ignored, files = StripFiles(files, ignore_re)
print "\t%d files ignored" % len(ignored)

print "Excluding files that will remain loose"
excluded, files = StripFiles(files, loose_re)
print "\t%d files are marked as loose files" % len(excluded)

missingLog = []
if (sort_file != ""):
    print "Applying sort order to files"
    files = SortFiles(files, sort_re)
else:
    print "Collecting patched files"
    files, missingLog = FilterPatchList(files, patch_file)
print "\t%d files for big files" % len(files)

# enlist unmatched patch file entries as missing files in the log file
MkDir(dest_path)
logfile_name = os.path.join(os.path.dirname(FixPath(dest_path)), "sol.log")
logfile = open(logfile_name, "wt")
if (patch_file != ""):
    logfile.write("Missing patch files:\n--------------------\n")
    for m in missingLog:
        logfile.write(m + "\n")
    logfile.write("\n")
logfile.close()

print "Scanning files for ones to compress"
uncompress, compress_files = StripFiles(files, uncompressed_re)
print "Number of compressable files %d out of %d" % (len(compress_files), len(files))

print "Building compression makefile"
if (compressor_path.find("chunkpack.exe") != -1):
    if (platform == "capilano"):
        compressor_opt = chunkpack_opt_capilano
    if (platform == "kettle"):
        compressor_opt = chunkpack_opt_kettle
    if (platform == "pc64"):
        compressor_opt = chunkpack_opt_pc64

number_files2compress = BuildCompressionMakefile(compress_files, tmp_path, compressed_path, compressor_path, compressor_opt, (4096 * 1024), 32)
print "\tnumber of files to compress %d" % number_files2compress
if (number_files2compress > 0):
    if (xgconsole_path != "" and remote_jobs > 0):
        MakeXGEProfile(tmp_path + "sol-profile.xml")

        fp = open(tmp_path + "makepack.bat", "wt")
        fp.write("%s -j %d --no-builtin-rules -f %sMakefile.pack" % (make_path, remote_jobs, tmp_path))
        fp.close()
        os.system('"%s" /command=%smakepack.bat /profile=%ssol-profile.xml' % (xgconsole_path, tmp_path, tmp_path))
    else:
        os.system("%s -j %d --no-builtin-rules -f %sMakefile.pack" % (make_path, local_jobs, tmp_path))

print "Reconciling compressed files"
ReconcileCompressedFiles(compress_files, compressed_path, src_path)

if (extra_file != "" and launch_file != ""):
    print "Filter files to create streaming install layout"
    launch_re = LoadRE(launch_file)
    extra_re = LoadRE(extra_file)
    files = FilterStreamingInstallFiles(files, launch_re, extra_re)
else:
    print "No extra or launch files specified. Skipping streaming install layout creation"

print "Building bigfiles, format %s, bighead index: %s" %(bigFormat, bigHead)
BuildBigFiles(files, dest_path, src_path, compressed_path, bigFormat, bigHead, file_reader)

print "Copying excluded loose files over"
ncopied = CopyExcludedFiles(excluded, dest_path, src_path, file_reader)
print "\t-- %d updated" % ncopied

endTime = time.time()

print ""
print "Complete, Time %g seconds" % (endTime - startTime)
print ""


