#!/usr/bin/env python
# JV website generator
# (c) 2008, Johan Tuitman
# Licentie: GPL
#
# Hoofd programma voor generatie van website
import cPickle
import os
import sys
import dataStruct
import shutil
import re
import genSubs
from ftplib import FTP

class genjvweb:
    def __init__(self):
        # Regeneration of website off by default,
        # check command line argument
        self.reGen = False
        if len(sys.argv) > 1:
           if sys.argv[1].lower() == '-r':
               self.reGen = True

        print 50*'-'
        print 'JV website generator'
        print 50*'-'
        print 'Lees databases'

        self.readData()
        self.wd = self.db.getWorkDir()
        self.sd = os.path.normpath(os.path.dirname(self.db.getIndexPage().getUrl()))
        # If regeneration mode, delete old files
        if self.reGen:
            if os.path.exists(self.wd):
                shutil.rmtree(self.wd)


        # No old files? Regeneration mode on
        if not os.path.exists(self.wd):
            os.mkdir(self.wd)
            self.reGen = True

        if self.reGen:
            print 50*'-'
            print 'Waarschuwing: website wordt opnieuw opgebouwd.'
            print 'Verwijder alle files van de ftp-server' 
            raw_input('Druk op enter om verder te gaan')

        self.newDirs = []
        # Generate menu structure
        self.mkm = genSubs.mkMenus(self.reGen,self.db,self.db_o)

        self.mkp = genSubs.mkPict(self.reGen,self.db,self.db_o)

        self.sourcePages = self.processPages()

        self.sourcePages.update(self.walkThirthDir())

        self.presentPages = self.getPresentFiles()

        self.changedFiles, self.delFiles = self.getChanges()

        self.processFiles(self.changedFiles)

        self.delDirs = self.delFilesAndDirs(self.delFiles)

        self.ftpFiles()

        shutil.copyfile('jvwebsite.db','jvwebsite_prev.db')

        print 50*'-'
        print 'Klaar'

    def delFilesAndDirs(self,delFiles):
        delDirs = []
        for fl in delFiles:
            os.remove(os.path.join(self.wd,fl))
            
        FoundDir = True
        while FoundDir:
            FoundDir = False
            for root, dirs, files in os.walk(self.wd,topdown=False):
                if len(dirs) == 0 and len(files) == 0:
                    rt = root.replace(self.wd,'')
                    delDirs.append(str(rt))
                    os.rmdir(os.path.join(self.wd,root))
                    FoundDir = True

        return delDirs

    def walkThirthDir(self):
        flst = {}
        flist = self.sourcePages.keys()

        picDirs = set()
        for fl in self.mkp.getPicFiles():
            picDirs.add(os.path.dirname(fl))

        for fl in flist:
            if genSubs.countPath(fl)>1:
                bp = os.path.dirname(fl)
                if bp not in picDirs:
                    del self.sourcePages[fl]

                    for root, dirs, files in os.walk(os.path.join(self.sd,bp)):
                        rt = root[len(self.sd)+1:]
                        for f in files:
                            fn = os.path.join(rt,f)
                            flst[fn]=os.path.getmtime(os.path.join(root,f))
        return flst

    def ftpFiles(self):
        print 50*'-'
        print 'ftp:'
        # append newdir list with menu dirs
        files = self.mkm.getMenuCFiles()+self.mkp.getPicCFiles()
        nd = []
        for fl in files:
            if os.path.isdir(os.path.join(self.wd,fl)):
                self.newDirs.append(fl)

        changed = set(self.changedFiles)
        changed.update(self.mkm.getMenuCFiles()+self.mkp.getPicCFiles())
        changed = list(changed-set(self.newDirs))

        # connect
        ftpd = self.db.getFTP()
        try:
            self.ftp = FTP(ftpd.getSite())
        except:
            print 'fout bij het connecteren van "'+ftpd.getSite()+'".'
            self.ftpErrorSync(0,0,changed)
            return

        # login
        try:
            self.ftp.login(ftpd.getUser(),ftpd.getPasswd())
        except:
            print "Fout bij het inloggen. Controleer je inlog gegevens."
            self.ftpErrorSync(0,0,changed)
            return

        # correcte passieve mode
        self.ftp.set_pasv(True)

        # naar dir
        try:
            self.ftp.cwd(ftpd.getDirectory())
        except:
            print"Fout bij het veranderen naar werk map"
            self.ftpErrorSync(0,0,changed)
            return

        try:
            for i, dr in enumerate(self.sortDirs(self.newDirs)):
                print 'create dir', dr
                self.ftp.mkd(dr)
        except:
            print"Fout bij het maken van de directories"
            self.ftpErrorSync(1,i,changed)
            return


        try:
            for i, fl in enumerate(changed):
                print 'upload:', fl
                f = open(os.path.join(self.wd,fl),'rb')
                self.ftp.storbinary('STOR '+fl,f)
                f.close()

        except:
            print"Fout bij het uploaden"
            self.ftpErrorSync(2,i,changed)
            return


        try:
            for i, fl in enumerate(self.delFiles):
                print 'delete:', fl
                self.ftp.delete(fl)
        except:
            print"Fout bij het verwijderen van de files."
            self.ftpErrorSync(3,i,changed)
            return

        try:
            for i, dr in enumerate(self.sortDirs(self.delDirs,True)):
                print 'delete dir', dr
                self.ftp.rmd(dr)

        except:
            print"Fout bij het verwijderen de directories."
            self.ftpErrorSync(4,i,changed)
            return

        self.ftp.close()

    def ftpErrorSync(self,lv,item,changed):
            # keep sync
            if lv == 4:
                i = item
            else:
                i = 0

            drl = self.sortDirs(self.delDirs,True)[i:]
            for dr in drl[::-1]:
                os.mkdir(os.path.join(self.wd,dr))

            if lv > 3:
                return
            elif lv == 3:
                i = item
            else:
                i = 0

            fll = self.delFiles[i:]
            for fl in fll:
                f = open(os.path.join(self.wd,fl),'w')
                f.write('dummy')
                f.close()

            if lv > 2:
                return
            elif lv == 2:
                i = item
            else:
                i = 0

            chl = changed[i:]
            for fl in chl:
                os.remove(os.path.join(self.wd,fl))

            if lv > 1:
                return
            elif lv == 1:
                i = item
            else:
                i = 0
            drl = self.sortDirs(self.newDirs)[i:]
            for dr in drl[::-1]:
                os.rmdir(os.path.join(self.wd,dr))

    def sortDirs(self,dirl,rev=False):
        dird = {}
        dirs = []
        mxd = 0
        for dr in dirl:
            dird[dr] = genSubs.countPath(dr)
            if dird[dr] > mxd:
                mxd = dird[dr]

        for i in range(mxd+1):
            for dr in dirl:
                if dird[dr] == i:
                    dirs.append(dr)

        if rev:
            dirs = dirs[::-1]

        return dirs

    def processFiles(self,changedFiles):
        print 50*'-'
        print 'Veranderde bestanden:'
        for fl in changedFiles:
            print fl
            self.checkDir(fl)

            if fl.endswith('.htm') or  fl.endswith('.html'):
                self.processFile(fl)

            else:
                fli = os.path.join(self.sd,fl)
                flo = os.path.join(self.wd,fl)
                shutil.copyfile(fli,flo)

    def checkDir(self,dr):
        """check if dir exsist, if not created it"""
        drn = os.path.dirname(dr)
        i = 0
        while len(drn) > 0:
            if os.path.exists(os.path.join(self.wd,drn)):
                if i == 0:
                    return
                else:
                    break
            else:
                i += 1
                drn=os.path.dirname(drn)
                    
        for j in range(i):
            drn = os.path.dirname(dr)
            for k in range(i-j-1):
                drn = os.path.dirname(drn)

            os.mkdir(os.path.join(self.wd,drn))
            self.newDirs.append(drn)

    def processFile(self,fl):
        mpages = self.mkm.getMPages()
        if mpages.has_key(fl):
            mp = True
            Menu = mpages[fl].getMenu()
            subMenu = mpages[fl].getSMenu()
            page = mpages[fl].getPage()

        else:
            mp = False

        fli = os.path.join(self.sd,fl)

        flo = os.path.join(self.wd,fl)
        fin = open(fli)
        txtin = fin.readlines()
        fin.close()
        fout = open('temp.htm','w')
        findDocType = False
        i = 0
        while re.search('^ *.?$', txtin[i]) and len(txtin)-1> i:
            i += 1

        if txtin[i].find('DOCTYPE') < 0:
            for cline in genSubs.pageHeader(fl):
                fout.write(cline)

            if mp:
                bspath = os.path.dirname(self.db.getPage(Menu).getUrl())

                name = self.db.getPage(Menu).getFullName()

                # header
                sth = '<h1>%s</h1>\n <hr>\n ' %name
                if len(self.db.getSubMenus(Menu)) > 0:
                    sth += self.mkm.createTextMenu(Menu,subMenu)
                if subMenu != None:
                    sth += '<h2>%s</h2>\n' %self.db.getSubPage(Menu,subMenu).getFullName()

                fout.write(sth)


        for cline in txtin:
            fout.write(cline)

        fout.close()

        os.system('tidy -q -o %s temp.htm' %flo)

    def getChanges(self):
        sourceSet = set(self.sourcePages.keys())
        presentSet = set(self.presentPages.keys())
        menuFilesSet = set(self.mkm.getMenuFiles()+self.mkp.getPicFiles())
        menuHtmlSet = set(self.mkm.getMenuCHtml())

        delFiles = list(presentSet-sourceSet-menuFilesSet)
        changedFiles = set(sourceSet-presentSet)
        changedFiles.update(menuHtmlSet)

        for page in list(sourceSet&presentSet):
            if self.sourcePages[page] > self.presentPages[page]:
                changedFiles.add(page)

        return list(changedFiles), delFiles

    def getPresentFiles(self):
        ppage = {}
        for root, dirs, files in os.walk(self.wd):
            rt = root[len(self.wd):]
            for f in files:
                fn = os.path.join(rt,f)
                ppage[fn]=os.path.getmtime(os.path.join(root,f))
        return ppage

    def processPages(self):
        pages = {}
        pgl = self.mkm.getMPages().keys()

        for page in pgl:
            pg = self.mkm.getMPages()[page].getPage()
            if page.endswith('.htm') or page.endswith('.html'):
                if pg.getProcess() and pg.getSubProcess():
                    pages.update( self.processPage(page))
                elif pg.getProcess():
                    pages.update(self.processPage(page,SubP=False))

        for page in self.mkm.getAddPages():
            if page.endswith('.htm') or page.endswith('.html'):
                pages.update(self.processPage(page))
            else:
                pages.update(self.processPage(page,SubP=False))

        return pages

    def processPage(self,page,hasPr=[],SubP=True):
        ppage = {}
        menuFiles = self.mkm.getMenuFiles()+self.mkp.getPicFiles()
        errorNotFound = True
        sd = self.sd
        dsPage = os.path.join(sd,page)
        direc = os.path.dirname(page)


        if self.checkPage(dsPage,errorNotFound,hasPr):
            ppage[page] = os.path.getmtime(dsPage)

            if SubP:
                f = open(dsPage)
                txt = f.readlines()
                f.close()

                for cline in txt:
                    tags = re.findall('<[^/].*?>',cline)
                    for t in tags:
                        tt = t.lower()
                        npage = None
                        if re.search(r'a +href *= *["\'][^"\']+["\']',tt):
                            npage = re.search(r'a +href *= *["\']([^"\']+)["\']',tt).group(1)
                        elif re.search(r'frame.*src *= *["\'][^"\']+["\']',tt):
                            npage = re.search(r'frame.*src *= *["\']([^"\']+)["\']',tt).group(1)
                        if npage != None:
                            if npage.find(':') < 0:
                                dnpage = os.path.normpath(os.path.join(direc,npage))
                                if (dnpage not in menuFiles) and (dnpage not in hasPr):
                                    if genSubs.countPath(dnpage) <2:
                                        ppage.update(self.processPage(dnpage,ppage.keys()))
                                    else:
                                        ppage.update(self.processPage(dnpage,ppage.keys(),False))

                        if re.search(r'img +.*src *= *["\'][^"\']+["\']',tt):
                            npic = re.search(r'img +.*src *= *["\']([^"\']+)["\']',tt).group(1)
                            if npic.find(':') < 0:
                                dnpic = os.path.normpath(os.path.join(direc,npic))
                                if self.checkPage(os.path.join(sd,dnpic),errorNotFound,hasPr+[dsPage]):
                                    ppage[dnpic] = os.path.getmtime(os.path.join(sd,dnpic))
                    
        return ppage

    def checkPage(self,page,errorNotFound,backTrace):
        if os.path.exists(page):
            return True
        else:
            if errorNotFound:
                print "Bestand '"+page+"' nodig voor het maken van van de website ontbreekt."
                if len(backTrace)>0:
                    print "Bestandlijst:"
                    for i, fl in enumerate(backTrace):
                        print '%i. %s' %(i, fl)

                #sys.exit(1)
            else:
                return False

    def readData(self):
        """Lees database en ook de oude"""
        try:
            f = open('jvwebsite.db')
            self.db = cPickle.load(f)
            f.close()
        except:
            print 'Fout bij lezen database "jvwebsite.db".'
            sys.exit(1)

        self.db_o = False
        if not self.reGen:
            if os.path.exists('jvwebsite_prev.db'):
                try:
                    f = open('jvwebsite_prev.db')
                    self.db_o = cPickle.load(f)
                    f.close()
                except:
                    print 'Fout bij lezen database "jvwebsite_prev.db".'
                    sys.exit(1)
            else:
                self.reGen = True


if __name__ == '__main__':
    genjvweb()