~jtv/corpusfiltergraph/cross-python

#! /usr/bin/env python

# -*- coding: utf8 -*-

 

#===============================================================================

# Author: Tom Hoar

#===============================================================================

 

#version:

#4.0.264 - version update

 

import sys

import os

import shutil

import codecs

from fnmatch import fnmatch

from flock import flock

import logging

import common as cf

 

logger = logging.getLogger('.'.join([os.path.splitext(os.path.basename(sys.argv[0]))[0],'manager','filtergraph',__name__]))

 

class writer(object):

 

        cfg = {

                'append': False,

                'deletezerolength': True,

                'encoding': 'utf8',

                'eol': 'linux',

                'reversecolumns': False,

                'rendertypes': '.tab',

                'rootfolder': '',

                'roottype': '',

                'safemode': False,

                'stage' : None,

                'version': '4.0.264',

                }

        append = 'w'

        deletezerolength = True

        encoding = 'utf8'

        eol = '\x0a'

        reversecolumns = False

        rendertypes = '.*'

        rootfolder = ''

        roottype = ''

        safemode = False

        stage = None

        isopen = False

        p = object

        errors = []

 

        def open(self,parent,cfg):

                if cfg['stage']:

                        self.stage = cfg['stage']

                else:

                        self.errors.append([__name__,'missing','[%s] \"stage=<missing>\"'%(__name__.split(',')[-1])])

                        logger.error('%s\t%s',*self.errors[-1][1:])

                if cfg['roottype']:

                        self.roottype = cfg['roottype'].strip(',').split(',')[0]

                        self.p.roottype = cfg['roottype'].strip(',').split(',')[0]

                else:

                        self.errors.append([__name__,'missing','[%s] \"roottype=<missing>\"'%(__name__.split(',')[-1])])

                        logger.error('%s\t%s',*self.errors[-1][1:])

                self.append = 'a' if cfg['append'] else 'w'

                self.deletezerolength = cfg['deletezerolength']

                self.encoding = 'utf8' if 'utf8' in cfg['encoding'].lower().replace('-','') else cfg['encoding']

                self.eol = '\x0d\x0a' if 'win' in cfg['eol'].lower() else '\x0a' if 'linux' in cfg['eol'].lower() or 'unix' in cfg['eol'].lower() or 'posix' in cfg['eol'].lower() else cfg['eol']

                self.reversecolumns = cfg['reversecolumns']

                self.rendertypes = os.extsep+cfg['rendertypes'].lstrip(os.extsep)

                self.rootfolder = self.normalizeroot(cfg['rootfolder'])

                self.safemode = cfg['safemode']

 

        def run(self,fileobj,k=None,mode=None):

                '''saves buffer to file'''

                # find srclang key

 

                # skip if k defined but not in fileobj

                if k and not k in fileobj: return -2

 

                keys = [k] if k else fileobj.keys()

 

                allkeys = list(keys)

 

                # remove empty output buffers

                for key in keys:

                        result = self.__removeempty(fileobj,key)

                        if result: return result

 

                # test if any removed buffers are 'tm' alignments and skip to avoid creating mis-aligned files

                for key in allkeys:

                        if not key in fileobj.keys() and len(key) > 3 and key[cf.kind] == 'tm':

                                result = -2

                if result: return result

 

                # rename output files and find srckey

                ktgt = []

                ksrc = None

                for k in fileobj.keys():

                        if not fnmatch(os.path.splitext(fileobj[key]['filelist'][cf.basename])[1],self.rendertypes):

                                fileobj[key]['filelist'][cf.basename] = fileobj[key]['filelist'][cf.basename]+self.rendertypes

                        if k[cf.rdrlang] == k[cf.srclang]:

                                ksrc = k

                        else:

                                ktgt.append(k)

                if not ksrc: return -2

 

                # shuffle buffers

                for k in ktgt:

                        if self.reversecolumns:

                                fileobj[k]['tempbuff'] = ['\t'.join([fileobj[k]['tempbuff'][linenum],fileobj[ksrc]['tempbuff'][linenum]]) for linenum in range(len(fileobj[ksrc]['tempbuff'])) if fileobj[ksrc]['tempbuff'][linenum] or fileobj[k]['tempbuff'][linenum]]

                        else:

                                fileobj[k]['tempbuff'] = ['\t'.join([fileobj[ksrc]['tempbuff'][linenum],fileobj[k]['tempbuff'][linenum]]) for linenum in range(len(fileobj[ksrc]['tempbuff'])) if fileobj[ksrc]['tempbuff'][linenum] or fileobj[k]['tempbuff'][linenum]]

                # dump ksrc buffer to prevent saving file

                fileobj[ksrc]['tempbuff'] = ['' for line in fileobj[ksrc]['tempbuff']]

 

                result = self.__save(fileobj,key,mode) or result

 

                return result

 

        def flush(self,fileobj,k=None):

                '''move temp output files to server'''

                result = None

                if not self.null:

                        if k:

                                result = -2 if self.__mkfolder(fileobj,k) else self.__push(fileobj,k)

                        else:

                                # create destination folders (test write access)

                                for k in fileobj.keys():

                                        result = self.__mkfolder(fileobj,k) or result

                                # move tempfile to output path

                                if not result:

                                        for k in fileobj.keys():

                                                result = self.__push(fileobj,k) or result

                for k in fileobj.keys():

                        del(fileobj[k])

                return result

 

        def close(self):

                return

 

        def getroot(self,rootfolder):

                '''Splits rootfolder to file system and rootfolder'''

                folder = os.path.abspath(os.path.expanduser(rootfolder.rstrip().replace('/',os.sep).replace('\\',os.sep)))

                if fnmatch(folder,'?:*'):

                        # DOS drive letter : folder

                        fss, folder = folder.split(':')

                        fss = fss + ':'

                elif fnmatch(folder,'*:*'):

                        # ssh hostname : folder

                        fss, folder = folder.split(':')

                        fss = fss + ':'

                elif fnmatch(folder,'\\\\*\\*'):

                        # MS Windows UNC \ folder

                        fss = '\\\\'+folder.lstrip(os.sep).split(os.sep)[0]

                        folder = folder.replace(fss,'')

                else:

                        # NFS mount / folder

                        fss = ''

                        folder = os.path.normpath(folder)

                        if folder == '.': folder = '*'

                folder = folder.lstrip('\\/')

                return [unicode(fss), unicode(folder)]

 

        def normalizeroot(self,rootfolder):

                return rootfolder.replace('\\',os.sep).replace('/',os.sep)

 

        def __save(self,fileobj,k,mode):

                mode = self.append if not mode else mode

                try:

                        f = codecs.open(fileobj[k]['tempname'],mode,self.encoding)

                        f.write('%s%s'%(self.eol.join(fileobj[k]['tempbuff']),self.eol))

                        f.close()

                except Exception,e:

                        logger.exception('%s\t%s - %s',*['failed',self.p.cfinput[k]['tempname'],str(e)])

                        return -2

                logger.debug('%s\t%s',*['file',fileobj[k]['tempname']])

 

        def __removeempty(self,fileobj,k):

                if os.path.exists(os.sep.join(fileobj[k]['filelist'])) and self.safemode:

                        logger.warn('%s\t%s - %s',*['skip','output already exists',os.sep.join(fileobj[k]['filelist']),])

                        del(fileobj[k])

                        return -2

                elif not sum([len(line.strip()) for line in fileobj[k]['tempbuff']]):

                        if not 'workbench' in fileobj[k]['filelist'][cf.stage] and not 'extract' in fileobj[k]['filelist'][cf.stage]:

                                logger.warn('%s\t%s - %s',*['skip','empty data',fileobj[k]['tempname']])

                        if os.path.exists(os.sep.join(fileobj[k]['filelist'])) and self.deletezerolength:

                                os.unlink(os.sep.join(fileobj[k]['filelist']))

                                try:

                                        os.removedirs(os.path.dirname(os.sep.join(fileobj[k]['filelist'])))

                                except OSError,e:

                                        if not e.errno == 39:

                                                logger.exception('%s\t%s, %s, %s',*['failed',e.errno,e.strerror,os.sep.join(fileobj[k]['filelist']),])

                        del(fileobj[k])

 

        def __mkfolder(self,fileobj,k):

                if os.path.exists(fileobj[k]['tempname']):

                        try:

                                os.makedirs(os.path.dirname(os.sep.join(fileobj[k]['filelist'])))

                        except OSError,e:

                                if not e.errno == 17:

                                        logger.exception('%s\t%s, %s, %s',*['failed',e.errno,e.strerror,e.filename,])

                                        return -2

 

        def __push(self,fileobj,k):

                if not k in fileobj: return

                if not fnmatch(os.path.splitext(fileobj[k]['filelist'][cf.basename])[1],self.rendertypes):

                        fileobj[k]['filelist'][cf.basename] = fileobj[k]['filelist'][cf.basename]+self.rendertypes

                if os.path.exists(fileobj[k]['tempname']):

                        try:

                                shutil.move(fileobj[k]['tempname'],os.sep.join(fileobj[k]['filelist']))

                        except:

                                return -2

                        else:

                                logger.debug('%s\t%s',*['file',os.sep.join(fileobj[k]['filelist'])])

 

def usage():

        '''Command prompt help.'''

        return "\n%s\n\tUsage:\n\tfrom %s import reader\n"%(

        os.path.basename(sys.argv[0]),

        os.path.splitext(os.path.basename(sys.argv[0]))[0]

        )

 

licensetxt=u'''CorpusFiltergraph™ v4.0

Copyright © 2010-2012 Precision Translation Tools Co., Ltd.

 

This program is free software: you can redistribute it and/or modify

it under the terms of the GNU Lesser General Public License as published by

the Free Software Foundation, either version 3 of the License, or

(at your option) any later version.

 

This program is distributed in the hope that it will be useful,

but WITHOUT ANY WARRANTY; without even the implied warranty of

MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

GNU Lesser General Public License for more details.

 

You should have received a copy of the GNU Lesser General Public License

along with this program.  If not, see http://www.gnu.org/licenses/.

 

For more information, please contact Precision Translation Tools Co., Ltd.

at: http://www.precisiontranslationtools.com'''

 

if __name__ == "__main__":

        import os

        import sys

        sys.stdout.write(usage().encode('utf8')+'\n')