~jtv/corpusfiltergraph/cross-python

#! /usr/bin/env python

# -*- coding: utf8 -*-

 

#===============================================================================

# Author: Walapa Muangjeen

#===============================================================================

 

#version:

#4.0.264 - version update

 

import os

import sys

import string

import re

import common as cf

import logging

 

logger = logging.getLogger('.'.join([os.path.splitext(os.path.basename(sys.argv[0]))[0],'manager','filtergraph',__name__]))

skipclose = True

 

class filter(object):

 

        cfg = {

                'deleteline': False,

                'irrelevant': string.punctuation+string.whitespace+string.digits,

                'encoding': 'utf8',

                'inputfile': '',

                'outputfile': '',

                'version': '4.0.264',

                }

        deleteline = False

        irrelevant = string.punctuation+string.whitespace+string.digits

        encoding = 'utf8'

        inputfile = ''

        outputfile = ''

        isopen = False

        p = object

        errors = []

 

        def open(self,parent,cfg):

                self.encoding = 'utf8' if 'utf8' in cfg['encoding'].lower().replace('-','') else cfg['encoding']

                self.inputfile = cfg['inputfile'].replace('%(rootfolder)s',parent.rootfolder) if cfg['inputfile'] else self.inputfile

                self.outputfile = cfg['outputfile'].replace('%(rootfolder)s',parent.rootfolder) if cfg['outputfile'] else self.outputfile

                if (self.inputfile and not self.outputfile) or (not self.inputfile and self.outputfile):

                        self.errors.append([__name__,'invalid','[%s] inputfile=%s without outputfile= value'%(__name__,cfg['inputfile'])])

                        logger.warn('%s\t%s',*self.errors[-1][1:])

 

                self.deleteline = cfg['deleteline']

                self.irrelevant = string.punctuation+string.whitespace+string.digits+''.join([ char for char in cf.dedupelist(cfg['irrelevant']) if not char in string.punctuation+string.whitespace+string.digits ])

                self.regex = re.compile('[%s]'%re.escape(self.irrelevant))

        def run(self,k):

                global skipclose

                skipclose = not self.inputfile

                if self.inputfile: return

 

                if self.deleteline:

                        self.p.cfoutput[k]['tempbuff'] = [line for line in self.p.cfoutput[k]['tempbuff'] if len(self.regex.sub('', line))]

                else:

                        self.p.cfoutput[k]['tempbuff'] = [self.__filter(line) for line in self.p.cfoutput[k]['tempbuff']]

 

        def __filter(self,line):

                if not self.regex.sub('', line): line = ''

                return line

 

        def flush(self,k):

                return

 

        def close(self):

                if skipclose: return

 

                import codecs

 

                if not os.path.exists(self.inputfile):

                        self.errors.append([__name__,'missing','[%s] %s'%(__name__,self.inputfile)])

                        logger.error('%s\t%s',*self.errors[-1][1:])

                        return

 

                # make output folder

                try:

                        os.makedirs(os.path.dirname(self.outputfile))

                except OSError,e:

                        if not e.errno == 17:

                                logger.exception('%s\t%s, %s, %s',*['failed',e.errno,e.strerror,e.filename,])

                                raise OSError(e)

 

                # open input and output files

                out = self.outputfile

                if out == self.inputfile:

                        import tempfile

                        fd,out = tempfile.mkstemp(suffix='.tmp', prefix='~', dir=self.p.tempdir)

                        os.close(fd)

                try:

                        o = codecs.open(out,'w',self.encoding)

                        i = codecs.open(self.inputfile,'r',self.encoding)

                except:

                        raise RuntimeError('Failed to open [%s] input/output files'%(__name__))

 

                sys.stderr.write('[%s] %s\n   Please wait'%(__name__,self.outputfile))

                cnt = 0

                try:

                        # loop writes output line-by-line

                        for line in i:

                                o.write('%s\n'%(self.__filter(line.rstrip('\r\n'))))

                                cnt += 1

                                if not cnt%5000: sys.stderr.write('.')

                        sys.stderr.write('\n')

                        # close input and output files

                        i.close()

                        o.close()

 

                        if not out == self.outputfile:

                                import shutil

                                shutil.move(out,self.outputfile)

 

                except KeyboardInterrupt:

                        os.unlink(out)

                        raise KeyboardInterrupt()

 

                if not os.path.exists(self.outputfile):

                        self.errors.append([__name__,'missing','[%s] %s'%(__name__,self.outputfile)])

                        logger.error('%s\t%s',*self.errors[-1][1:])

 

def usage():

        '''Command prompt help.'''

        return "\n%s\n\tUsage:\n\tfrom %s import filter\n"%(

        os.path.basename(sys.argv[0]),

        os.path.splitext(os.path.basename(sys.argv[0]))[0]

        )

 

licensetxt=u'''CorpusFiltergraph™ v4.0

Copyright © 2010-2012 Precision Translation Tools Co., Ltd.

 

This program is free software: you can redistribute it and/or modify

it under the terms of the GNU Lesser General Public License as published by

the Free Software Foundation, either version 3 of the License, or

(at your option) any later version.

 

This program is distributed in the hope that it will be useful,

but WITHOUT ANY WARRANTY; without even the implied warranty of

MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the

GNU Lesser General Public License for more details.

 

You should have received a copy of the GNU Lesser General Public License

along with this program.  If not, see http://www.gnu.org/licenses/.

 

For more information, please contact Precision Translation Tools Co., Ltd.

at: http://www.precisiontranslationtools.com'''

 

if __name__ == "__main__":

        import os

        import sys

        sys.stdout.write(usage().encode('utf8')+'\n')