#!/usr/bin/python
# -*- coding: utf-8  -*-
#
#v.0104
"""
Checks Translation tables in wiktionaries that use the schema:
{{translationtablestart|optionalexplanation}}
* {{isocode}} : {{templateforlanguage|isocode|translation}}
{{translationtablemiddle}}
{{translationtableend}}

Possibly using some template for redirecting translations

Creates sorted translation tables plus one new which will include all problematic
(if such table does not already exists).

If table is corrupted reports only the problem and returns!

Use to check something in a dump.
"""
#TODO 3. get language from some conf file or, better, from dump's xml first tag . 

import os
import sys
import simplexmlreader as xmlreader
import codecs
import argparse
#import clsfetchnewdump
from collections import OrderedDict

fproblematic=codecs.open('W_problematictable', 'w', 'utf-8')


mylang = u'el'
myproject = u'wiki'

#mydumpspath = u'/home/wiki/Λήψεις/'
mydumpspath = u'/media/FORMANY/wiki/dumps/'


RETURNPROBLEMATIC = 1
RETURNOK = 0 

atona = u'αεηιουωϊϋ'#.split(',')
entona = u'άέήίόύώΐΰ'#.split(',')
#general function
def extract_lang_iso(wholeline):
    """Extracts possible iso code from translations
    
        ISO code is the first found "Template" in line.
    """
    return wholeline.split('}}')[0].split('{{')[1]

def cleartranslationtable(whichtable):
    '''
    Deletes all isos without translation (hidden or visible)
    from a sorted dict
    '''
    for x in whichtable:
        if whichtable[x] == '':
            del whichtable[x]
    return whichtable

def writeunfinished(lemma):
    with codecs.open('W_unfinished', 'a', 'utf-8') as f:
        f.write(lemma + u'\n')
        
def extractsomething2(whichlemma, whichtext):#this function is constantly changed to extract something 
    a = whichlemma.decode('utf-8')
    b = whichtext.decode('utf-8')
#        _text = b.replace('>', '>')
#        _text = _text.replace('<', '<')
#        _text = _text.replace('"', '"')
#        b = _text.replace('&', '&') #must be last 
    if u'Κατηγορία:Κατάλογοι που αφορούν την Κύπρο' in b:
        with codecs.open('ΚατΚύπρ.txt', 'a', 'utf-8') as f:
            f.write( a + u'\n')
        #print a
        return 1
    return 0

def extractsomething4(whichlemma, whichtext, lemmas):
    #βρες τα θηλυκά σε "-μένη" που δεν έχουν "μορφή μετοχής"
    #φυσικά... θα φέρει και όσα έχουν "μορφή επιθέτου"
    a = whichlemma.decode('utf-8')
    b = whichtext.decode('utf-8')
    exeiepitheto = u'*0\t'
    if a.endswith(u'μένη'):
        if u'==={{μορφή μετοχής|el}}===' not in b:
            c = a[:len(a)-1]+u'ος'#το παροξύτονο αρσενικό του
            #print c
            #αλλά μπορεί το αρσενικό του να είναι προπαροξύτονο
            d = a[:len(a)-4]
            if u'==={{μορφή επιθέτου|el}}===' in b:
                exeiepitheto = u'*1\t'
            for xcounter in range(3):
                #ποιο γράμμα να ελέγξουμε
                lettertocheck = len(d) - xcounter 
                if d[lettertocheck-1:lettertocheck] in atona:
                    #print lettertocheck
                    #αν έχει φωνήεν (άτονο φυσικά)
                    #φέρε το αντίστοιχο τονισμένο
                    newletter = atona.find(d[lettertocheck-1:lettertocheck])
                    #αντικατέστησέ το και φτιάξε το "πιθανό προπαροξύτονο αρσενικό" της λέξης
                    d = d[:lettertocheck-1] + entona[newletter:newletter+1] + d[lettertocheck+1:] + u'μενος'
                    #βρήκαμε το φωνήεν
                    break
                    #print d
                    #exit()
            if c in lemmas:#αν υπάρχει το παροξύτονο αρσενικό δεν θα ελεγχθεί η ύπαρξη προπαροξύτονου
                with codecs.open('θηλμετοχ.txt', 'a', 'utf-8') as f:
                    f.write(exeiepitheto + u'1\t[[' + a + u']]\t' + c + u'\n')
                    return 1                    
            if d in lemmas:#αν υπάρχει το προπαροξύτονο αρσενικό
                with codecs.open('θηλμετοχ.txt', 'a', 'utf-8') as f:
                    f.write(exeiepitheto + u'2\t[[' + a + u']]\t'+ d + u'\n')
                    return 1
            with codecs.open('θηλμετοχ.txt', 'a', 'utf-8') as f:
                #δεν υπάρχει ούτε παροξύτονο ούτε προπαροξύτονο
                #Μπορεί πράγματι να μην υπάρχει (π.χ. "Ωραία Κοιμωμένη"),
                #αλλά μπορεί απλά να μην έχει καταχωρηθεί.
                #θα μπορούσαν να μπουν σε διαφορετικό αρχείο αλλά είναι λίγα τελικά...                
                f.write(exeiepitheto + u'3\t[[' + a + u']]\n')
                #print d
                #exit()
                return 1                        
            #print a
            #return 1
    return 0
    

def extractsomething3(whichlemma, whichtext):#this function is constantly changed to extract something 
    a = whichlemma.decode('utf-8')
    b = whichtext.decode('utf-8')
    #c = langsortdata.el.mappedletters
    langs = {}
    langssplit1 = b.split('-}}==\n')
    try:
        if len(langssplit1)>1:
            prevlang = langssplit1[0].split('=={{-')[1]
            prevtext = u''
            #print a
            #print langssplit1 
            #for text in langssplit1:            
            for xcounter in range(1,len(langssplit1)-1):
                splitted = langssplit1[xcounter].split('\n=={{-')
                #print splitted
                langs[prevlang] = splitted [0] 
                prevlang = splitted[1]
            #for lang in langs:
                #print lang
            return 0, ''
        elif len(langssplit1) == 0:
            return 1, a + u'\tno lang\n'
        else:
            if b.startswith(u'#ΑΝΑΚΑΤΕΥΘΥΝΣΗ') or b.startswith(u'#REDIRECT'):            
                with codecs.open(fredirect, 'a', 'utf-8') as f:
                    f.write( a + u'\t' + b.split('\n')[0] + u'\n')
            else:
                with codecs.open(fother, 'a', 'utf-8') as f:
                    f.write( u'* [[' + a + u']]\t#' + b.split('\n')[0] + u'#\n')            
            return 0, ''
    except IndexError:
        return 1, a + u'\tindex error\n'

        
def extractsomething(whichlemma, whichtext):#this function is constantly changed to extract something 
    a = whichlemma.decode('utf-8')
    b = whichtext.decode('utf-8')
    if u'{{język starogrecki' in b:
#        _text = b.replace('>', '>')
#        _text = _text.replace('<', '<')
#        _text = _text.replace('"', '"')
#        b = _text.replace('&', '&') #must be last 
        if u'[[el:' + a + u']]' not in b:
            with codecs.open('noiwel.txt', 'a', 'utf-8') as f:
                f.write(a + u'\n')
            print a
        if u'{{język nowogrecki' not in b:
            with codecs.open('noelinpl.txt', 'a', 'utf-8') as f:
                f.write(a + u'\n')
            return 1
    return 0
        
class LangStrings():
    '''Reads strings of translation table for the language.
    
        Creates a sorted dict with iso codes of all known langs
        to use with translation table.
    '''
    def __init__(self, whichlang):    
        self.langlangs = __import__('wiktlangdata.%s' % whichlang, fromlist=['wiktlangdata'])  
        self.whichlang = whichlang      
        self.langsort = __import__('sorting')
        self.sortednames = []
        self.sortedisos = []

    def isocodeisknown(self, isocode):
        return (isocode in self.langlangs.langnames)
          
    def startoftablestring(self):
        return self.langlangs.startoftablestring
                
    def middleoftablestring(self):
        return self.langlangs.middleoftablestring        
        
    def endoftablestring(self):
        return self.langlangs.endoftablestring        
        
    def redirectionsstring(self):
        return self.langlangs.redirectionsstring        
        
    def problemtablestring(self):
        return self.langlangs.problemtablestring        
        
    def isosinorder(self):
        '''Creates an ordered dict with all known iso codes
        in specified language with empty values for each one of them.                
        
            Time consuming....
        '''        
        b = OrderedDict(sorted(self.langlangs.langnames.items(), key=lambda t: self.langsort.Word(self.whichlang, self.langlangs.langnames[t[0]])))
        #for x in b:
            #print b[x] 
        #exit()
        #clear values
        for x in b:
            b[x] = u''
        self._readydict = b
        #for x in self._readydict:
            #self._readydict[x] = 0
        #return empty dictionary with lang codes sorted by language names
        return b
     
class ProcessTranslationsOfLemma():
    def __init__(self, whichlang, whichlemma, whichtext,usingLangStrings):    
        """ Constructor
                        
        """   
        self.ls =  usingLangStrings    
        
        self.lemma = whichlemma.decode('utf-8')
        _text = whichtext.decode('utf-8')
        # unescape characters
        _text = _text.replace('>', '>')
        _text = _text.replace('<', '<')
        _text = _text.replace('"', '"')
        _text = _text.replace('&', '&') #must be last 
        _trstart = _text.find(self.ls.startoftablestring())
        self.notranslationsstart = False
        if _trstart > 0 :
            _text =_text[_trstart:] 
        else:
            if _text.find(self.ls.middleoftablestring()) > -1 or _text.find(self.ls.endoftablestring()) > -1 :
                pass #no start of tables but middle or end
            else:
                self.notranslationsstart = True
                #print _text
                #exit()           
        self.lines = _text.splitlines()
        self.tablescounter = 0 #will hold the number of translation tables found
        self.translations = {} #will hold which is the "title" of every translation table found
        self.alltraslationtables = {} #
        self.onetranslationtable = {} #will hold contents of one found traslation table
                                    #iso code as key and full line as value

        self.doubletranslation = False
        self.started = False
        self.tablewithproblems = {} #single table that will hold all problems found in all tables
        self.problemsfoundintable = 0

    def _processpossibletranslation(self, oneline):
        """Checks if a translation line for the ISO lang exists and if is sorted
        
            Assumes a line with hidden or normal translation
        """
        langISO = extract_lang_iso(oneline)
        if self.ls.isocodeisknown(langISO):
            if langISO in self.onetranslationtable:
                #print 'double: ',langISO, '  (' , self.lemma,')'
                self.problemsfoundintable += 1
                #self.tablewithproblems[u'Pr-Dbl-' + unicode(self.problemsfoundintable)] = oneline #debuging problems
                self.tablewithproblems[ unicode(self.problemsfoundintable)] = oneline
            else:            
                self.onetranslationtable[langISO] = oneline 
                #if oneline.startswith(u'*'):
                    #self.ls._readydict[langISO] += 1
        else:
            #print 'unknown iso: ',langISO, '  (' , self.lemma,')'
            self.problemsfoundintable += 1
            #self.tablewithproblems[u'Pr-iso-' + unicode(self.problemsfoundintable)] = oneline #debuging problems
            self.tablewithproblems[unicode(self.problemsfoundintable)] = oneline
            
    def _isdummyline(self, oneline):
        if oneline.startswith(self.ls.middleoftablestring()):#center of table
            return True
        elif oneline.strip() == '':#empty line
            return True
        else:
            return False                
        
    def findandchecktranslations(self):
        """Checks translations of one entry for errors.
        
            Processes lines. Tables are contained in a "self" dict named alltraslationtables.
            Their keys are numbers from counter.
            Each item of the dict contains a tuple with title found, another dict with translations
            and another dict with problems.
            Dict with translations contains lines of that table and iso codes as keys.
            Since redirections do not have isocode
            any "table" dict that has a redirection contains only a key '0' and the line.
            Dict with problems also does not have isocodes so his members are counted
            and counter is used as key.
        """ 
        #if self.notranslationsstart:
            #return 
        tablesfound = 0
        tabletitle = u'' 
        self.problemsfoundintable = 0 
        self.alltraslationtables = {}
        self.tablewithproblems = {}
        self.onetranslationtable = OrderedDict()
        #only one line with redirection
        self.redirectionstr = u''
        redirectfound = False
        if self.notranslationsstart:
            return self.alltraslationtables
        for oneline in self.lines:
            if oneline.startswith(self.ls.startoftablestring()):#start of a translation table
                if self.started:#report problem and return. Do not search that title untill problem is fixed
                    #fproblematic.write(thislemma)
                    #print 'table not ended________________ ', '  (' , self.lemma,')'
                    with codecs.open('unfinished.txt', 'a', 'utf-8') as fproblem:
                        fproblem.write(self.lemma + u'\n')
                    return RETURNPROBLEMATIC                    
                tablesfound += 1 #increase table counter
                tabletitle =  oneline #table "title"
                self.problemsfoundintable = 0 #restart counting problems
                self.started = True #set that a table has begun
                self.onetranslationtable = OrderedDict() #clear translation table
                self.tablewithproblems = {} #clear table with probles for the above translation table
                self.redirectionstr = u''
                redirectfound = False
            #TODO 
            #elif oneline.startswith(self.ls.problemtablestring):#start of a table with problems
                
            elif oneline.startswith(self.ls.endoftablestring()):#found end of table            
                if not self.started:#report problem and return. Do not search that title untill problem is fixed
                    #fproblematic.write(thislemma)
                    print 'table not started.......... ', '  (' , self.lemma,')'
                    with codecs.open('notstarted.txt', 'a', 'utf-8') as fproblem:
                        fproblem.write(self.lemma + u'\n')
                    return RETURNPROBLEMATIC 
                else:#end of that traslation table
                    #add the table to translation tables
                    if len(self.tablewithproblems):
                        b = self.ls.problemtablestring, self.tablewithproblems
                    else:
                        b = None
                    if redirectfound:
                        c = self.redirectionstr
                    else:
                        c = None
                    #tablesorted = self._sorttables()                    
                    self.alltraslationtables[unicode(tablesfound)] = tabletitle, self.onetranslationtable, b, c
                    #not inside a table after here
                    self.started = False                
            elif self.started:
                #TODO change it to specific lang start
                if oneline.startswith(u'* {{') or oneline.startswith(u'<!-- * {{'):
                    #possible translation visible or hidden
                    self._processpossibletranslation(oneline)
                elif self._isdummyline(oneline):
                    #nothing to do. Do not add that line.
                    continue
                elif oneline.startswith(self.ls.redirectionsstring()):
                    #line with redirection
                    if redirectfound:
                        print 'table with at least two redirections________________ ', '  (' , self.lemma,')'
                        with codecs.open('tworedirs.txt', 'a', 'utf-8') as fproblem:
                            fproblem.write(self.lemma + u'\n')
                        return RETURNPROBLEMATIC                    
                    else:
                        redirectfound = True
                    #add the line
                    self.redirectionstr = oneline                   
                else: #problematic line, add it to problematic table
                    #increase problem counter for that translation table
                    self.problemsfoundintable += 1
                    #self.tablewithproblems[u'Pr-other-' + unicode(self.problemsfoundintable)] = oneline #debuging problems
                    self.tablewithproblems[unicode(self.problemsfoundintable)] = oneline
            else: #not inside a table
                pass

        #all lines searched
        if self.started:
            #last table has not been closed
            #print 'table not ended________________ ', '  (' , self.lemma,')'
            with codecs.open('unfinished.txt', 'a', 'utf-8') as fproblem:
                fproblem.write(self.lemma + u'\n')
            return RETURNPROBLEMATIC                    
        return self.alltraslationtables
        

def checktranslation(langofwiktionary, lemma, lemmatext, usingLangStrings):
    #print "Working on..." , lemma                      
    b = usingLangStrings
    #get sorted isocodes
    #emptysortedisos = b.isosinorder() 
    #exit() 
    mycenter = b.middleoftablestring()
    myend = b.endoftablestring()                
    m = ProcessTranslationsOfLemma(langofwiktionary, lemma, lemmatext, b)
    lemmaalltables = m.findandchecktranslations()
    if lemmaalltables == RETURNPROBLEMATIC:
        print 'problematic tables in ', lemma
    elif len(lemmaalltables):
        #print '============================ START OF TABLES FOR: ', lemma , ' =============================='
        for tablecounter, onetable in enumerate(sorted(lemmaalltables)):                                                   
            #print '===== Table num: ', tablecounter + 1, '====='
            tabletitle = lemmaalltables[onetable][0]
            unsortedtablewithtranslations = lemmaalltables[onetable][1]
            hasproblems = (lemmaalltables[onetable][2] != None)
            if hasproblems:
                tablewithproblemsstring = lemmaalltables[onetable][2][0]
                tablewithproblems = lemmaalltables[onetable][2][1]
            tablewithredirect = lemmaalltables[onetable][3]
            print 'Tablestart: '
            middleoftable = - (len(unsortedtablewithtranslations) / -2)
            linesprintedcounter = 0
            print tabletitle 
            #sort table and insert Template for middle of table
            sortedtable = OrderedDict(b._readydict)
            unsortedtablewithtranslations = cleartranslationtable(unsortedtablewithtranslations)
            print sortedtable            
            print unsortedtablewithtranslations            
            for x in unsortedtablewithtranslations:
                sortedtable[x] = unsortedtablewithtranslations[x]
            print 'sorted...'
            sortedtable = cleartranslationtable(sortedtable)
            #for x in sortedtable:
                #print sortedtable[x]
            #print 'end sorted..'
            #sortedtable = cleartranslationtable(sortedtable)
            #oldtable = []
            #newtable = []
            #linesprintedcounter = 0
            #for x in sortedtable:
                #if sortedtable[x] <> u'':
                    #linesprintedcounter += 1
                   # newtable.append(sortedtable[x])                                   
                    #if linesprintedcounter == middleoftable:
                        #newtable.append(mycenter)
            #for x in unsortedtablewithtranslations:
                #oldtable.append(sortedtable[x])                                   
            #if oldtable == newtable:
                #print '\tTable is sorted (place of "Template for center" is not checked)'
                #exit()
            #else:
                #print tabletitle , ' is unsorted or with problems'
                #linesprintedcounter = 0 
                #print newtable               
                #for x in newtable:
                    #linesprintedcounter += 1
                    #print x
                    #if linesprintedcounter == middleoftable:
                        #print mycenter                                               
                #print myend                                                   
            print tabletitle , ' is unsorted or with problems'
            linesprintedcounter = 0 
            for x in sortedtable:
                linesprintedcounter += 1
                print sortedtable[x]
                if linesprintedcounter == middleoftable:
                    print mycenter                                               
            print myend                                                   
            print 'Table end: ', tablecounter + 1, '---------'
            if tablewithredirect != None:
                #pass
                print 'Table with redirect start: ', tabletitle
                print tablewithredirect
                print myend
                print 'end table with redirect: ', tablecounter, '---------'
                #exit()
            if hasproblems:
                #pass
                middleoftable = - (len(tablewithproblems) / -2)
                linesprintedcounter = 0
                print 'Table with problems start: ????????'
                print b.problemtablestring()
                print tablewithproblems
                for tableline in tablewithproblems:
                    if  linesprintedcounter == middleoftable:
                        print mycenter 
                        #exit()                                                                    
                    print tableline, tablewithproblems[tableline] #'.....', tableline, lemmaalltables[x][1][tableline]
                    linesprintedcounter += 1 
                print myend
                print 'end table with problems. ', '????????'
                #exit()
                
        print '============================ END OF TABLES FOR: ', lemma , ' =============================='
    #else:
        #print '============================ no translation tables for: ',  lemma
              
def main2(langofwiktionary):

    with codecs.open('translation.txt', 'r', 'utf-8') as f:
        onetransltext = f.read()

           
#for checking class?
def main(passedargs):
    langofwiktionary = passedargs[1]
    #if not passedargs[5]:
    try:
        usingLangStrings = LangStrings(langofwiktionary)
    #next statement will also create an empty sorted _readydict
        sortedisos =  usingLangStrings.isosinorder()
    except:
        usingLangStrings = None
    try:
        if passedargs[4]:#just read a file with translations
            #with codecs.open('onetranslation.txt', 'r', 'utf-8') as f:
            with open('onetranslation.txt', 'r') as f:
                onetransltext = f.read()
                checktranslation(langofwiktionary, 'dummy', onetransltext, usingLangStrings)
        else:
            #print passedargs
            #exit()
            if len(passedargs[7]):
                dump = xmlreader.WikiDump(langofwiktionary, passedargs[7], mydumpspath, passedargs[2], passedargs[3])
            else:
                dump = xmlreader.WikiDump(langofwiktionary, u'wiktionary', mydumpspath, passedargs[2], passedargs[3])
            #dump = xmlreader.XmlDump(passedargs[2])                        
            if dump._ISOK:
                #βρίσκουμε το αρχείο με τους τίτλους σκέτους
                dumpfilename, titlesfilename = dump.workingdump_filenames()
                with codecs.open(titlesfilename, 'r', 'utf-8') as ftemp:
                    fulllines = ftemp.readlines()
                #δημιουργούμε ένα list μόνο με τους τίτλους (χωρίς τη θέση κλπ)
                onlylemmas = []
                for line in fulllines:
                    onlylemmas.append(line.split('<title>')[1].split('</title>')[0])                    
                lemmascounter = 0
                found = 0 
                for entry in dump.parse(): 
                    #print entry.ns
                    lemmascounter += 1
                    if(entry.ns == '0'): 
                        #print entry.title                        
                        if passedargs[6]:#αν θέλει να βρει κάτι μπορεί να χρησιμοποιήσει και το αρχείο των τίτλων
                            foundsomething = extractsomething4(entry.title, entry.text, onlylemmas)
                            if foundsomething:
                                found += 1
                                print found, '/', lemmascounter
                            #with codecs.open('el-Κατηγορίες.txt', 'a', 'utf-8') as f:
                                #f.write( entry.title.decode('utf-8') + '\n')
                            #found += 1
                            
                        else:
                            checktranslation(langofwiktionary, entry.title, entry.text, usingLangStrings)
                #alltrnasls = sorted(usingLangStrings._readydict, key=lambda t: t[0]]))
                #for x in usingLangStrings._readydict:
                    #print usingLangStrings.langlangs.langnames[x], ':', x, ':', usingLangStrings._readydict[x]
            else:
                print 'not all ok'                          
    finally:
        fproblematic.close()
        print "Έτοιμα!"        

def getonearticle():
    print 'came here to get one one article...'
    exit()
        
def _getargs():
    '''Parameters:
    -n, --onlyone means we want to check a simple translation table
    in a text file named onetranslation.txt
    
    -l, --lang (=a language iso code) defaults to that lang
    as lang of wiktionary
    
    -c, --checklatest if a newer dump exists, get it
    -g, --getlatest get latest dump
    
    '''
    _parser = argparse.ArgumentParser()
    #if user is online may want to check for newer dump.
    #the default xml filename is given at start of this python sctript.
    _parser.add_argument("-a", "--articlename",  default = u'')
    _parser.add_argument("-c", "--checklatest", action="store_true")
    _parser.add_argument("-e", "--extractsomething", action="store_true")
    _parser.add_argument("-g", "--getlatest", action="store_true")
    _parser.add_argument("-n", "--onlyone", action="store_true")
    _parser.add_argument("-o", "--online", action="store_true")
    _parser.add_argument("-s", "--something", action="store_true")
    _parser.add_argument("-w", "--wikiproject") #other wikimedia project
    
    #language of wiktionary. Default is given at start of this python sctript.
    _parser.add_argument("-l", "--lang",  default = mylang)
    #defaults:
    _num = False
    #parse args:
    _args = _parser.parse_args()
    _num = _args.onlyone
    _online = _args.online
    if len(_args.lang):
        _lang = _args.lang
    else:
        _lang = mylang
    if _online:
        if _args.articlename == u'':
            #TODO report misuse
            _online = False
        else:
            print 'came here?'
            exit()
            getonearticle(_args.articlename) #will save an xml file as 'onetranslation.txt'
    whichwikiproject = u''
    if _args.wikiproject <> None:
        whichwikiproject = _args.wikiproject
    #print   _num, _lang, _args.checklatest, _args.getlatest
    #exit()    
    return _num, _lang, _args.checklatest, _args.getlatest, _args.onlyone, _args.something , _args.extractsomething , whichwikiproject            

if __name__=="__main__":
    g = _getargs()
    main(g)#pass language and xml file name