...

[iramuteq] / corpus.py

diff --git a/corpus.py b/corpus.py
index e043707..e222940 100644 (file)
--- a/corpus.py
+++ b/corpus.py
@@ -8,17 +8,15 @@ _ = gettext.gettext
 import locale
 import sys
 from time import time
 import locale
 import sys
 from time import time

-from functions import decoupercharact, ReadDicoAsDico, DoConf
+from functions import decoupercharact, ReadDicoAsDico, DoConf, ReadLexique

 import re
 import sqlite3
 import re
 import sqlite3

-import numpy

 import itertools
 import logging
 from operator import itemgetter
 from uuid import uuid4
 from chemins import PathOut
 from dialog import CorpusPref
 import itertools
 import logging
 from operator import itemgetter
 from uuid import uuid4
 from chemins import PathOut
 from dialog import CorpusPref

-from functions import ReadLexique, ReadDicoAsDico

 from colors import colors
 import datetime
 
 from colors import colors
 import datetime
 


@@ -39,8 +37,7 @@ def copycorpus(corpus) :
 
 class Corpus :
     """Corpus class
 
 class Corpus :
     """Corpus class

-    list of uci
-
+    list of text

     """
     def __init__(self, parent, parametres = {}, read = False) :
         self.parent = parent
     """
     def __init__(self, parent, parametres = {}, read = False) :
         self.parent = parent


@@ -78,10 +75,10 @@ class Corpus :
                 gramtype = self.parent.lexique[word][1]
                 lem = self.parent.lexique[word][0]
             elif word.isdigit() :
                 gramtype = self.parent.lexique[word][1]
                 lem = self.parent.lexique[word][0]
             elif word.isdigit() :

-                gramtype = 'num'
+                gramtype = u'num'

                 lem = word
             else :
                 lem = word
             else :

-                gramtype = 'nr'
+                gramtype = u'nr'

                 lem = word
             self.formes[word] =  Word(word, gramtype, len(self.formes), lem)
             self.idformesuces[self.formes[word].ident] = {self.ucis[-1].uces[-1].ident : 1}
                 lem = word
             self.formes[word] =  Word(word, gramtype, len(self.formes), lem)
             self.idformesuces[self.formes[word].ident] = {self.ucis[-1].uces[-1].ident : 1}


@@ -211,6 +208,28 @@ class Corpus :
     def getucesfrometoile(self, etoile) :
         return [uce.ident for uci in self.ucis for uce in uci.uces if etoile in uci.etoiles]
 
     def getucesfrometoile(self, etoile) :
         return [uce.ident for uci in self.ucis for uce in uci.uces if etoile in uci.etoiles]
 

+    def getetoileuces(self) :
+        log.info('get uces etoiles')
+        etoileuces = {}
+        idpara = 0
+        for uci in self.ucis :
+            etoiles = uci.etoiles[1:]
+            for et in etoiles :
+                if et in etoileuces :
+                    etoileuces[et] += [uce.ident for uce in uci.uces]
+                else :
+                    etoileuces[et] = [uce.ident for uce in uci.uces]
+            if uci.paras != [] :
+                for et in uci.paras :
+                    if et in etoileuces :
+                        etoileuces[et] += [uce.ident for uce in uci.uces if uce.para == idpara]
+                    else :
+                        etoileuces[et] = [uce.ident for uce in uci.uces if uce.para == idpara]
+                    idpara += 1
+            else :
+                idpara += 1
+        return etoileuces
+

     def getucefromid(self, uceid) :
         if self.iduces is None : self.make_iduces()
         return self.iduces[uceid]
     def getucefromid(self, uceid) :
         if self.iduces is None : self.make_iduces()
         return self.iduces[uceid]


@@ -254,14 +273,18 @@ class Corpus :
         if self.iduces is None :
             self.iduces = dict([[uce.ident, uce] for uci in self.ucis for uce in uci.uces])
 
         if self.iduces is None :
             self.iduces = dict([[uce.ident, uce] for uci in self.ucis for uce in uci.uces])
 

-    def make_lexitable(self, mineff, etoiles) :
-        tokeep = [lem for lem in self.lems if self.lems[lem].freq >= mineff]
+    def make_lexitable(self, mineff, etoiles, gram = 0) :
+        if gram == 0 :
+            grams = {1:'', 2:''}
+        else :
+            grams = {gram :''}
+        tokeep = [lem for lem in self.lems if self.lems[lem].freq >= mineff and self.lems[lem].act in grams]

         etuces = [[] for et in etoiles]
         for uci in self.ucis :
             get = list(set(uci.etoiles).intersection(etoiles))
             if len(get) > 1 :
         etuces = [[] for et in etoiles]
         for uci in self.ucis :
             get = list(set(uci.etoiles).intersection(etoiles))
             if len(get) > 1 :

-                return '2 variables sur la meme ligne'
-            elif get != [] :
+                log.info('2 variables sur une ligne')
+            if get != [] :

                 etuces[etoiles.index(get[0])] += [uce.ident for uce in uci.uces]
         etuces = [set(val) for val in etuces]
         tab = []
                 etuces[etoiles.index(get[0])] += [uce.ident for uce in uci.uces]
         etuces = [set(val) for val in etuces]
         tab = []


@@ -412,6 +435,18 @@ class Corpus :
                 f.write(etline.encode(self.parametres['syscoding']) + '\n')
                 f.write(guce.encode(self.parametres['syscoding']) + '\n\n')
 
                 f.write(etline.encode(self.parametres['syscoding']) + '\n')
                 f.write(guce.encode(self.parametres['syscoding']) + '\n\n')
 

+    def export_classe(self, outf, classe, lem = False) :
+        sts = self.lc[classe - 1] 
+        res = self.getconcorde(sts)
+        self.make_iduces()
+        with open(outf, 'w') as f :
+            for uce in res :
+                guce = uce[1]
+                f.write(' '.join(self.ucis[self.iduces[uce[0]].uci].etoiles).encode(self.parametres['syscoding']) + '\n')
+                if lem :
+                    guce = ' '.join([self.formes[forme].lem for forme in guce.split()])
+                f.write(guce.encode(self.parametres['syscoding']) + '\n\n')
+

     def make_and_write_sparse_matrix_from_uces(self, actives, outfile, listuce = False) :
         log.info('make_and_write_sparse_matrix_from_uces %s' % outfile)
         nbl = 0
     def make_and_write_sparse_matrix_from_uces(self, actives, outfile, listuce = False) :
         log.info('make_and_write_sparse_matrix_from_uces %s' % outfile)
         nbl = 0


@@ -473,6 +508,17 @@ class Corpus :
                 table_uce[uces[uce]][i] = 1
         table_uce.insert(0, list_act)
         return table_uce      
                 table_uce[uces[uce]][i] = 1
         table_uce.insert(0, list_act)
         return table_uce      

+    
+    def make_pondtable_with_classe(self, uces, list_act) :
+        table_uce = [[0 for val in list_act] for line in range(0,len(uces))]
+        uces = dict([[uce, i] for i, uce in enumerate(uces)])
+        for i, lem in enumerate(list_act) :
+            uceseff = self.getlemuceseff(lem)
+            lemuces = list(set(uceseff.keys()).intersection(uces))
+            for uce in lemuces :
+                table_uce[uces[uce]][i] = uceseff[uce]
+        table_uce.insert(0, list_act)
+        return table_uce 

 
     def parse_active(self, gramact, gramsup = None) :
         log.info('parse actives')
 
     def parse_active(self, gramact, gramsup = None) :
         log.info('parse actives')


@@ -481,7 +527,7 @@ class Corpus :
                 self.lems[lem].act = 2
             elif self.lems[lem].gram in gramact :
                 self.lems[lem].act = 1
                 self.lems[lem].act = 2
             elif self.lems[lem].gram in gramact :
                 self.lems[lem].act = 1

-            elif gramsup is not None :
+            elif gramsup is not None and self.lems[lem].gram not in gramact:

                 if self.lems[lem].gram in gramsup :
                     self.lems[lem].act = 2
                 else :
                 if self.lems[lem].gram in gramsup :
                     self.lems[lem].act = 2
                 else :


@@ -501,6 +547,8 @@ class Corpus :
         allactives = [[self.lems[lem].freq, lem] for lem in self.lems if self.lems[lem].act == key and self.lems[lem].freq >= 3]
         self.activenb = len(allactives)
         allactives = sorted(allactives, reverse = True)
         allactives = [[self.lems[lem].freq, lem] for lem in self.lems if self.lems[lem].act == key and self.lems[lem].freq >= 3]
         self.activenb = len(allactives)
         allactives = sorted(allactives, reverse = True)

+        if self.activenb == 0 :
+            return [], 0

         if len(allactives) <= nbmax :
             log.info('nb = %i - eff min = %i ' % (len(allactives), allactives[-1][0]))
             return [val[1] for val in allactives], allactives[-1][0]
         if len(allactives) <= nbmax :
             log.info('nb = %i - eff min = %i ' % (len(allactives), allactives[-1][0]))
             return [val[1] for val in allactives], allactives[-1][0]


@@ -531,7 +579,7 @@ class Corpus :
     def make_etoiles(self) :
         etoiles = set([])
         for uci in self.ucis :
     def make_etoiles(self) :
         etoiles = set([])
         for uci in self.ucis :

-            etoiles.update(uci.etoiles[1:] + uci.paras)
+            etoiles.update(uci.etoiles[1:])

         return list(etoiles)
 
     def make_etoiles_dict(self) :
         return list(etoiles)
 
     def make_etoiles_dict(self) :


@@ -542,16 +590,16 @@ class Corpus :
             if et[0] in det :
                 try :
                     endet = '_'.join(et[1:])
             if et[0] in det :
                 try :
                     endet = '_'.join(et[1:])

-                    if endet in det[et[0]] :
-                        det[et[0]][endet] += 1
+                    if etoile in det[et[0]] :
+                        det[et[0]][etoile] += 1

                     else :
                     else :

-                        det[et[0]][endet] = 1
+                        det[et[0]][etoile] = 1

                 except IndexError :
                     det[et[0]] += 1
             else :
                 try :
                     endet = '_'.join(et[1:])
                 except IndexError :
                     det[et[0]] += 1
             else :
                 try :
                     endet = '_'.join(et[1:])

-                    det[et[0]] = {endet :1}
+                    det[et[0]] = {etoile :1}

                 except IndexError :
                     det[et[0]] = 1
         return det
                 except IndexError :
                     det[et[0]] = 1
         return det


@@ -565,13 +613,16 @@ class Corpus :
             elif get != [] :
                 etuces[listet.index(get[0])] += [uce.ident for uce in uci.uces]
         return etuces
             elif get != [] :
                 etuces[listet.index(get[0])] += [uce.ident for uce in uci.uces]
         return etuces

-            

 
     def make_and_write_profile_et(self, ucecl, fileout) :
         log.info('etoiles/classes')
 
     def make_and_write_profile_et(self, ucecl, fileout) :
         log.info('etoiles/classes')

-        etoiles = self.make_etoiles()
+        etoileuces = self.getetoileuces()
+        etoileuces = dict([[et, etoileuces[et]] for et in etoileuces if len(etoileuces[et]) > 1])

         with open(fileout, 'w') as f :
         with open(fileout, 'w') as f :

-            f.write('\n'.join([';'.join([etoile] + [`len(set(self.getucesfrometoile(etoile)).intersection(classe))` for classe in ucecl]) for etoile in etoiles]).encode(self.parametres['syscoding']))
+            f.write('\n'.join([';'.join([et] + [`len(set(etoileuces[et]).intersection(classe))` for classe in ucecl]) for et in etoileuces]).encode(self.parametres['syscoding']))
+        #etoiles = self.make_etoiles()
+        #with open(fileout, 'w') as f :
+        #    f.write('\n'.join([';'.join([etoile] + [`len(set(self.getucesfrometoile(etoile)).intersection(classe))` for classe in ucecl]) for etoile in etoiles]).encode(self.parametres['syscoding']))

 
     def make_colored_corpus(self) :
         ucecl = {}
 
     def make_colored_corpus(self) :
         ucecl = {}


@@ -683,27 +734,37 @@ class Corpus :
         self.lc0 = self.lc.pop(0)
         #return ucecl
     
         self.lc0 = self.lc.pop(0)
         #return ucecl
     

-    def get_stat_by_cluster(self, outf) :
+    def get_stat_by_cluster(self, outf, lclasses = None) :

         log.info('get_stat_by_cluster')
         log.info('get_stat_by_cluster')

+        if lclasses is None :
+            lclasses = self.lc

         t1 = time()
         t1 = time()

-        occurrences = dict([[i + 1, 0] for i in range(len(self.lc))])
-        formescl = dict([[i + 1, 0] for i in range(len(self.lc))])
-        hapaxcl = dict([[i + 1, 0] for i in range(len(self.lc))])
-        lenclasses = dict([[i+1,len(cl)] for i, cl in enumerate(self.lc)])
-        sets = [set(cl) for cl in self.lc]
+        occurrences = dict([[i + 1, 0] for i in range(len(lclasses))])
+        formescl = dict([[i + 1, 0] for i in range(len(lclasses))])
+        hapaxcl = dict([[i + 1, 0] for i in range(len(lclasses))])
+        lenclasses = dict([[i+1,len(cl)] for i, cl in enumerate(lclasses)])
+        sets = [set(cl) for cl in lclasses]

         for forme in self.formes :
             formeuceeff = self.getformeuceseff(forme)
         for forme in self.formes :
             formeuceeff = self.getformeuceseff(forme)

-            for i, classe in enumerate(self.lc) :
+            for i, classe in enumerate(lclasses) :

                 concern = sets[i].intersection(formeuceeff.keys())
                 if len(concern) :
                     occurrences[i+1] += sum([formeuceeff[uce] for uce in concern])
                     formescl[i+1] += 1
                     if self.formes[forme].freq == 1 :
                         hapaxcl[i+1] += 1
                 concern = sets[i].intersection(formeuceeff.keys())
                 if len(concern) :
                     occurrences[i+1] += sum([formeuceeff[uce] for uce in concern])
                     formescl[i+1] += 1
                     if self.formes[forme].freq == 1 :
                         hapaxcl[i+1] += 1

-        toprint = '\n'.join([';'.join([`i`, `occurrences[i]`, `formescl[i]`, `hapaxcl[i]`, `lenclasses[i]`, `float(hapaxcl[i])/float(formescl[i])`]) for i in occurrences])
-        with open(outf, 'w') as f :
-            f.write(toprint)

         log.info('%f' % (time() - t1))        
         log.info('%f' % (time() - t1))        

+        if outf is not None :
+            toprint = '\n'.join([';'.join([`i`, `occurrences[i]`, `formescl[i]`, `hapaxcl[i]`, `lenclasses[i]`, `float(hapaxcl[i])/float(formescl[i])`]) for i in occurrences])
+            with open(outf, 'w') as f :
+                f.write(toprint)
+        else :
+            return [[`occurrences[i]`, `formescl[i]`, `hapaxcl[i]`, `lenclasses[i]`, `float(hapaxcl[i])/float(formescl[i])`] for i in occurrences]
+
+    def get_stat_by_et(self, outf, etoiles) :
+        lclasses = [self.getucesfrometoile(etoile) for etoile in etoiles]
+        stats = self.get_stat_by_cluster(None, lclasses)
+        stats = [[etoiles[i]] + val for i, val in enumerate(stats)]

 
     def gethapaxbyet(self, etoiles) :
         hapaxuces = [self.getlemuces(forme)[0] for forme in self.lems if self.lems[forme].freq == 1]
 
     def gethapaxbyet(self, etoiles) :
         hapaxuces = [self.getlemuces(forme)[0] for forme in self.lems if self.lems[forme].freq == 1]


@@ -762,14 +823,29 @@ class Corpus :
         with open('/tmp/testhapxuce.html','w') as f :
             f.write(txt)
 
         with open('/tmp/testhapxuce.html','w') as f :
             f.write(txt)
 

+    def export_dictionary(self, fileout, syscoding) :
+        listformes = [[self.formes[forme].freq, forme, self.formes[forme].lem, self.formes[forme].gram] for forme in self.formes]
+        listformes.sort(reverse = True)
+        listformes = [forme[1:] + [`forme[0]`] for forme in listformes]
+        with open(fileout, 'w') as f :
+            f.write('\n'.join(['\t'.join(forme) for forme in listformes]).encode(syscoding))
+
+    def export_lems(self, fileout, syscoding) :
+        self.make_idformes()
+        listlem = [[lem, '\t'.join(['\t'.join([self.idformes[forme].forme, `self.lems[lem].formes[forme]`]) for forme in self.lems[lem].formes])] for lem in self.lems]
+        listlem.sort()
+        with open(fileout, 'w') as f :
+            f.write('\n'.join(['\t'.join(lem) for lem in listlem]).encode(syscoding))
+
+
+

 
 class MakeUciStat :
     def __init__(self, corpus) :
         ucinb = corpus.getucinb()
         ucisize = corpus.getucisize()
         ucimean = float(sum(ucisize))/float(ucinb)
 
 class MakeUciStat :
     def __init__(self, corpus) :
         ucinb = corpus.getucinb()
         ucisize = corpus.getucisize()
         ucimean = float(sum(ucisize))/float(ucinb)

-        detoile = corpus.make_etoiles_dict()
-        
+        detoile = corpus.make_etoiles_dict() 

 
 class Uci :
     def __init__(self, iduci, line, paraset = None) :
 
 class Uci :
     def __init__(self, iduci, line, paraset = None) :


@@ -1019,7 +1095,7 @@ class BuildCorpus :
     def firstclean(self, txt) :
         txt = txt.replace(u'’',"'")
         txt = txt.replace(u'œ', u'oe')
     def firstclean(self, txt) :
         txt = txt.replace(u'’',"'")
         txt = txt.replace(u'œ', u'oe')

-        return txt.replace('...',u' £$£ ').replace('?',' ? ').replace('.',' . ').replace('!', ' ! ').replace(',',' , ').replace(';', ' ; ').replace(':',' : ').replace(u'…', ' £$£ ')
+        return txt.replace('...',u' £$£ ').replace('?',' ? ').replace('.',' . ').replace('!', ' ! ').replace(',',' , ').replace(';', ' ; ').replace(':',' : ').replace(u'…', u' £$£ ')

 
     def make_cleans(self, txt) :
         for clean in self.cleans :
 
     def make_cleans(self, txt) :
         for clean in self.cleans :


@@ -1092,7 +1168,6 @@ class BuildFromAlceste(BuildCorpus) :
                                     log.info(u'Empty text : %i' % linenb)
                                     iduci -= 1
                                     self.corpus.ucis.pop()
                                     log.info(u'Empty text : %i' % linenb)
                                     iduci -= 1
                                     self.corpus.ucis.pop()

-                                    #raise Exception("EmptyText %i" % linenb)

                             self.corpus.ucis.append(Uci(iduci, line))
                         if self.dlg is not None :
                             if not (iduci + 1) % 10 :
                             self.corpus.ucis.append(Uci(iduci, line))
                         if self.dlg is not None :
                             if not (iduci + 1) % 10 :


@@ -1105,19 +1180,24 @@ class BuildFromAlceste(BuildCorpus) :
                             idpara += 1
                             self.corpus.ucis[-1].paras.append(line.split()[0])
                         else :
                             idpara += 1
                             self.corpus.ucis[-1].paras.append(line.split()[0])
                         else :

-                            raise Exception('paragrapheOT')
+                            raise Exception('paragrapheOT %i' % linenb)

                     elif line.strip() != '' and iduci != -1 :
                         txt.append(line)
             if txt != [] and iduci != -1 :
                 iduce, idpara = self.treattxt(txt, iduce, idpara, iduci)
                 del(txt)
             else :
                     elif line.strip() != '' and iduci != -1 :
                         txt.append(line)
             if txt != [] and iduci != -1 :
                 iduce, idpara = self.treattxt(txt, iduce, idpara, iduci)
                 del(txt)
             else :

-                raise Exception("EmptyText")
+                if iduci != -1 :
+                    iduci -= 1
+                    self.corpus.ucis.pop()
+                    log.info(Exception("Empty text %i" % linenb))
+                else :
+                    raise Exception('EmptyText %i' % linenb)

             if iduci != -1  and iduce != -1:
                 self.backup_uce()
             else : 
                 log.info(_(u"No Text in corpora. Are you sure of the formatting ?"))
             if iduci != -1  and iduce != -1:
                 self.backup_uce()
             else : 
                 log.info(_(u"No Text in corpora. Are you sure of the formatting ?"))

-                raise Exception('TextBeforeTextMark')
+                raise Exception('TextBeforeTextMark %i' % linenb)

         except UnicodeDecodeError :
             raise Exception("CorpusEncoding")
 
         except UnicodeDecodeError :
             raise Exception("CorpusEncoding")
 


@@ -1188,7 +1268,10 @@ class Builder :
             parametres['originalpath'] = parent.filename
             PathOut().createdir(parametres['pathout'])
             ReadLexique(self.parent, lang = parametres['lang'])
             parametres['originalpath'] = parent.filename
             PathOut().createdir(parametres['pathout'])
             ReadLexique(self.parent, lang = parametres['lang'])

-            self.parent.expressions = ReadDicoAsDico(self.parent.DictPath.get(parametres['lang']+'_exp', 'french_exp'))
+            if parametres['lang'] != 'other' and  os.path.exists(self.parent.DictPath.get(parametres['lang']+'_exp', 'french_exp')):
+                self.parent.expressions = ReadDicoAsDico(self.parent.DictPath.get(parametres['lang']+'_exp', 'french_exp'))
+            else :
+                self.parent.expressions = {}

             self.parametres = parametres
         else :
             if self.dlg is not None :
             self.parametres = parametres
         else :
             if self.dlg is not None :