...

[iramuteq] / corpus.py

diff --git a/corpus.py b/corpus.py
index e043707..3ced482 100644 (file)
--- a/corpus.py
+++ b/corpus.py
@@ -8,17 +8,16 @@ _ = gettext.gettext
 import locale
 import sys
 from time import time
 import locale
 import sys
 from time import time

-from functions import decoupercharact, ReadDicoAsDico, DoConf
+from functions import decoupercharact, ReadDicoAsDico, DoConf, ReadLexique

 import re
 import sqlite3
 import re
 import sqlite3

-import numpy

 import itertools
 import logging
 from operator import itemgetter
 from uuid import uuid4
 from chemins import PathOut
 import itertools
 import logging
 from operator import itemgetter
 from uuid import uuid4
 from chemins import PathOut

-from dialog import CorpusPref
-from functions import ReadLexique, ReadDicoAsDico
+from dialog import CorpusPref, SubTextFromMetaDial
+from copy import copy

 from colors import colors
 import datetime
 
 from colors import colors
 import datetime
 


@@ -35,12 +34,21 @@ def copycorpus(corpus) :
     copy_corpus.conn_all()
     return copy_corpus
 
     copy_corpus.conn_all()
     return copy_corpus
 

+def CopyUce(uce) :
+    return Uce(uce.ident, uce.para, uce.uci)
+    
+
+def CopyUci(uci):
+    nuci = Uci(uci.ident, '')
+    nuci.etoiles = copy(uci.etoiles)
+    nuci.uces = [CopyUce(uce) for uce in uci.uces]
+    return nuci
+    

 
 
 class Corpus :
     """Corpus class
 
 
 class Corpus :
     """Corpus class

-    list of uci
-
+    list of text

     """
     def __init__(self, parent, parametres = {}, read = False) :
         self.parent = parent
     """
     def __init__(self, parent, parametres = {}, read = False) :
         self.parent = parent


@@ -78,13 +86,27 @@ class Corpus :
                 gramtype = self.parent.lexique[word][1]
                 lem = self.parent.lexique[word][0]
             elif word.isdigit() :
                 gramtype = self.parent.lexique[word][1]
                 lem = self.parent.lexique[word][0]
             elif word.isdigit() :

-                gramtype = 'num'
+                gramtype = u'num'

                 lem = word
             else :
                 lem = word
             else :

-                gramtype = 'nr'
+                gramtype = u'nr'

                 lem = word
             self.formes[word] =  Word(word, gramtype, len(self.formes), lem)
             self.idformesuces[self.formes[word].ident] = {self.ucis[-1].uces[-1].ident : 1}
                 lem = word
             self.formes[word] =  Word(word, gramtype, len(self.formes), lem)
             self.idformesuces[self.formes[word].ident] = {self.ucis[-1].uces[-1].ident : 1}

+    
+    def add_word_from_forme(self, word, stident):
+        if word.forme in self.formes :
+            self.formes[word.forme].freq += 1
+            if self.formes[word.forme].ident in self.idformesuces :
+                if stident in self.idformesuces[self.formes[word.forme].ident] :
+                    self.idformesuces[self.formes[word.forme].ident][stident] += 1
+                else :
+                    self.idformesuces[self.formes[word.forme].ident][stident] = 1
+            else :
+                self.idformesuces[self.formes[word.forme].ident] = {stident: 1}
+        else :
+            self.formes[word.forme] = Word(word.forme, word.gram, len(self.formes), word.lem)
+            self.idformesuces[self.formes[word.forme].ident] = {stident : 1}       

 
     def conn_all(self): 
         """connect corpus to db"""
 
     def conn_all(self): 
         """connect corpus to db"""


@@ -126,6 +148,10 @@ class Corpus :
             wordid = self.formes[wordid].ident
         res = self.cformes.execute('SELECT uces FROM uces where id=? ORDER BY id;', (`wordid`,))
         return list(itertools.chain(*[[int(val) for val in row[0].split()] if not isinstance(row[0], int) else [row[0]] for row in res]))
             wordid = self.formes[wordid].ident
         res = self.cformes.execute('SELECT uces FROM uces where id=? ORDER BY id;', (`wordid`,))
         return list(itertools.chain(*[[int(val) for val in row[0].split()] if not isinstance(row[0], int) else [row[0]] for row in res]))

+    
+    def getworducis(self, wordid) :
+        res = self.getworduces(wordid)
+        return list(set([self.getucefromid(uce).uci for uce in res]))

 
     def getformeuceseff(self, formeid) :
         if isinstance(formeid, basestring) :
 
     def getformeuceseff(self, formeid) :
         if isinstance(formeid, basestring) :


@@ -146,6 +172,16 @@ class Corpus :
         res = self.cformes.execute(query)
         return list(set(list(itertools.chain(*[[int(val) for val in row[0].split()] if not isinstance(row[0], int) else [row[0]] for row in res]))))
 
         res = self.cformes.execute(query)
         return list(set(list(itertools.chain(*[[int(val) for val in row[0].split()] if not isinstance(row[0], int) else [row[0]] for row in res]))))
 

+    def gettgenst(self, tgen):
+        formesid = ', '.join([`val` for lem in tgen for val in self.lems[lem].formes if lem in self.lems])
+        query = 'SELECT uces FROM uces where id IN (%s) ORDER BY id' % formesid
+        res = self.cformes.execute(query)
+        return list(set(list(itertools.chain(*[[int(val) for val in row[0].split()] if not isinstance(row[0], int) else [row[0]] for row in res]))))
+    
+    def gettgentxt(self, tgen):
+        sts = self.gettgenst(tgen)
+        return list(set([self.getucefromid(val).uci for val in sts]))
+

     def getlemucis(self, lem) :
         uces = self.getlemuces(lem)
         return list(set([self.getucefromid(val).uci for val in uces]))
     def getlemucis(self, lem) :
         uces = self.getlemuces(lem)
         return list(set([self.getucefromid(val).uci for val in uces]))


@@ -161,7 +197,7 @@ class Corpus :
         lemuceeff = {}
         for i, uce in enumerate(uces) :
             lemuceeff[uce] = lemuceeff.get(uce, 0) + eff[i]
         lemuceeff = {}
         for i, uce in enumerate(uces) :
             lemuceeff[uce] = lemuceeff.get(uce, 0) + eff[i]

-        return lemuceeff    
+        return lemuceeff

 
     def getlemclustereff(self, lem, cluster) :
         return len(list(set(self.lc[cluster]).intersection(self.getlemuces(lem))))
 
     def getlemclustereff(self, lem, cluster) :
         return len(list(set(self.lc[cluster]).intersection(self.getlemuces(lem))))


@@ -197,7 +233,12 @@ class Corpus :
         return [len(uce[1].split()) for uce in res]
 
     def getconcorde(self, uces) :
         return [len(uce[1].split()) for uce in res]
 
     def getconcorde(self, uces) :

-        return self.cuces.execute('select * from uces where id IN (%s);' % ', '.join([`i` for i in uces])) 
+        return self.cuces.execute('select * from uces where id IN (%s) ORDER BY id;' % ', '.join([`i` for i in uces])) 
+    
+    def getuciconcorde(self, ucis) :
+        uces = [[val,[uce.ident for uce in self.ucis[val].uces]] for val in ucis]
+        uces = [[val[0], '\n'.join([row[1] for row in self.getconcorde(val[1])])] for val in uces]
+        return uces

 
     def getwordconcorde(self, word) :
         return self.getconcorde(self.getworduces(word))
 
     def getwordconcorde(self, word) :
         return self.getconcorde(self.getworduces(word))


@@ -207,10 +248,47 @@ class Corpus :
 
     def getalluces(self) :
         return self.cuces.execute('SELECT * FROM uces')
 
     def getalluces(self) :
         return self.cuces.execute('SELECT * FROM uces')

-
+    
+    def getallucis(self):
+        uces = [row[1] for row in self.getalluces()]
+        return [[uci.ident, '\n'.join([uces[uce.ident] for uce in uci.uces])] for uci in self.ucis]
+        

     def getucesfrometoile(self, etoile) :
         return [uce.ident for uci in self.ucis for uce in uci.uces if etoile in uci.etoiles]
 
     def getucesfrometoile(self, etoile) :
         return [uce.ident for uci in self.ucis for uce in uci.uces if etoile in uci.etoiles]
 

+    def getetoileuces(self) :
+        log.info('get uces etoiles')
+        etoileuces = {}
+        idpara = 0
+        for uci in self.ucis :
+            etoiles = uci.etoiles[1:]
+            for et in etoiles :
+                if et in etoileuces :
+                    etoileuces[et] += [uce.ident for uce in uci.uces]
+                else :
+                    etoileuces[et] = [uce.ident for uce in uci.uces]
+            if uci.paras != [] :
+                for et in uci.paras :
+                    if et in etoileuces :
+                        etoileuces[et] += [uce.ident for uce in uci.uces if uce.para == idpara]
+                    else :
+                        etoileuces[et] = [uce.ident for uce in uci.uces if uce.para == idpara]
+                    idpara += 1
+            else :
+                idpara += 1
+        return etoileuces
+    
+    def getetoileucis(self):
+        etoileuces = {}
+        for uci in self.ucis :
+            etoiles = uci.etoiles[1:]
+            for et in etoiles :
+                if et in etoileuces :
+                    etoileuces[et] += [uci.ident]
+                else :
+                    etoileuces[et] = [uci.ident]
+        return etoileuces
+

     def getucefromid(self, uceid) :
         if self.iduces is None : self.make_iduces()
         return self.iduces[uceid]
     def getucefromid(self, uceid) :
         if self.iduces is None : self.make_iduces()
         return self.iduces[uceid]


@@ -246,6 +324,30 @@ class Corpus :
                     self.lems[self.formes[forme].lem] = Lem(self, self.formes[forme])
         else :
             self.lems = dict([[forme, Lem(self, self.formes[forme])] for forme in self.formes])
                     self.lems[self.formes[forme].lem] = Lem(self, self.formes[forme])
         else :
             self.lems = dict([[forme, Lem(self, self.formes[forme])] for forme in self.formes])

+    
+    def make_lems_from_dict(self, dictionnaire, dolem = True) :
+        log.info('make lems from dict')
+        self.lems = {}
+        for forme in self.formes :
+            if self.formes[forme].forme in dictionnaire :
+                lem = dictionnaire[forme][0]
+                gram = dictionnaire[forme][1]
+            elif forme.isdigit() :
+                gram = u'num'
+                lem = forme
+            else :
+                gram = u'nr'
+                lem = forme
+            self.formes[forme].lem = lem
+            self.formes[forme].gram = gram
+            if dolem :
+                if self.formes[forme].lem in self.lems :
+                    if self.formes[forme].ident not in self.lems[self.formes[forme].lem].formes :
+                        self.lems[self.formes[forme].lem].add_forme(self.formes[forme])
+                else :
+                    self.lems[self.formes[forme].lem] = Lem(self, self.formes[forme])
+            else :
+                self.lems[forme] = Lem(self, self.formes[forme])

                 
     def make_idformes(self) :
         self.idformes = dict([[self.formes[forme].ident, self.formes[forme]] for forme in self.formes])
                 
     def make_idformes(self) :
         self.idformes = dict([[self.formes[forme].ident, self.formes[forme]] for forme in self.formes])


@@ -254,14 +356,18 @@ class Corpus :
         if self.iduces is None :
             self.iduces = dict([[uce.ident, uce] for uci in self.ucis for uce in uci.uces])
 
         if self.iduces is None :
             self.iduces = dict([[uce.ident, uce] for uci in self.ucis for uce in uci.uces])
 

-    def make_lexitable(self, mineff, etoiles) :
-        tokeep = [lem for lem in self.lems if self.lems[lem].freq >= mineff]
+    def make_lexitable(self, mineff, etoiles, gram = 0) :
+        if gram == 0 :
+            grams = {1:'', 2:''}
+        else :
+            grams = {gram :''}
+        tokeep = [lem for lem in self.lems if self.lems[lem].freq >= mineff and self.lems[lem].act in grams]

         etuces = [[] for et in etoiles]
         for uci in self.ucis :
             get = list(set(uci.etoiles).intersection(etoiles))
             if len(get) > 1 :
         etuces = [[] for et in etoiles]
         for uci in self.ucis :
             get = list(set(uci.etoiles).intersection(etoiles))
             if len(get) > 1 :

-                return '2 variables sur la meme ligne'
-            elif get != [] :
+                log.info('2 variables sur une ligne')
+            if get != [] :

                 etuces[etoiles.index(get[0])] += [uce.ident for uce in uci.uces]
         etuces = [set(val) for val in etuces]
         tab = []
                 etuces[etoiles.index(get[0])] += [uce.ident for uce in uci.uces]
         etuces = [set(val) for val in etuces]
         tab = []


@@ -272,6 +378,49 @@ class Corpus :
         tab.insert(0, [''] + etoiles)
         return tab
     
         tab.insert(0, [''] + etoiles)
         return tab
     

+    def make_tgen_table(self, tgen, etoiles, tot = None):
+        lclasses = [self.getucesfrometoile(etoile) for etoile in etoiles]
+        sets = [set(cl) for cl in lclasses]
+        totoccurrences = dict([[val, 0] for val in etoiles])
+        if tot is None :
+            for forme in self.formes :
+                formeuceeff = self.getformeuceseff(forme)
+                for i, classe in enumerate(lclasses) :
+                    concern = sets[i].intersection(formeuceeff.keys())
+                    if len(concern) :
+                        totoccurrences[etoiles[i]] += sum([formeuceeff[uce] for uce in concern])
+        #tgenoccurrences = dict([[val, 0] for val in etoiles])
+        tgenoccurrences = {}
+        for t in tgen.tgen :
+            tgenoccurrences[t] = dict([[val, 0] for val in etoiles])
+            for lem in tgen[t] :
+                lemuceeff = self.getlemuceseff(lem)
+                for i, classe in enumerate(lclasses) :
+                    concern = sets[i].intersection(lemuceeff.keys())
+                    if len(concern) :
+                        tgenoccurrences[t][etoiles[i]] += sum([lemuceeff[uce] for uce in concern])
+        return tgenoccurrences, totoccurrences
+    
+    def make_tgen_profile(self, tgen, ucecl, uci = False) :
+        log.info('tgen/classes')
+        if uci :
+            tab = [[lem] + [len(set(self.gettgentxt(tgen[lem])).intersection(classe)) for classe in ucecl] for lem in tgen]
+        else :
+            tab = [[lem] + [len(set(self.gettgenst(tgen[lem])).intersection(classe)) for classe in ucecl] for lem in tgen]
+        tab = [[line[0]] + [val for val in line[1:]] for line in tab if sum(line[1:]) >= 3]
+        return tab
+        #i = 0
+        #nam = 'total'
+        #while nam + `i` in tgen :
+        #    i += 1
+        #nam = nam + `i`
+        #last = [nam] + [`len(classe)` for classe in ucecl]
+        #tab += [last]
+        #line0 = ['tgen'] + ['_'.join(['cluster', `i+1`]) for i in range(len(ucecl))]
+        #tab = [line0] + tab
+        #with open(fileout, 'w') as f :
+        #    f.write('\n'.join(['\t'.join(line) for line in tab]).encode(self.parametres['syscoding']))        
+    

     def make_efftype_from_etoiles(self, etoiles) :
         dtype = {}
         etuces = [[] for et in etoiles]
     def make_efftype_from_etoiles(self, etoiles) :
         dtype = {}
         etuces = [[] for et in etoiles]


@@ -390,28 +539,92 @@ class Corpus :
                     ident += 1
                     f.write('\n'.join([self.ucis[self.iduces[uce[0]].uci].paras[ident].encode(self.parametres['syscoding']), uce[1].encode(self.parametres['syscoding'])]) + '\n')
     
                     ident += 1
                     f.write('\n'.join([self.ucis[self.iduces[uce[0]].uci].paras[ident].encode(self.parametres['syscoding']), uce[1].encode(self.parametres['syscoding'])]) + '\n')
     

-    def export_corpus_classes(self, outf, alc = True, lem = False) :
+    def export_meta_table(self, outf) :
+        metas = [[`i`] + text.etoiles[1:] for i, text in enumerate(self.ucis)]
+        longueur_max = max([len(val) for val in metas])
+        first = ['column_%i' % i for i in range(longueur_max)]
+        metas.insert(0, first)
+        with open(outf, 'w') as f :
+            f.write('\n'.join(['\t'.join(line) for line in metas]).encode(self.parametres['syscoding']))
+    
+    def export_corpus_classes(self, outf, alc = True, lem = False, uci = False) :

         ucecl = {}
         for i, lc in enumerate(self.lc) :
             for uce in lc : 
                 ucecl[uce] = i + 1
         for uce in self.lc0 :
             ucecl[uce] = 0
         ucecl = {}
         for i, lc in enumerate(self.lc) :
             for uce in lc : 
                 ucecl[uce] = i + 1
         for uce in self.lc0 :
             ucecl[uce] = 0

-        res = self.getalluces()
-        self.make_iduces()
+        if not uci :
+            res = self.getalluces()
+            self.make_iduces()
+        else :
+            res = self.getallucis()

         with open(outf, 'w') as f :
             for uce in res :
                 guce = uce[1]
         with open(outf, 'w') as f :
             for uce in res :
                 guce = uce[1]

-                actuci = self.iduces[uce[0]].uci
+                if not uci :
+                    actuci = self.iduces[uce[0]].uci
+                else :
+                    actuci = uce[0]

                 if lem :
                     guce = ' '.join([self.formes[forme].lem for forme in guce.split()])
                 if alc :
                 if lem :
                     guce = ' '.join([self.formes[forme].lem for forme in guce.split()])
                 if alc :

-                    etline = ' '.join(self.ucis[self.iduces[uce[0]].uci].etoiles + ['*classe_%i' % ucecl[uce[0]]])
+                    etline = ' '.join(self.ucis[actuci].etoiles + ['*classe_%i' % ucecl[uce[0]]])

                 else :
                 else :

-                    etline = ' '.join(['<' + '='.join(et.split('_')) + '>' for et in self.ucis[self.iduces[uce[0]].uci].etoiles[1:]])
+                    etline = ' '.join(['<' + '='.join(et.split('_')) + '>' for et in self.ucis[actuci].etoiles[1:]])

                 f.write(etline.encode(self.parametres['syscoding']) + '\n')
                 f.write(guce.encode(self.parametres['syscoding']) + '\n\n')
 
                 f.write(etline.encode(self.parametres['syscoding']) + '\n')
                 f.write(guce.encode(self.parametres['syscoding']) + '\n\n')
 

+    def export_classe(self, outf, classe, lem = False, uci = False) :
+        sts = self.lc[classe - 1] 
+        if not uci :
+            res = self.getconcorde(sts)
+            self.make_iduces()
+        else :
+            res = self.getuciconcorde(sts)
+        with open(outf, 'w') as f :
+            for uce in res :
+                guce = uce[1]
+                if not uci :
+                    f.write(' '.join(self.ucis[self.iduces[uce[0]].uci].etoiles).encode(self.parametres['syscoding']) + '\n')
+                else :
+                    f.write(' '.join(self.ucis[uce[0]].etoiles).encode(self.parametres['syscoding']) + '\n')
+                if lem :
+                    guce = ' '.join([self.formes[forme].lem for forme in guce.split()])
+                f.write(guce.encode(self.parametres['syscoding']) + '\n\n')
+    
+    def export_owledge(self, rep, classe, lem = False, uci = False) :
+        sts = self.lc[classe - 1]
+        if not uci :
+            res = self.getconcorde(sts)
+            self.make_iduces()
+        else :
+            res = self.getuciconcorde(sts)
+        for uce in res :
+            ident = uce[0]
+            guce = uce[1]
+            outf = '.'.join([`ident`, 'txt'])
+            outf = os.path.join(rep, outf)
+            if lem :
+                guce = ' '.join([self.formes[forme].lem for forme in guce.split()])
+            with open(outf, 'w') as f :
+                f.write(guce.encode('cp1252', errors = 'replace'))
+
+    def export_tropes(self, fileout, classe, lem = False, uci = False) :
+        sts = self.lc[classe - 1]
+        if not uci :
+            res = self.getconcorde(sts)
+            self.make_iduces()
+        else :
+            res = self.getuciconcorde(sts)
+        with open(fileout, 'w') as f :
+            for uce in res :
+                guce = uce[1]
+                if lem :
+                    guce = ' '.join([self.formes[forme].lem for forme in guce.split()])
+                f.write(guce.encode('cp1252', errors = 'replace'))
+                f.write('\n')
+

     def make_and_write_sparse_matrix_from_uces(self, actives, outfile, listuce = False) :
         log.info('make_and_write_sparse_matrix_from_uces %s' % outfile)
         nbl = 0
     def make_and_write_sparse_matrix_from_uces(self, actives, outfile, listuce = False) :
         log.info('make_and_write_sparse_matrix_from_uces %s' % outfile)
         nbl = 0


@@ -464,15 +677,30 @@ class Corpus :
                     ffin.write(line)
         os.remove(outfile + '~')
     
                     ffin.write(line)
         os.remove(outfile + '~')
     

-    def make_table_with_classe(self, uces, list_act) :
+    def make_table_with_classe(self, uces, list_act, uci = False) :

         table_uce = [[0 for val in list_act] for line in range(0,len(uces))]
         uces = dict([[uce, i] for i, uce in enumerate(uces)])
         table_uce = [[0 for val in list_act] for line in range(0,len(uces))]
         uces = dict([[uce, i] for i, uce in enumerate(uces)])

+        if uci :
+            getlem = self.getlemucis
+        else :
+            getlem = self.getlemuces

         for i, lem in enumerate(list_act) :
         for i, lem in enumerate(list_act) :

-            lemuces = list(set(self.getlemuces(lem)).intersection(uces))
+            lemuces = list(set(getlem(lem)).intersection(uces))

             for uce in lemuces :
                 table_uce[uces[uce]][i] = 1
         table_uce.insert(0, list_act)
         return table_uce      
             for uce in lemuces :
                 table_uce[uces[uce]][i] = 1
         table_uce.insert(0, list_act)
         return table_uce      

+    
+    def make_pondtable_with_classe(self, uces, list_act) :
+        table_uce = [[0 for val in list_act] for line in range(0,len(uces))]
+        uces = dict([[uce, i] for i, uce in enumerate(uces)])
+        for i, lem in enumerate(list_act) :
+            uceseff = self.getlemuceseff(lem)
+            lemuces = list(set(uceseff.keys()).intersection(uces))
+            for uce in lemuces :
+                table_uce[uces[uce]][i] = uceseff[uce]
+        table_uce.insert(0, list_act)
+        return table_uce 

 
     def parse_active(self, gramact, gramsup = None) :
         log.info('parse actives')
 
     def parse_active(self, gramact, gramsup = None) :
         log.info('parse actives')


@@ -481,7 +709,7 @@ class Corpus :
                 self.lems[lem].act = 2
             elif self.lems[lem].gram in gramact :
                 self.lems[lem].act = 1
                 self.lems[lem].act = 2
             elif self.lems[lem].gram in gramact :
                 self.lems[lem].act = 1

-            elif gramsup is not None :
+            elif gramsup is not None and self.lems[lem].gram not in gramact:

                 if self.lems[lem].gram in gramsup :
                     self.lems[lem].act = 2
                 else :
                 if self.lems[lem].gram in gramsup :
                     self.lems[lem].act = 2
                 else :


@@ -501,6 +729,8 @@ class Corpus :
         allactives = [[self.lems[lem].freq, lem] for lem in self.lems if self.lems[lem].act == key and self.lems[lem].freq >= 3]
         self.activenb = len(allactives)
         allactives = sorted(allactives, reverse = True)
         allactives = [[self.lems[lem].freq, lem] for lem in self.lems if self.lems[lem].act == key and self.lems[lem].freq >= 3]
         self.activenb = len(allactives)
         allactives = sorted(allactives, reverse = True)

+        if self.activenb == 0 :
+            return [], 0

         if len(allactives) <= nbmax :
             log.info('nb = %i - eff min = %i ' % (len(allactives), allactives[-1][0]))
             return [val[1] for val in allactives], allactives[-1][0]
         if len(allactives) <= nbmax :
             log.info('nb = %i - eff min = %i ' % (len(allactives), allactives[-1][0]))
             return [val[1] for val in allactives], allactives[-1][0]


@@ -521,9 +751,12 @@ class Corpus :
         log.info('nb actives = %i - eff min = %i ' % (stop + 1, lim))
         return [val[1] for val in allactives[0:stop + 1]], lim
 
         log.info('nb actives = %i - eff min = %i ' % (stop + 1, lim))
         return [val[1] for val in allactives[0:stop + 1]], lim
 

-    def make_and_write_profile(self, actives, ucecl, fileout) :
+    def make_and_write_profile(self, actives, ucecl, fileout, uci = False) :

         log.info('formes/classes')
         log.info('formes/classes')

-        tab = [[lem] + [len(set(self.getlemuces(lem)).intersection(classe)) for classe in ucecl] for lem in actives]
+        if uci :
+            tab = [[lem] + [len(set(self.getlemucis(lem)).intersection(classe)) for classe in ucecl] for lem in actives]
+        else :
+            tab = [[lem] + [len(set(self.getlemuces(lem)).intersection(classe)) for classe in ucecl] for lem in actives]

         tab = [[line[0]] + [`val` for val in line[1:]] for line in tab if sum(line[1:]) >= 3]
         with open(fileout, 'w') as f :
             f.write('\n'.join([';'.join(line) for line in tab]).encode(self.parametres['syscoding']))
         tab = [[line[0]] + [`val` for val in line[1:]] for line in tab if sum(line[1:]) >= 3]
         with open(fileout, 'w') as f :
             f.write('\n'.join([';'.join(line) for line in tab]).encode(self.parametres['syscoding']))


@@ -531,8 +764,14 @@ class Corpus :
     def make_etoiles(self) :
         etoiles = set([])
         for uci in self.ucis :
     def make_etoiles(self) :
         etoiles = set([])
         for uci in self.ucis :

-            etoiles.update(uci.etoiles[1:] + uci.paras)
+            etoiles.update(uci.etoiles[1:])

         return list(etoiles)
         return list(etoiles)

+    
+    def make_themes(self):
+        themes = set([])
+        for uci in self.ucis :
+            themes.update(uci.paras)
+        return list(themes)

 
     def make_etoiles_dict(self) :
         etoiles = [et for uci in self.ucis for et in uci.etoiles[1:]]
 
     def make_etoiles_dict(self) :
         etoiles = [et for uci in self.ucis for et in uci.etoiles[1:]]


@@ -542,19 +781,41 @@ class Corpus :
             if et[0] in det :
                 try :
                     endet = '_'.join(et[1:])
             if et[0] in det :
                 try :
                     endet = '_'.join(et[1:])

-                    if endet in det[et[0]] :
-                        det[et[0]][endet] += 1
+                    if etoile in det[et[0]] :
+                        det[et[0]][etoile] += 1

                     else :
                     else :

-                        det[et[0]][endet] = 1
+                        det[et[0]][etoile] = 1

                 except IndexError :
                     det[et[0]] += 1
             else :
                 try :
                     endet = '_'.join(et[1:])
                 except IndexError :
                     det[et[0]] += 1
             else :
                 try :
                     endet = '_'.join(et[1:])

-                    det[et[0]] = {endet :1}
+                    det[et[0]] = {etoile :1}

                 except IndexError :
                     det[et[0]] = 1
         return det
                 except IndexError :
                     det[et[0]] = 1
         return det

+    
+    def make_theme_dict(self):
+        themes = [val for uci in self.ucis for val in uci.paras]
+        det = {}
+        for theme in themes :
+            th = theme.split('_')
+            if th[0] in det :
+                try :
+                    endth = '_'.join(th[1:])
+                    if theme in det[th[0]] :
+                        det[th[0]][theme] += 1
+                    else :
+                        det[th[0]][theme] = 1
+                except IndexError :
+                    det[th[0]] += 1
+            else :
+                try :
+                    endth = '_'.join(th[1:])
+                    det[th[0]] = {theme:1}
+                except IndexError :
+                    det[th[0]] = 1
+        return det

 
     def make_etline(self, listet) :
         etuces = [[] for et in listet]
 
     def make_etline(self, listet) :
         etuces = [[] for et in listet]


@@ -565,15 +826,21 @@ class Corpus :
             elif get != [] :
                 etuces[listet.index(get[0])] += [uce.ident for uce in uci.uces]
         return etuces
             elif get != [] :
                 etuces[listet.index(get[0])] += [uce.ident for uce in uci.uces]
         return etuces

-            

 
 

-    def make_and_write_profile_et(self, ucecl, fileout) :
+    def make_and_write_profile_et(self, ucecl, fileout, uci = False) :

         log.info('etoiles/classes')
         log.info('etoiles/classes')

-        etoiles = self.make_etoiles()
+        if not uci :
+            etoileuces = self.getetoileuces()
+        else :
+            etoileuces = self.getetoileucis()
+        etoileuces = dict([[et, etoileuces[et]] for et in etoileuces if len(etoileuces[et]) > 1])

         with open(fileout, 'w') as f :
         with open(fileout, 'w') as f :

-            f.write('\n'.join([';'.join([etoile] + [`len(set(self.getucesfrometoile(etoile)).intersection(classe))` for classe in ucecl]) for etoile in etoiles]).encode(self.parametres['syscoding']))
+            f.write('\n'.join([';'.join([et] + [`len(set(etoileuces[et]).intersection(classe))` for classe in ucecl]) for et in etoileuces]).encode(self.parametres['syscoding']))
+        #etoiles = self.make_etoiles()
+        #with open(fileout, 'w') as f :
+        #    f.write('\n'.join([';'.join([etoile] + [`len(set(self.getucesfrometoile(etoile)).intersection(classe))` for classe in ucecl]) for etoile in etoiles]).encode(self.parametres['syscoding']))

 
 

-    def make_colored_corpus(self) :
+    def make_colored_corpus(self, uci = False) :

         ucecl = {}
         for i, lc in enumerate(self.lc) :
             for uce in lc : 
         ucecl = {}
         for i, lc in enumerate(self.lc) :
             for uce in lc : 


@@ -585,17 +852,28 @@ class Corpus :
         <meta http-equiv="content-Type" content="text/html; charset=%s" />
         <body>
 ''' % sys.getdefaultencoding()
         <meta http-equiv="content-Type" content="text/html; charset=%s" />
         <body>
 ''' % sys.getdefaultencoding()

-        res = self.getalluces()
-        self.make_iduces()
-        actuci = ''
-        actpara = False
-        for uce in res :
-            if self.iduces[uce[0]].uci != actuci :
-                actuci = self.iduces[uce[0]].uci
-                txt += '<br><hr>' + ' '.join(self.ucis[self.iduces[uce[0]].uci].etoiles) + '<br><br>'
-                txt += '<font color="%s">' % (color[ucecl[uce[0]]]) + uce[1] + '</font><br><br>'
-            else :
-                txt += '<font color="%s">' % (color[ucecl[uce[0]]]) + uce[1] + '</font><br><br>'
+        if not uci :
+            res = self.getalluces()
+            self.make_iduces()
+            actuci = ''
+            actpara = False
+            for uce in res :
+                if self.iduces[uce[0]].uci != actuci :
+                    actuci = self.iduces[uce[0]].uci
+                    txt += '<br><hr>' + ' '.join(self.ucis[self.iduces[uce[0]].uci].etoiles) + '<br><br>'
+                    txt += '<font color="%s">' % (color[ucecl[uce[0]]]) + uce[1] + '</font><br><br>'
+                else :
+                    txt += '<font color="%s">' % (color[ucecl[uce[0]]]) + uce[1] + '</font><br><br>'
+        else :
+            res = self.getallucis()
+            actuci = ''
+            for uce in res :
+                if self.ucis[uce[0]].ident != actuci :
+                    actuci = self.ucis[uce[0]].ident
+                    txt += '<br><hr>' + ' '.join(self.ucis[self.ucis[uce[0]].ident].etoiles) + '<br><br>'
+                    txt += '<font color="%s">' % (color[ucecl[uce[0]]]) + uce[1] + '</font><br><br>'
+                else :
+                    txt += '<font color="%s">' % (color[ucecl[uce[0]]]) + uce[1] + '</font><br><br>'

         return txt + '\n</body></html>'
 
     def count_from_list(self, l, d) :
         return txt + '\n</body></html>'
 
     def count_from_list(self, l, d) :


@@ -627,9 +905,13 @@ class Corpus :
             l = l[-taille_limite:]
         return l
 
             l = l[-taille_limite:]
         return l
 

-    def find_segments_in_classe(self, list_uce, taille_segment, taille_limite):
+    def find_segments_in_classe(self, list_uce, taille_segment, taille_limite, uci = False):

         d={}
         d={}

-        for uce in self.getconcorde(list_uce) :
+        if not uci : 
+            concorde = self.getconcorde
+        else :
+            concorde = self.getuciconcorde
+        for uce in concorde(list_uce) :

             uce = uce[1].split()
             d =self.count_from_list([' '.join(uce[i:i+taille_segment]) for i in range(len(uce)-(taille_segment - 1))], d)
         l = [[d[val], val, taille_segment] for val in d if d[val] >= 3]
             uce = uce[1].split()
             d =self.count_from_list([' '.join(uce[i:i+taille_segment]) for i in range(len(uce)-(taille_segment - 1))], d)
         l = [[d[val], val, taille_segment] for val in d if d[val] >= 3]


@@ -683,27 +965,37 @@ class Corpus :
         self.lc0 = self.lc.pop(0)
         #return ucecl
     
         self.lc0 = self.lc.pop(0)
         #return ucecl
     

-    def get_stat_by_cluster(self, outf) :
+    def get_stat_by_cluster(self, outf, lclasses = None) :

         log.info('get_stat_by_cluster')
         log.info('get_stat_by_cluster')

+        if lclasses is None :
+            lclasses = self.lc

         t1 = time()
         t1 = time()

-        occurrences = dict([[i + 1, 0] for i in range(len(self.lc))])
-        formescl = dict([[i + 1, 0] for i in range(len(self.lc))])
-        hapaxcl = dict([[i + 1, 0] for i in range(len(self.lc))])
-        lenclasses = dict([[i+1,len(cl)] for i, cl in enumerate(self.lc)])
-        sets = [set(cl) for cl in self.lc]
+        occurrences = dict([[i + 1, 0] for i in range(len(lclasses))])
+        formescl = dict([[i + 1, 0] for i in range(len(lclasses))])
+        hapaxcl = dict([[i + 1, 0] for i in range(len(lclasses))])
+        lenclasses = dict([[i+1,len(cl)] for i, cl in enumerate(lclasses)])
+        sets = [set(cl) for cl in lclasses]

         for forme in self.formes :
             formeuceeff = self.getformeuceseff(forme)
         for forme in self.formes :
             formeuceeff = self.getformeuceseff(forme)

-            for i, classe in enumerate(self.lc) :
+            for i, classe in enumerate(lclasses) :

                 concern = sets[i].intersection(formeuceeff.keys())
                 if len(concern) :
                     occurrences[i+1] += sum([formeuceeff[uce] for uce in concern])
                     formescl[i+1] += 1
                     if self.formes[forme].freq == 1 :
                         hapaxcl[i+1] += 1
                 concern = sets[i].intersection(formeuceeff.keys())
                 if len(concern) :
                     occurrences[i+1] += sum([formeuceeff[uce] for uce in concern])
                     formescl[i+1] += 1
                     if self.formes[forme].freq == 1 :
                         hapaxcl[i+1] += 1

-        toprint = '\n'.join([';'.join([`i`, `occurrences[i]`, `formescl[i]`, `hapaxcl[i]`, `lenclasses[i]`, `float(hapaxcl[i])/float(formescl[i])`]) for i in occurrences])
-        with open(outf, 'w') as f :
-            f.write(toprint)

         log.info('%f' % (time() - t1))        
         log.info('%f' % (time() - t1))        

+        if outf is not None :
+            toprint = '\n'.join([';'.join([`i`, `occurrences[i]`, `formescl[i]`, `hapaxcl[i]`, `lenclasses[i]`, `float(hapaxcl[i])/float(formescl[i])`]) for i in occurrences])
+            with open(outf, 'w') as f :
+                f.write(toprint)
+        else :
+            return [[`occurrences[i]`, `formescl[i]`, `hapaxcl[i]`, `lenclasses[i]`, `float(hapaxcl[i])/float(formescl[i])`] for i in occurrences]
+
+    def get_stat_by_et(self, outf, etoiles) :
+        lclasses = [self.getucesfrometoile(etoile) for etoile in etoiles]
+        stats = self.get_stat_by_cluster(None, lclasses)
+        stats = [[etoiles[i]] + val for i, val in enumerate(stats)]

 
     def gethapaxbyet(self, etoiles) :
         hapaxuces = [self.getlemuces(forme)[0] for forme in self.lems if self.lems[forme].freq == 1]
 
     def gethapaxbyet(self, etoiles) :
         hapaxuces = [self.getlemuces(forme)[0] for forme in self.lems if self.lems[forme].freq == 1]


@@ -762,14 +1054,28 @@ class Corpus :
         with open('/tmp/testhapxuce.html','w') as f :
             f.write(txt)
 
         with open('/tmp/testhapxuce.html','w') as f :
             f.write(txt)
 

+    def export_dictionary(self, fileout, syscoding) :
+        listformes = [[self.formes[forme].freq, forme, self.formes[forme].lem, self.formes[forme].gram] for forme in self.formes]
+        listformes.sort(reverse = True)
+        listformes = [forme[1:] + [`forme[0]`] for forme in listformes]
+        with open(fileout, 'w') as f :
+            f.write('\n'.join(['\t'.join(forme) for forme in listformes]).encode(syscoding))
+
+    def export_lems(self, fileout, syscoding) :
+        self.make_idformes()
+        listlem = [[lem, '\t'.join(['\t'.join([self.idformes[forme].forme, `self.lems[lem].formes[forme]`]) for forme in self.lems[lem].formes])] for lem in self.lems]
+        listlem.sort()
+        with open(fileout, 'w') as f :
+            f.write('\n'.join(['\t'.join(lem) for lem in listlem]).encode(syscoding))
+    
+

 
 class MakeUciStat :
     def __init__(self, corpus) :
         ucinb = corpus.getucinb()
         ucisize = corpus.getucisize()
         ucimean = float(sum(ucisize))/float(ucinb)
 
 class MakeUciStat :
     def __init__(self, corpus) :
         ucinb = corpus.getucinb()
         ucisize = corpus.getucisize()
         ucimean = float(sum(ucisize))/float(ucinb)

-        detoile = corpus.make_etoiles_dict()
-        
+        detoile = corpus.make_etoiles_dict() 

 
 class Uci :
     def __init__(self, iduci, line, paraset = None) :
 
 class Uci :
     def __init__(self, iduci, line, paraset = None) :


@@ -893,7 +1199,7 @@ class BuildCorpus :
         self.corpus.pathout = PathOut(filename = parametres_corpus['originalpath'], dirout = parametres_corpus['pathout'])
         self.corpus.pathout.createdir(parametres_corpus['pathout'])
         self.corpus.parametres['uuid'] = str(uuid4())
         self.corpus.pathout = PathOut(filename = parametres_corpus['originalpath'], dirout = parametres_corpus['pathout'])
         self.corpus.pathout.createdir(parametres_corpus['pathout'])
         self.corpus.parametres['uuid'] = str(uuid4())

-        self.corpus.parametres['corpus_name'] = os.path.split(self.corpus.parametres['pathout'])[1]
+        self.corpus.parametres['corpus_name'] = parametres_corpus['corpus_name']#os.path.split(self.corpus.parametres['pathout'])[1]

         self.corpus.parametres['type'] = 'corpus'
         if self.corpus.parametres['keep_ponct'] :
             self.ponctuation_espace = [' ', '']
         self.corpus.parametres['type'] = 'corpus'
         if self.corpus.parametres['keep_ponct'] :
             self.ponctuation_espace = [' ', '']


@@ -963,7 +1269,7 @@ class BuildCorpus :
         self.cf.execute('CREATE INDEX ideff ON eff (id);')
         self.c.close()
         self.cf.close()
         self.cf.execute('CREATE INDEX ideff ON eff (id);')
         self.c.close()
         self.cf.close()

-        #backup corpora
+        #backup corpus

         self.conn_corpus = sqlite3.connect(self.corpus.pathout['corpus.db'])
         self.ccorpus = self.conn_corpus.cursor()
         self.ccorpus.execute('CREATE TABLE IF NOT EXISTS etoiles (uci INTEGER, et TEXT, paras TEXT);')
         self.conn_corpus = sqlite3.connect(self.corpus.pathout['corpus.db'])
         self.ccorpus = self.conn_corpus.cursor()
         self.ccorpus.execute('CREATE TABLE IF NOT EXISTS etoiles (uci INTEGER, et TEXT, paras TEXT);')


@@ -997,10 +1303,10 @@ class BuildCorpus :
             self.cleans.append(self.dotiret)
 
     def make_expression(self,txt) :
             self.cleans.append(self.dotiret)
 
     def make_expression(self,txt) :

-         for expression in self.expressions:
+        for expression in self.expressions:

             if expression in txt :
                 txt = txt.replace(expression, self.expressions[expression][0])
             if expression in txt :
                 txt = txt.replace(expression, self.expressions[expression][0])

-         return txt
+        return txt

     
     def dolower(self, txt) :
         return txt.lower()
     
     def dolower(self, txt) :
         return txt.lower()


@@ -1019,7 +1325,7 @@ class BuildCorpus :
     def firstclean(self, txt) :
         txt = txt.replace(u'’',"'")
         txt = txt.replace(u'œ', u'oe')
     def firstclean(self, txt) :
         txt = txt.replace(u'’',"'")
         txt = txt.replace(u'œ', u'oe')

-        return txt.replace('...',u' £$£ ').replace('?',' ? ').replace('.',' . ').replace('!', ' ! ').replace(',',' , ').replace(';', ' ; ').replace(':',' : ').replace(u'…', ' £$£ ')
+        return txt.replace('...',u' £$£ ').replace('?',' ? ').replace('.',' . ').replace('!', ' ! ').replace(',',' , ').replace(';', ' ; ').replace(':',' : ').replace(u'…', u' £$£ ')

 
     def make_cleans(self, txt) :
         for clean in self.cleans :
 
     def make_cleans(self, txt) :
         for clean in self.cleans :


@@ -1061,6 +1367,109 @@ class BuildCorpus :
         pourhapaxocc = (float(hapaxnb) / self.corpus.parametres['occurrences']) * 100
         self.corpus.parametres['hapax'] = '%i - %.2f %% des formes - %.2f %% des occurrences' % (hapaxnb, pourhapaxf, pourhapaxocc)
 
         pourhapaxocc = (float(hapaxnb) / self.corpus.parametres['occurrences']) * 100
         self.corpus.parametres['hapax'] = '%i - %.2f %% des formes - %.2f %% des occurrences' % (hapaxnb, pourhapaxf, pourhapaxocc)
 

+class BuildSubCorpus(BuildCorpus):
+    def __init__(self, corpus, parametres, dlg = None) :
+        log.info('begin subcorpus...')
+        self.dlg = dlg
+        self.ori = corpus
+        self.infile = None
+        self.corpus = Corpus(self, {'type' : 'corpus', 'originalpath' : corpus.parametres['originalpath'], 'encoding' : corpus.parametres['encoding']})
+        self.last = 0
+        self.parametres = parametres
+        self.encoding = corpus.parametres['encoding']
+        self.corpus.parametres['corpus_name'] = parametres['corpus_name']
+        self.corpus.pathout = PathOut(filename = corpus.parametres['originalpath'], dirout = parametres['pathout'])
+        self.corpus.pathout.createdir(parametres['pathout'])
+        self.corpus.parametres['pathout'] = parametres['pathout']
+        self.corpus.parametres['meta'] = parametres.get('meta', False)
+        self.corpus.parametres['uuid'] = str(uuid4())
+        if parametres.get('frommeta', False) :
+            print 'make subtexts'
+            self.corpus.ucis = [CopyUci(uci) for uci in self.ori.ucis if set(parametres['meta']).intersection(uci.etoiles) != set()]
+        elif parametres.get('fromtheme', False) :
+            print 'make subtexts from theme'
+            idpara = 0
+            for uci in self.ori.ucis :
+                if uci.paras != [] :
+                    newuce = []
+                    newpara = []
+                    for et in uci.paras :
+                        if et in parametres['meta'] :
+                            newuce += [CopyUce(uce) for uce in uci.uces if uce.para == idpara]
+                            newpara.append(et)
+                        idpara += 1
+                    if newuce != [] :
+                        nuci = CopyUci(uci)
+                        nuci.uces = newuce
+                        nuci.paras = newpara
+                        self.corpus.ucis.append(nuci)
+                else :
+                    idpara += 1  
+        elif parametres.get('fromclusters', False) :
+            self.parametres['uceids'] = [st for i in self.parametres['meta'] for st in self.parametres['lc'][i]]
+            self.fromuceids()
+        elif parametres.get('fromuceids', False) :
+            self.fromuceids()
+        #create database
+        self.connect()
+        self.dobuild()
+
+    def fromuceids(self):
+        print 'fromuceids'
+        dictucekeep = dict(zip(self.parametres['uceids'], self.parametres['uceids']))
+        idpara = 0
+        for uci in self.ori.ucis :
+            if uci.paras == [] :
+                keepuces = [CopyUce(uce) for uce in uci.uces if uce.ident in dictucekeep]
+                if keepuces != [] :
+                    nuci = CopyUci(uci)
+                    nuci.uces = keepuces
+                    self.corpus.ucis.append(nuci)
+                idpara += 1
+            else :
+                newuces = []
+                newpara = []
+                for et in uci.paras :
+                    keepuces = [CopyUce(uce) for uce in uci.uces if uce.ident in dictucekeep]
+                    idpara += 1
+                    if keepuces != [] :
+                        newuces += keepuces
+                        newpara.append(et)
+                if newuces != [] :
+                    nuci = CopyUci(uci)
+                    nuci.uces = newuces
+                    nuci.paras = newpara
+                    self.corpus.ucis.append(nuci)       
+    
+    def read_corpus(self, infile = None):
+        self.olduceid = [uce.ident for uci in self.corpus.ucis for uce in uci.uces]
+        ident_uci = 0
+        ident_uce = 0
+        ident_para = -1
+        lastpara = -1
+        newuceident = {}
+        print 'redo text, para and st ident'
+        for uci in self.corpus.ucis :
+            uci.ident = ident_uci
+            ident_uci += 1
+            for uce in uci.uces :
+                uce.uci = uci.ident
+                if uce.para != lastpara :
+                    ident_para += 1
+                    lastpara = uce.para
+                    uce.para = ident_para
+                else :
+                    uce.para = ident_para
+                newuceident[uce.ident] = ident_uce
+                uce.ident = ident_uce
+                ident_uce += 1
+        print 'backup st text and forms'
+        for row in self.ori.getconcorde(self.olduceid) :
+            self.c.execute('INSERT INTO uces VALUES(?,?);', (`newuceident[row[0]]`, row[1]))
+            for word in row[1].split() :
+                self.corpus.add_word_from_forme(self.ori.formes[word], newuceident[row[0]])
+        self.backup_uce()
+        print 'done'

 
 class BuildFromAlceste(BuildCorpus) :
     def read_corpus(self, infile) :
 
 class BuildFromAlceste(BuildCorpus) :
     def read_corpus(self, infile) :


@@ -1079,7 +1488,7 @@ class BuildFromAlceste(BuildCorpus) :
         try :
             with codecs.open(infile, 'r', self.encoding) as f :
                 for linenb, line in enumerate(f) :
         try :
             with codecs.open(infile, 'r', self.encoding) as f :
                 for linenb, line in enumerate(f) :

-                    line = line.rstrip('\n\r')
+                    line = line.rstrip('\n\r')#FIXME .lstrip(codecs.BOM).lstrip(codecs.BOM_UTF8)

                     if self.testuci(line) :
                         iduci += 1
                         if txt != [] :
                     if self.testuci(line) :
                         iduci += 1
                         if txt != [] :


@@ -1092,7 +1501,6 @@ class BuildFromAlceste(BuildCorpus) :
                                     log.info(u'Empty text : %i' % linenb)
                                     iduci -= 1
                                     self.corpus.ucis.pop()
                                     log.info(u'Empty text : %i' % linenb)
                                     iduci -= 1
                                     self.corpus.ucis.pop()

-                                    #raise Exception("EmptyText %i" % linenb)

                             self.corpus.ucis.append(Uci(iduci, line))
                         if self.dlg is not None :
                             if not (iduci + 1) % 10 :
                             self.corpus.ucis.append(Uci(iduci, line))
                         if self.dlg is not None :
                             if not (iduci + 1) % 10 :


@@ -1105,19 +1513,24 @@ class BuildFromAlceste(BuildCorpus) :
                             idpara += 1
                             self.corpus.ucis[-1].paras.append(line.split()[0])
                         else :
                             idpara += 1
                             self.corpus.ucis[-1].paras.append(line.split()[0])
                         else :

-                            raise Exception('paragrapheOT')
+                            raise Exception('paragrapheOT %i' % linenb)

                     elif line.strip() != '' and iduci != -1 :
                         txt.append(line)
             if txt != [] and iduci != -1 :
                 iduce, idpara = self.treattxt(txt, iduce, idpara, iduci)
                 del(txt)
             else :
                     elif line.strip() != '' and iduci != -1 :
                         txt.append(line)
             if txt != [] and iduci != -1 :
                 iduce, idpara = self.treattxt(txt, iduce, idpara, iduci)
                 del(txt)
             else :

-                raise Exception("EmptyText")
+                if iduci != -1 :
+                    iduci -= 1
+                    self.corpus.ucis.pop()
+                    log.info(Exception("Empty text %i" % linenb))
+                else :
+                    raise Exception('EmptyText %i' % linenb)

             if iduci != -1  and iduce != -1:
                 self.backup_uce()
             else : 
             if iduci != -1  and iduce != -1:
                 self.backup_uce()
             else : 

-                log.info(_(u"No Text in corpora. Are you sure of the formatting ?"))
-                raise Exception('TextBeforeTextMark')
+                log.info(_(u"No Text in corpus. Are you sure of the formatting ?"))
+                raise Exception('TextBeforeTextMark %i' % linenb)

         except UnicodeDecodeError :
             raise Exception("CorpusEncoding")
 
         except UnicodeDecodeError :
             raise Exception("CorpusEncoding")
 


@@ -1178,6 +1591,7 @@ class Builder :
         self.dlg = dlg
         parametres = DoConf(os.path.join(self.parent.UserConfigPath,'corpus.cfg')).getoptions('corpus')
         parametres['pathout'] = PathOut(parent.filename, 'corpus').mkdirout()
         self.dlg = dlg
         parametres = DoConf(os.path.join(self.parent.UserConfigPath,'corpus.cfg')).getoptions('corpus')
         parametres['pathout'] = PathOut(parent.filename, 'corpus').mkdirout()

+        parametres['corpus_name'] = os.path.split(parametres['pathout'])[1]

         dial = CorpusPref(parent, parametres)
         dial.CenterOnParent()
         dial.txtpath.SetLabel(parent.filename)
         dial = CorpusPref(parent, parametres)
         dial.CenterOnParent()
         dial.txtpath.SetLabel(parent.filename)


@@ -1187,20 +1601,72 @@ class Builder :
             parametres = dial.doparametres()
             parametres['originalpath'] = parent.filename
             PathOut().createdir(parametres['pathout'])
             parametres = dial.doparametres()
             parametres['originalpath'] = parent.filename
             PathOut().createdir(parametres['pathout'])

-            ReadLexique(self.parent, lang = parametres['lang'])
-            self.parent.expressions = ReadDicoAsDico(self.parent.DictPath.get(parametres['lang']+'_exp', 'french_exp'))
+            if parametres.get('dictionary', False) :
+                filein = parametres['dictionary']
+            else :
+                filein = None
+            if dial.corpusname.GetValue() != '' :
+                parametres['corpus_name'] = dial.corpusname.GetValue()
+            dial.Destroy()
+            ReadLexique(self.parent, lang = parametres['lang'], filein = filein)
+            if parametres['lang'] != 'other' and  os.path.exists(self.parent.DictPath.get(parametres['lang']+'_exp', 'french_exp')):
+                self.parent.expressions = ReadDicoAsDico(self.parent.DictPath.get(parametres['lang']+'_exp', 'french_exp'))
+            else :
+                self.parent.expressions = {}

             self.parametres = parametres
         else :
             self.parametres = parametres
         else :

+            dial.Destroy()

             if self.dlg is not None :
                 self.dlg.Destroy()
             if self.dlg is not None :
                 self.dlg.Destroy()

-        dial.Destroy()

 
     def doanalyse(self) :
         return BuildFromAlceste(self.parent.filename, self.parametres, self.parent.lexique, self.parent.expressions, dlg = self.dlg).corpus
 
 
     def doanalyse(self) :
         return BuildFromAlceste(self.parent.filename, self.parametres, self.parent.lexique, self.parent.expressions, dlg = self.dlg).corpus
 

-
-if __name__ == '__main__' :
-    t1 = time()
-    parametres = {'formesdb':'formes.db', 'ucesdb': 'uces.db', 'corpusdb' : 'corpus.db', 'syscoding' : 'utf-8', 'encoding' : encoding}
-    intro = BuildCorpus(infile, parametres)#, tar_in, tar_infouce)#, tar_formes)
-    print time() - t1
+class SubBuilder :
+    def __init__(self, parent, corpus, parametres = None, dlg = None):
+        self.parent = parent
+        self.ori = corpus
+        self.dlg = dlg
+        corpus_name = 'Sub' + corpus.parametres['corpus_name']
+        if dlg is not None :
+            busy = wx.BusyInfo(_("Please wait...").decode('utf8'), self)
+            wx.SafeYield()
+        parametres['corpus_name'] = corpus_name
+        if parametres.get('frommeta', False) :
+            parametres['meta'] = corpus.make_etoiles()
+        elif parametres.get('fromtheme', False) :
+            parametres['meta'] = corpus.make_themes()
+        elif parametres.get('fromclusters', False) :
+            parametres['meta'] = [' '.join(['classe', `i`]) for i in range(1,parametres['clnb'] + 1)]
+        else :
+            parametres['meta'] = []
+        if 'fromclusters' not in parametres :
+            parametres['meta'].sort()
+        if dlg is not None :
+            del busy
+        dial = SubTextFromMetaDial(parent, parametres)
+        self.res = dial.ShowModal()
+        if self.res == 5100 :
+            if dial.subcorpusname.GetValue() != '' :
+                corpus_name = ''.join([l for l in dial.subcorpusname.GetValue() if l.isalnum() or l in ['_']])
+            if corpus_name != '' :
+                parametres['corpus_name'] = corpus_name
+            else :
+                parametres['corpus_name'] = 'Sub' + corpus.parametres['corpus_name']
+            pathout = os.path.join(corpus.parametres['pathout'], parametres['corpus_name'])
+            i = 1
+            while os.path.exists(pathout + '_%i' % i) :
+                i += 1
+            parametres['pathout'] = pathout + '_%i' % i
+            meta = dial.m_listBox1.GetSelections()
+            if not 'fromclusters' in parametres :
+                parametres['meta'] = [parametres['meta'][val] for val in meta]
+            else :
+                parametres['meta'] = meta
+            self.parametres = parametres
+            dial.Destroy()
+        else :
+            dial.Destroy()
+    
+    def doanalyse(self):
+        return BuildSubCorpus(self.ori, parametres = self.parametres, dlg = self.dlg).corpus