X-Git-Url: http://iramuteq.org/git?p=iramuteq;a=blobdiff_plain;f=functions.py;h=6a3d9689a4298c5cfbf9bf41d78c9006a80aa886;hp=bb7e9063280d0a2d17bea947d31c59fe98f5ca5c;hb=ae32485b960c98387ae7987751198335ad549ab3;hpb=eb3837fa8ebc881ef1df81980ec576ca0451eb40 diff --git a/functions.py b/functions.py old mode 100644 new mode 100755 index bb7e906..6a3d968 --- a/functions.py +++ b/functions.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- #Author: Pierre Ratinaud #Copyright (c) 2008-2012 Pierre Ratinaud -#Lisense: GNU/GPL +#License: GNU/GPL import wx import re @@ -20,6 +20,8 @@ import locale import datetime from copy import copy from shutil import copyfile +import shelve +import json #from dialog import BugDialog import logging @@ -29,89 +31,266 @@ log = logging.getLogger('iramuteq') indices_simi = [u'cooccurrence' ,'pourcentage de cooccurrence',u'Russel',u'Jaccard', 'Kulczynski1', 'Kulczynski2', 'Mountford', 'Fager', 'simple matching', 'Hamman', 'Faith', 'Tanimoto', 'Dice', 'Phi', 'Stiles', 'Michael', 'Mozley', 'Yule', 'Yule2', 'Ochiai', 'Simpson', 'Braun-Blanquet','Chi-squared', 'Phi-squared', 'Tschuprow', 'Cramer', 'Pearson', 'binomial'] + +def open_folder(folder): + if sys.platform == "win32": + os.startfile(folder) + else: + opener ="open" if sys.platform == "darwin" else "xdg-open" + #call([opener, folder]) + call([u"%s %s &" % (opener, folder)], shell=True) + +def normpath_win32(path) : + if not sys.platform == 'win32' : + return path + while '\\\\' in path : + path = path.replace('\\\\', '\\') + if path.startswith('\\') and not path.startswith('\\\\') : + path = '\\' + path + return path + +class TGen : + def __init__(self, path = None, encoding = 'utf8'): + self.path = path + self.tgen = {} + self.encoding = encoding + + def __getitem__(self, key): + return self.tgen[key] + + def read(self, path = None): + if path is None : + path = self.path + with codecs.open(path, 'r', self.encoding) as f : + tgen = f.read() + tgen = [line.split('\t') for line in tgen.splitlines()] + tgen = dict([[line[0], line[1:]] for line in tgen]) + self.tgen = tgen + self.path = path + + def write(self, path = None): + if path is None : + path = self.path + with open(path, 'w') as f : + f.write('\n'.join(['\t'.join([val] + self.tgen[val]) for val in self.tgen]).encode(self.encoding)) + + def writetable(self, pathout, tgens, totocc): + etoiles = totocc.keys() + etoiles.sort() + with open(pathout, 'w') as f : + line = '\t'.join([u'tgens'] + etoiles) + '\n' + f.write(line.encode(self.encoding)) + for t in tgens : + line = '\t'.join([t] + [`tgens[t][et]` for et in etoiles]) + '\n' + f.write(line.encode(self.encoding)) + i = 0 + totname = 'total' + while totname + `i` in tgens : + i += 1 + totname = totname + `i` + line = '\t'.join([totname] + [`totocc[et]` for et in etoiles]) + '\n' + f.write(line.encode(self.encoding)) + class History : def __init__(self, filein, syscoding = 'utf8') : self.filein = filein self.syscoding = syscoding - self.corpora = {} + self.corpus = {} self.openedcorpus = {} + self.openedmatrix = {} + self.orph = [] self.analyses = {} - self.history = {} + self.history = [] self.opened = {} self.read() def read(self) : - self.conf = DoConf(self.filein) - self.order = {} - self.ordera = {} - for i, section in enumerate(self.conf.conf.sections()) : - if self.conf.conf.has_option(section, 'corpus_name') : - self.corpora[section] = self.conf.getoptions(section) - self.order[len(self.order)] = section - else : - self.analyses[section] = self.conf.getoptions(section) - self.ordera[len(self.ordera)] = section - todel = [] - for corpus in self.corpora : - self.history[corpus] = copy(self.corpora[corpus]) - for analyse in self.analyses : - if self.analyses[analyse]['corpus'] in self.corpora : - if 'analyses' in self.history[self.analyses[analyse]['corpus']] : - self.history[self.analyses[analyse]['corpus']]['analyses'].append(self.analyses[analyse]) - todel.append(analyse) - else : - self.history[self.analyses[analyse]['corpus']]['analyses'] = [self.analyses[analyse]] - todel.append(analyse) - else : - self.history[analyse] = self.analyses[analyse] - #for analyse in todel : - # del self.analyses[analyse] - + d = shelve.open(self.filein) + self.history = d.get('history', []) + self.matrix = d.get('matrix', []) + self.ordercorpus = dict([[corpus['uuid'], i] for i, corpus in enumerate(self.history)]) + self.corpus = dict([[corpus['uuid'], corpus] for corpus in self.history]) + self.analyses = dict([[analyse['uuid'], analyse] for corpus in self.history for analyse in corpus.get('analyses', [])]) + self.matrixanalyse = dict([[mat['uuid'], mat] for mat in self.matrix]) + self.ordermatrix = dict([[matrix['uuid'], i] for i, matrix in enumerate(self.matrix)]) + d.close() + d = {} + d['history'] = self.history + d['matrix'] = self.matrix +# with open('/home/pierre/hystory.json', 'w') as f : +# f.write(json.dumps(d, indent=4, default=str)) + def write(self) : - sections = self.corpora.keys() + self.analyses.keys() - parametres = [self.corpora[key] for key in self.corpora.keys() if key != 'analyses'] + [self.analyses[key] for key in self.analyses.keys()] - self.conf.makeoptions(sections, parametres) - log.info('write history') + d = shelve.open(self.filein) + d['history'] = self.history + d['matrix'] = self.matrix + d.close() def add(self, analyse) : + log.info('add to history %s' % analyse.get('corpus_name', 'pas un corpus')) tosave = {'uuid' : analyse['uuid'], 'ira': analyse['ira'], 'type' : analyse['type']} + if tosave['uuid'] in self.corpus : + log.info('problem : this uuid is already in history : %s' % tosave['uuid']) + return if analyse.get('corpus', False) : + if analyse['uuid'] in self.analyses : + return tosave['corpus'] = analyse['corpus'] tosave['name'] = analyse['name'] acorpus_uuid = analyse['corpus'] - if acorpus_uuid in self.corpora : - if 'analyses' in self.history[acorpus_uuid] : - self.history[acorpus_uuid]['analyses'].append(tosave) + if acorpus_uuid in self.corpus : + if 'analyses' in self.history[self.ordercorpus[acorpus_uuid]] : + self.history[self.ordercorpus[acorpus_uuid]]['analyses'].append(tosave) else : - self.history[acorpus_uuid]['analyses'] = [tosave] - self.analyses[analyse['uuid']] = tosave + self.history[self.ordercorpus[acorpus_uuid]]['analyses'] = [tosave] else : - self.analyses[analyse['uuid']] = tosave - elif 'corpus_name' in analyse : + self.orph.append(tosave) + else : tosave['corpus_name'] = analyse['corpus_name'] - self.history[analyse['uuid']] = tosave - self.corpora[analyse['uuid']] = tosave + #self.ordercorpus[tosave['uuid']] = len(history) + #self.corpus[tosave['uuid']] = analyse + self.history.append(tosave) self.write() + self.read() - def delete(self, uuid, corpus = False) : + def addMatrix(self, analyse) : + tosave = analyse + #tosave['matrix_name'] = analyse['matrix_name'] + tosave['analyses'] = [] + self.matrix.append(tosave) + self.write() + self.read() + + def addMatrixAnalyse(self, analyse) : + tosave = {'uuid' : analyse['uuid'], 'ira': analyse['ira'], 'type' : analyse['type'], 'matrix' : analyse['matrix']} + tosave['name'] = analyse['name'] + if tosave['matrix'] in self.ordermatrix : + self.matrix[self.ordermatrix[tosave['matrix']]]['analyses'].append(tosave) + self.write() + self.read() + + def addmultiple(self, analyses) : + log.info('add multiple') + for analyse in analyses : + tosave = {'uuid' : analyse['uuid'], 'ira': analyse['ira'], 'type' : analyse['type']} + corpus = analyse['corpus'] + tosave['corpus'] = corpus + tosave['name'] = analyse['name'] + if corpus in self.corpus : + if 'analyses' in self.history[self.ordercorpus[corpus]] : + self.history[self.ordercorpus[corpus]]['analyses'].append(tosave) + else : + self.history[self.ordercorpus[corpus]]['analyses'] = [tosave] + self.write() + self.read() + + def delete(self, analyse, corpus = False) : + log.info('delete %s' % analyse.get('name', 'noname')) if corpus : - del self.corpora[uuid] - self.conf.conf.remove_section(uuid) - for analyse in self.history[uuid].get('analyses', [False]) : - if analyse : - del self.analyses[analyse['uuid']] - self.conf.conf.remove_section(analyse['uuid']) - else : - del self.analyses[uuid] - self.conf.conf.remove_section(uuid) + self.history.pop(self.ordercorpus[analyse['uuid']]) + if analyse['uuid'] in self.openedcorpus : + del self.openedcorpus[analyse['uuid']] + log.info('delete corpus : %s' % analyse['uuid']) + elif analyse['uuid'] in self.analyses : + todel = [i for i, ana in enumerate(self.corpus[analyse['corpus']]['analyses']) if ana['uuid'] == analyse['uuid']][0] + self.history[self.ordercorpus[analyse['corpus']]]['analyses'].pop(todel) + elif analyse['uuid'] in self.matrixanalyse : + self.matrix = [mat for mat in self.matrix if mat['uuid'] != analyse['uuid']] + elif analyse.get('matrix', False) in self.matrixanalyse : + analyses = self.matrix[self.ordermatrix[analyse['matrix']]]['analyses'] + topop = [i for i, val in enumerate(analyses) if analyse['uuid'] == val['uuid']][0] + analyses.pop(topop) + self.matrix[self.ordermatrix[analyse['matrix']]]['analyses'] = analyses self.write() + self.read() def addtab(self, analyse) : self.opened[analyse['uuid']] = analyse def rmtab(self, analyse) : del self.opened[analyse['uuid']] - + + def update(self, analyse) : + if 'matrix_name' in analyse : + self.matrixanalyse[analyse['uuid']].update(analyse) + elif 'corpus_name' in analyse : + self.corpus[analyse['uuid']].update(analyse) + elif 'corpus' in analyse : + self.analyses[analyse['uuid']].update(analyse) + else : + toupdate = [an for an in self.matrixanalyse[analyse['matrix']]['analyses'] if an['uuid'] == analyse['uuid']] + toupdate[0].update(analyse) + self.write() + self.read() + + def clean(self) : + corpustodel = [corpus for corpus in self.history if not os.path.exists(corpus['ira'])] + print corpustodel + for corpus in corpustodel : + print 'cleaning :', corpus['corpus_name'] + self.delete(corpus, corpus = True) + anatodel = [analyse for corpus in self.history for analyse in corpus.get('analyses', []) if not os.path.exists(analyse.get('ira', '/'))] + for analyse in anatodel : + print 'cleaning :', analyse['name'] + self.delete(analyse) + + def dostat(self): + todel = {} + tokens = 0 + corpusnb = {} + subnb = 0 + analysenb = 0 + hours = 0 + minutes = 0 + secondes = 0 + ha = 0 + ma = 0 + sa = 0 + for corpus in self.history : + analysenb += len(corpus.get('analyses', [])) + analyses = corpus.get('analyses', []) + for analyse in analyses : + if os.path.exists(analyse['ira']) : + ana = DoConf(analyse['ira']).getoptions() + if 'time' in ana : + time = ana['time'].split() + ha += int(time[0].replace('h','')) * 3600 + ma += int(time[1].replace('m','')) * 60 + sa += int(time[2].replace('s','')) + if os.path.exists(corpus['ira']) : + param = DoConf(corpus['ira']).getoptions() + time = param.get('time','0h 0m 0s') + time = time.split() + hours += int(time[0].replace('h','')) * 3600 + minutes += int(time[1].replace('m','')) * 60 + secondes += int(time[2].replace('s','')) + if param.get('originalpath', False) : + if param['originalpath'] in corpusnb : + corpusnb[param['originalpath']] += 1 + tokens += int(param['occurrences']) + else : + corpusnb[param['originalpath']] = 1 + #print param + else : + subnb += 1 + else : + if corpus['ira'] in todel : + todel['ira'] += 1 + else : + todel['ira'] = 1 + print u'Nbr total de corpus : %s' % len(self.history) + corpus_nb = len(corpusnb) + len(todel) + print u'Nbr de corpus différents : %s' % corpus_nb + lentodel = len(todel) + print u'Nbr de corpus à supprimer : %s' % lentodel + print u'Nbr de sous corpus : %s' % subnb + print u"Nbr total d'occurrences : %s" % tokens + print u'Moyenne occurrences par corpus : %f' % (tokens/corpus_nb) + print '---------------------' + print u"Nbr total d'analyses : %s" % analysenb + print u'Temps total indexation : %f h' % ((hours+minutes+secondes) / 3600) + print u'Temps total analyses : %f h' % ((ha+ma+sa) / 3600) + def __str__(self) : return str(self.history) @@ -119,7 +298,9 @@ class DoConf : def __init__(self, configfile=None, diff = None, parametres = None) : self.configfile = configfile self.conf = ConfigParser() + if configfile is not None : + configfile = normpath_win32(configfile) self.conf.readfp(codecs.open(configfile, 'r', 'utf8')) self.parametres = {} if parametres is not None : @@ -144,37 +325,54 @@ class DoConf : parametres[option] = True elif self.conf.get(section, option).startswith('(') and self.conf.get(section, option).endswith(')') : parametres[option] = ast.literal_eval(self.conf.get(section, option)) + elif self.conf.get(section, option).startswith('[') and self.conf.get(section, option).endswith(']') : + parametres[option] = ast.literal_eval(self.conf.get(section, option)) else : parametres[option] = self.conf.get(section, option) if 'type' not in parametres : parametres['type'] = section return parametres - + def makeoptions(self, sections, parametres, outfile = None) : + txt = '' for i, section in enumerate(sections) : + txt += '[%s]\n' % section if not self.conf.has_section(section) : self.conf.add_section(section) for option in parametres[i] : if isinstance(parametres[i][option], int) : self.conf.set(section, option, `parametres[i][option]`) + txt += '%s = %i\n' % (option, parametres[i][option]) elif isinstance(parametres[i][option], basestring) : self.conf.set(section, option, parametres[i][option].encode('utf8')) + txt += '%s = %s\n' % (option, parametres[i][option]) elif isinstance(parametres[i][option], wx.Colour) : self.conf.set(section, option, str(parametres[i][option])) + txt += '%s = %s\n' % (option, str(parametres[i][option])) + elif option == 'analyses' : + pass else : self.conf.set(section, option, `parametres[i][option]`) + txt += '%s = %s\n' % (option, `parametres[i][option]`) if outfile is None : outfile = self.configfile - with codecs.open(outfile, 'w', 'utf8') as f : - self.conf.write(f) + outfile = normpath_win32(outfile) + with open(outfile, 'w') as f : + f.write(txt.encode('utf8')) + #self.conf.write(f) def totext(self, parametres) : - txt = ['Corpus'] + #txt = ['Corpus'] + txt = [] for val in parametres : if isinstance(parametres[val], int) : txt.append(' \t\t: '.join([val, `parametres[val]`])) - else : + elif isinstance(parametres[val], basestring) : txt.append(' \t\t: '.join([val, parametres[val]])) + elif val in ['listet', 'stars'] : + pass + else : + txt.append(' \t\t: '.join([val, `parametres[val]`])) return '\n'.join(txt) @@ -185,10 +383,12 @@ def write_tab(tab, fileout) : class BugDialog(wx.Dialog): def __init__(self, *args, **kwds): # begin wxGlade: MyDialog.__init__ - kwds["style"] = wx.DEFAULT_DIALOG_STYLE + kwds["style"] = wx.DEFAULT_DIALOG_STYLE | wx.STAY_ON_TOP kwds["size"] = wx.Size(500, 200) wx.Dialog.__init__(self, *args, **kwds) + self.SetTitle(kwds['title']) self.text_ctrl_1 = wx.TextCtrl(self, -1, "", style=wx.TE_MULTILINE) + self.text_ctrl_1.SetBackgroundColour('#DDE8EB') self.button_1 = wx.Button(self, wx.ID_OK, "") self.__set_properties() @@ -197,10 +397,9 @@ class BugDialog(wx.Dialog): def __set_properties(self): # begin wxGlade: MyDialog.__set_properties - self.SetTitle("Bug") self.SetMinSize(wx.Size(500, 200)) self.text_ctrl_1.SetMinSize(wx.Size(500, 200)) - + # end wxGlade def __do_layout(self): @@ -238,7 +437,7 @@ def sortedby(list, direct, *indices): sortedby(list,0) will return [[1, 2], [2, 3], [3, 1]] """ - nlist = map(lambda x, indices=indices: + nlist = map(lambda x, indices=indices: map(lambda i, x=x: x[i], indices) + [x], list) if direct == 1: @@ -268,33 +467,26 @@ def treat_line_alceste(i, line) : line[5] = str(float(line[5].replace(',', '.')))[0:7] return [i, int(line[0]), int(line[1]), float(line[2]), float(line[3]), line[6], line[4], line[5]] -def ReadProfileAsDico(parent, File, Alceste=False, encoding = sys.getdefaultencoding()): - #print 'lecture des profils : ReadProfileAsDico' - #if Alceste : - # print 'lecture du dictionnaire de type' - # dictlem = {} - # for line in parent.corpus.lem_type_list : - # dictlem[line[0]] = line[1] +def ReadProfileAsDico(File, Alceste=False, encoding = sys.getdefaultencoding()): dictlem = {} print 'lecture des profiles' - #encoding = sys.getdefaultencoding() - print encoding FileReader = codecs.open(File, 'r', encoding) Filecontent = FileReader.readlines() FileReader.close() DictProfile = {} count = 0 - rows = [row.replace('\n', '').replace("'", '').replace('\"', '').replace(',', '.').replace('\r','').split(';') for row in Filecontent] + #rows = [row.replace('\n', '').replace("'", '').replace('\"', '').replace(',', '.').replace('\r','').split(';') for row in Filecontent] + rows = [row.replace('\n', '').replace("'", '').replace('\"', '').replace('\r','').split(';') for row in Filecontent] rows.pop(0) ClusterNb = rows[0][2] rows.pop(0) clusters = [row[2] for row in rows if row[0] == u'**'] valclusters = [row[1:4] for row in rows if row[0] == u'****'] lp = [i for i, line in enumerate(rows) if line[0] == u'****'] - prof = [rows[lp[i] + 1:lp[i+1] - 1] for i in range(0, len(lp)-1)] + [rows[lp[-1] + 1:len(rows)]] + prof = [rows[lp[i] + 1:lp[i+1] - 1] for i in range(0, len(lp)-1)] + [rows[lp[-1] + 1:len(rows)]] if Alceste : prof = [[add_type(row, dictlem) for row in pr] for pr in prof] - prof = [[treat_line_alceste(i,line) for i, line in enumerate(pr)] for pr in prof] + prof = [[treat_line_alceste(i,line) for i, line in enumerate(pr)] for pr in prof] else : prof = [[line + [''] for line in pr] for pr in prof] prof = [[treat_line_alceste(i,line) for i, line in enumerate(pr)] for pr in prof] @@ -302,23 +494,26 @@ def ReadProfileAsDico(parent, File, Alceste=False, encoding = sys.getdefaultenco DictProfile[cluster] = [valclusters[i]] + prof[i] return DictProfile -def GetTxtProfile(dictprofile) : +def GetTxtProfile(dictprofile, cluster_size) : proflist = [] for classe in range(0, len(dictprofile)) : prof = dictprofile[str(classe + 1)] - clinfo = prof[0] + clinfo = cluster_size[classe] proflist.append('\n'.join([' '.join(['classe %i' % (classe + 1), '-', '%s uce sur %s - %s%%' % (clinfo[0], clinfo[1], clinfo[2])]), '\n'.join(['%5s|%5s|%6s|%6s|%8s|%8s|%20s\t%10s' % tuple([str(val) for val in line]) for line in prof if len(line)==8])])) return '\n\n'.join(proflist) def formatExceptionInfo(maxTBlevel=5): - cla, exc, trbk = sys.exc_info() - excName = cla.__name__ - try: - excArgs = exc.__dict__["args"] - except KeyError: - excArgs = "" - excTb = traceback.format_tb(trbk, maxTBlevel) - return (excName, excArgs, excTb) + cla, exc, trbk = sys.exc_info() + try : + excName = cla.__name__ + except : + excName = 'None' + try: + excArgs = exc.args[0] + except : + excArgs = "" + excTb = traceback.format_tb(trbk, maxTBlevel) + return (excName, excArgs, excTb) #fonction des etudiants de l'iut @@ -328,16 +523,16 @@ def decoupercharact(chaine, longueur, longueurOptimale, separateurs = None) : Si on trouve un '$', c'est fini. Sinon, on cherche le meilleur candidat. C'est-à-dire le rapport poids/distance le plus important. """ - separateurs = [[u'.', 60.0], [u'?', 60.0], [u'!', 60.0], [u'£', 60], [u':', 50.0], [u';', 40.0], [u',', 10.0], [u' ', 0.1]] + separateurs = [[u'.', 60.0], [u'?', 60.0], [u'!', 60.0], [u'£$£', 60], [u':', 50.0], [u';', 40.0], [u',', 10.0], [u' ', 0.1]] trouve = False # si on a trouvé un bon séparateur iDecoupe = 0 # indice du caractere ou il faut decouper - + # on découpe la chaine pour avoir au maximum 240 caractères longueur = min(longueur, len(chaine) - 1) chaineTravail = chaine[:longueur + 1] nbCar = longueur meilleur = ['', 0, 0] # type, poids et position du meilleur separateur - + # on vérifie si on ne trouve pas un '$' indice = chaineTravail.find(u'$') if indice > -1: @@ -354,8 +549,8 @@ def decoupercharact(chaine, longueur, longueurOptimale, separateurs = None) : # on vérifie si le caractére courant est une marque de ponctuation for s in separateurs: if caractere == s[0]: - # si c'est une ponctuation - + # si c'est une ponctuation + if s[1] / distance > float(meilleur[1]) / meilleureDistance: # print nbCar, s[0] meilleur[0] = s[0] @@ -363,13 +558,13 @@ def decoupercharact(chaine, longueur, longueurOptimale, separateurs = None) : meilleur[2] = nbCar trouve = True iDecoupe = nbCar - + # et on termine la recherche break # on passe au caractère précédant nbCar = nbCar - 1 - + # si on a trouvé if trouve: fin = chaine[iDecoupe + 1:] @@ -378,35 +573,57 @@ def decoupercharact(chaine, longueur, longueurOptimale, separateurs = None) : # si on a rien trouvé return False, chaine.split(), '' -def BugReport(parent): + +exceptions = {'paragrapheOT' : u"Un problème de formatage (présence d'un marqueur de paragraphe (-*) en dehors d'un texte) est survenu à la ligne ", + 'EmptyText' : u"Texte vide (probablement un problème de formatage du corpus). Le problème est apparu à la ligne ", + 'CorpusEncoding' : u"Problème d'encodage.", + 'TextBeforeTextMark' : u"Problème de formatage : du texte avant le premier marqueur de texte (****). Le problème est survenu à la ligne ", + 'MissingAnalyse' : u'Aucun fichier à cet emplacement :\n', +} + +def BugReport(parent, error = None): for ch in parent.GetChildren(): if "" == str(type(ch)): - ch.Destroy() - dial = BugDialog(parent) - txt = u' !== BUG ==! \n' - txt += u'*************************************\n' - for line in formatExceptionInfo(): - if type(line) == type([]): - for don in line: - txt += don.replace(' ', ' ') - else: - txt += line + '\n' + ch.Destroy() + excName, exc, excTb = formatExceptionInfo() + if excName == 'Exception' : + print exc + if len(exc.split()) == 2 : + mss, linenb = exc.split() + if mss in exceptions : + txt = exceptions[mss] + linenb + else : + txt = exc + else : + if exc in exceptions : + txt = exceptions[exc] + else : + txt = exc + title = "Information" + else : + txt = u' !== BUG ==! \n' + txt += u'*************************************\n' + txt += '\n'.join(excTb).replace(' ', ' ') + txt += excName + '\n' + txt += `exc` + title = "Bug" + + dial = BugDialog(parent, **{'title' : title}) if 'Rerror' in dir(parent) : txt += parent.Rerror parent.Rerror = '' - print formatExceptionInfo() - log.error(txt) + log.info(txt) dial.text_ctrl_1.write(txt) dial.CenterOnParent() dial.ShowModal() - raise NameError('Bug') - + dial.Destroy() + def PlaySound(parent): if parent.pref.getboolean('iramuteq', 'sound') : try: if "gtk2" in wx.PlatformInfo: error = Popen(['aplay','-q',os.path.join(parent.AppliPath,'son_fin.wav')]) - else : + else : sound = wx.Sound(os.path.join(parent.AppliPath, 'son_fin.wav')) sound.Play(wx.SOUND_SYNC) except : @@ -415,33 +632,41 @@ def PlaySound(parent): def ReadDicoAsDico(dicopath): with codecs.open(dicopath, 'r', 'UTF8') as f: content = f.readlines() - dico = {} - for line in content : - if line[0] != u'': - line = line.replace(u'\n', '').replace('"', '').split('\t') - dico[line[0]] = line[1:] - return dico + lines = [line.rstrip('\n\r').replace(u'\n', '').replace('"', '').split('\t') for line in content if line != u''] + return dict([[line[0], line[1:]] for line in lines]) -def ReadLexique(parent, lang = 'french'): - parent.lexique = ReadDicoAsDico(parent.DictPath.get(lang, 'french')) +def ReadLexique(parent, lang = 'french', filein = None): + if lang != 'other' : + if filein is None : + parent.lexique = ReadDicoAsDico(parent.DictPath.get(lang, 'french')) + else : + parent.lexique = ReadDicoAsDico(filein) + else : + if filein is None : + parent.lexique = {} + else : + parent.lexique = ReadDicoAsDico(filein) -def ReadList(filein, encoding = sys.getdefaultencoding()): +def ReadList(filein, encoding = sys.getdefaultencoding(), sep = ';'): #file = open(filein) - file = codecs.open(filein, 'r', encoding) - content = file.readlines() - file.close() + with codecs.open(filein, 'r', encoding) as f : + content = f.read() + content = [line.replace('\n', '').replace('\r','').replace('\"', '').replace(',', '.').split(sep) for line in content.splitlines()] + #file = codecs.open(filein, 'r', encoding) + #content = file.readlines() + #file.close() first = content.pop(0) - first = first.replace('\n', '').replace('\r','').replace('\"', '').split(';') + #first = first.replace('\n', '').replace('\r','').replace('\"', '').split(sep) dict = {} i = 0 for line in content: - line = line.replace('\n', '').replace('\r','').replace('\"', '').replace(',', '.') - line = line.split(';') + #line = line.replace('\n', '').replace('\r','').replace('\"', '').replace(',', '.') + #line = line.split(';') nline = [line[0]] for val in line[1:]: if val == u'NA' : don = '' - else: + else: try: don = int(val) except: @@ -451,13 +676,18 @@ def ReadList(filein, encoding = sys.getdefaultencoding()): i += 1 return dict, first +def exec_RCMD(rpath, command) : + log.info('R CMD INSTALL %s' % command) + rpath = rpath.replace('\\','\\\\') + error = call(["%s" % rpath, 'CMD', 'INSTALL', "%s" % command]) + return error + def exec_rcode(rpath, rcode, wait = True, graph = False): - print rpath, rcode + log.info("R Script : %s" % rcode) needX11 = False if sys.platform == 'darwin' : try : macversion = platform.mac_ver()[0].split('.') - print macversion if int(macversion[1]) < 5 : needX11 = True else : @@ -466,18 +696,21 @@ def exec_rcode(rpath, rcode, wait = True, graph = False): needX11 = False rpath = rpath.replace('\\','\\\\') + env = os.environ.copy() + if sys.platform == 'darwin' and 'LC_ALL' not in env: + env['LC_ALL'] = 'en_US.UTF-8' if not graph : if wait : if sys.platform == 'win32': error = call(["%s" % rpath, "--vanilla","--slave","-f", "%s" % rcode]) else : - error = call([rpath, '--vanilla','--slave',"-f %s" % rcode]) + error = call([rpath, '--slave', "--vanilla", "-f %s" % rcode, "--encoding=UTF-8"], env = env) return error else : if sys.platform == 'win32': pid = Popen(["%s" % rpath, '--vanilla','--slave','-f', "%s" % rcode]) else : - pid = Popen([rpath, '--vanilla','--slave',"-f %s" % rcode], stderr = PIPE) + pid = Popen([rpath, '--slave', "--vanilla", "-f %s" % rcode, "--encoding=UTF-8"], stderr = PIPE, env = env) return pid else : if wait : @@ -485,18 +718,18 @@ def exec_rcode(rpath, rcode, wait = True, graph = False): error = call(["%s" % rpath, '--vanilla','--slave','-f', "%s" % rcode]) elif sys.platform == 'darwin' and needX11: os.environ['DISPLAY'] = ':0.0' - error = call([rpath, '--vanilla','--slave',"-f %s" % rcode]) + error = call([rpath, '--vanilla','--slave',"-f %s" % rcode, "--encoding=UTF-8"], env = env) else : - error = call([rpath, '--vanilla','--slave',"-f %s" % rcode]) + error = call([rpath, '--vanilla','--slave',"-f %s" % rcode, "--encoding=UTF-8"], env = env) return error else : if sys.platform == 'win32': pid = Popen(["%s" % rpath, '--vanilla','--slave','-f', "%s" % rcode]) elif sys.platform == 'darwin' and needX11: os.environ['DISPLAY'] = ':0.0' - pid = Popen([rpath, '--vanilla','--slave',"-f %s" % rcode], stderr = PIPE) + pid = Popen([rpath, '--vanilla','--slave',"-f %s" % rcode, "--encoding=UTF-8"], stderr = PIPE, env = env) else : - pid = Popen([rpath, '--vanilla','--slave',"-f %s" % rcode], stderr = PIPE) + pid = Popen([rpath, '--vanilla','--slave',"-f %s" % rcode, "--encoding=UTF-8"], stderr = PIPE, env = env) return pid def check_Rresult(parent, pid) : @@ -508,105 +741,306 @@ def check_Rresult(parent, pid) : error[1] = 'None' parent.Rerror = '\n'.join([str(pid.returncode), '\n'.join(error)]) try : - raise Exception('\n'.join(u'Erreur R', '\n'.join(error[1:]))) + raise Exception('\n'.join([u'Erreur R', '\n'.join(error[1:])])) except : BugReport(parent) + return False + else : + return True else : - if pid !=0 : + if pid != 0 : try : raise Exception(u'Erreur R') except : BugReport(parent) + return False + else : + return True + + +def launchcommand(mycommand): + Popen(mycommand) def print_liste(filename,liste): with open(filename,'w') as f : for graph in liste : - f.write(';'.join(graph)+'\n') + f.write(';'.join(graph).encode(sys.getdefaultencoding(), errors='replace')+'\n') def read_list_file(filename, encoding = sys.getdefaultencoding()): with codecs.open(filename,'rU', encoding) as f : content=f.readlines() ncontent=[line.replace('\n','').split(';') for line in content if line.strip() != ''] return ncontent - -class MessageImage(wx.Frame): - def __init__(self, *args, **kwds): - # begin wxGlade: MyFrame.__init__ - kwds["style"] = wx.DEFAULT_FRAME_STYLE - wx.Frame.__init__(self, *args, **kwds) - #self.text_ctrl_1 = wx.TextCtrl(self, -1, "", style=wx.TE_MULTILINE) - self.imageFile = False - self.imagename = u"chi_classe.png" - self.HtmlPage = wx.html.HtmlWindow(self, -1) - if "gtk2" in wx.PlatformInfo: - self.HtmlPage.SetStandardFonts() - self.HtmlPage.SetFonts('Courier', 'Courier') - - self.button_1 = wx.Button(self, -1, u"Fermer") - self.Bind(wx.EVT_BUTTON, self.OnCloseMe, self.button_1) - self.Bind(wx.EVT_CLOSE, self.OnCloseWindow) - self.__do_layout() - # end wxGlade - - def __do_layout(self): - # begin wxGlade: MyFrame.__do_layout - sizer_1 = wx.BoxSizer(wx.VERTICAL) - self.sizer_2 = wx.BoxSizer(wx.VERTICAL) - self.sizer_2.Add(self.HtmlPage, 1, wx.EXPAND | wx.ADJUST_MINSIZE, 0) - self.sizer_2.Add(self.button_1, 0, wx.ALIGN_CENTER_HORIZONTAL | wx.ADJUST_MINSIZE, 0) - sizer_1.Add(self.sizer_2, 1, wx.EXPAND, 0) - self.SetAutoLayout(True) - self.SetSizer(sizer_1) - # end wxGlade - - def addsaveimage(self, imageFile) : - self.imageFile = imageFile - self.button_2 = wx.Button(self, -1, u"Enregistrer l'image...") - self.Bind(wx.EVT_BUTTON, self.OnSaveImage, self.button_2) - self.sizer_2.Add(self.button_2, 0, wx.ALIGN_CENTER_HORIZONTAL | wx.ADJUST_MINSIZE, 0) - self.Layout() - - def OnCloseMe(self, event): - self.Close(True) - - def OnCloseWindow(self, event): - self.Destroy() - - def OnSaveImage(self, event) : - dlg = wx.FileDialog( - self, message="Enregistrer sous...", defaultDir=os.getcwd(), - defaultFile= self.imagename, wildcard="png|*.png", style=wx.SAVE | wx.OVERWRITE_PROMPT - ) - dlg.SetFilterIndex(2) - dlg.CenterOnParent() - if dlg.ShowModal() == wx.ID_OK: - path = dlg.GetPath() - copyfile(self.imageFile, path) - def progressbar(self, maxi) : - if 'parent' in dir(self) : - parent = self.parent - else : - parent = self - return wx.ProgressDialog("Traitements", + ira = wx.GetApp().GetTopWindow() + parent = ira + try : + maxi = int(maxi) + except : + maxi = 1 + prog = wx.ProgressDialog("Traitements", "Veuillez patienter...", maximum=maxi, parent=parent, style=wx.PD_APP_MODAL | wx.PD_AUTO_HIDE | wx.PD_ELAPSED_TIME | wx.PD_CAN_ABORT ) - + prog.SetSize((400,150)) + #prog.SetIcon(ira._icon) + return prog def treat_var_mod(variables) : var_mod = {} - for variable in variables : - if u'_' in variable : - forme = variable.split(u'_') - var = forme[0] - mod = forme[1] - if not var in var_mod : - var_mod[var] = [variable] - else : - if not mod in var_mod[var] : - var_mod[var].append(variable) + variables = list(set(variables)) + varmod = [variable.split('_') for variable in variables] + vars = list(set([var[0] for var in varmod if len(var) >=2])) + for var in vars : + mods = ['_'.join(v) for v in varmod if v[0] == var] + var_mod[var] = mods + +# for variable in variables : +# if u'_' in variable : +# forme = variable.split(u'_') +# var = forme[0] +# mod = forme[1] +# if not var in var_mod : +# var_mod[var] = [variable] +# else : +# if not mod in var_mod[var] : +# var_mod[var].append(variable) return var_mod + +def doconcorde(corpus, uces, mots, uci = False, et = False) : + if not uci : + ucestxt1 = [row for row in corpus.getconcorde(uces)] + else : + ucestxt1 = [row for row in corpus.getuciconcorde(uces)] + ucestxt1 = dict(ucestxt1) + ucestxt = [] + ucis_txt = [] + if not et : + listmot = [corpus.getlems()[lem].formes for lem in mots] + listmot = [corpus.getforme(fid).forme for lem in listmot for fid in lem] + else : + listmot = mots + mothtml = ['%s' % mot for mot in listmot] + dmots = dict(zip(listmot, mothtml)) + for uce in uces : + ucetxt = ucestxt1[uce].split() + ucetxt = ' '.join([dmots.get(mot, mot) for mot in ucetxt]) + if not uci : + uciid = corpus.getucefromid(uce).uci + ucis_txt.append('

' + ' '.join(corpus.ucis[corpus.getucefromid(uce).uci].etoiles) + ' *%i_%i

' % (uciid, uce, uciid, uce)) + else : + ucis_txt.append('

' + ' '.join(corpus.ucis[uce].etoiles) + '

') + ucestxt.append(ucetxt) + return ucis_txt, ucestxt + + +def getallstcarac(corpus, analyse) : + pathout = PathOut(analyse['ira']) + profils = ReadProfileAsDico(pathout['PROFILE_OUT'], Alceste, self.encoding) + print profils + +def read_chd(filein, fileout): + with open(filein, 'r') as f : + content = f.read() + #content = [line[3:].replace('"',"").replace(' ','') for line in content.splitlines()] + content = [line.split('\t') for line in content.splitlines()] + chd = {'name':1, 'children':[]} + mere={} + for i, line in enumerate(content) : + if i == 0 : + chd['children'] = [{'name': line[1],'size' : content[i+1][0]}, {'name':line[2], 'size': content[i+1][1]}] + mere[line[1]] = chd['children'][0] + mere[line[2]] = chd['children'][1] + elif not i % 2 : + if 'children' in mere[line[0]]: + mere[line[0]]['children'].append({'name': line[1],'size' : content[i+1][0]}) + mere[line[1]] = mere[line[0]]['children'][-1] + mere[line[0]]['children'].append({'name': line[2],'size' : content[i+1][1]}) + mere[line[2]] = mere[line[0]]['children'][-1] + else : + mere[line[0]]['children'] = [{'name': line[1],'size' : content[i+1][0]}, {'name':line[2], 'size': content[i+1][1]}] + mere[line[1]] = mere[line[0]]['children'][-2] + mere[line[2]] = mere[line[0]]['children'][-1] + with open(fileout, 'w') as f : + f.write(json.dumps(chd)) + + +translation_languages = {"Afrikaans":"af", "Albanian":"sq", "Amharic":"am", "Arabic":"ar", "Armenian":"hy", "Azeerbaijani":"az", "Basque":"eu", "Belarusian":"be", "Bengali":"bn", "Bosnian":"bs", "Bulgarian":"bg", "Catalan":"ca", "Cebuano":"ceb", "Chichewa":"ny", "Chinese (Simplified)":"zh-CN", "Chinese (Traditional)":"zh-TW", "Corsican":"co", "Croatian":"hr", "Czech":"cs", "Danish":"da", "Dutch":"nl", "English":"en", "Esperanto":"eo", "Estonian":"et", "Filipino":"tl", "Finnish":"fi", "French":"fr", "Frisian":"fy", "Galician":"gl", "Georgian":"ka", "German":"de", "Greek":"el", "Gujarati":"gu", "Haitian Creole":"ht", "Hausa":"ha", "Hawaiian":"haw", "Hebrew":"iw", "Hindi":"hi", "Hmong":"hmn ", "Hungarian":"hu", "Icelandic":"is", "Igbo":"ig", "Indonesian":"id", "Irish":"ga", "Italian":"it", "Japanese":"ja", "Javanese":"jw", "Kannada":"kn", "Kazakh":"kk", "Khmer":"km", "Korean":"ko", "Kurdish":"ku", "Kyrgyz":"ky", "Lao":"lo", "Latin":"la", "Latvian":"lv", "Lithuanian":"lt", "Luxembourgish":"lb", "Macedonian":"mk", "Malagasy":"mg", "Malay":"ms", "Malayalam":"ml", "Maltese":"mt", "Maori":"mi", "Marathi":"mr", "Mongolian":"mn", "Burmese":"my", "Nepali":"ne", "Norwegian":"no", "Pashto":"ps", "Persian":"fa", "Polish":"pl", "Portuguese":"pt", "Punjabi":"ma", "Romanian":"ro", "Russian":"ru", "Samoan":"sm", "Scots Gaelic":"gd", "Serbian":"sr", "Sesotho":"st", "Shona":"sn", "Sindhi":"sd", "Sinhala":"si", "Slovak":"sk", "Slovenian":"sl", "Somali":"so", "Spanish":"es", "Sundanese":"su", "Swahili":"sw", "Swedish":"sv", "Tajik":"tg", "Tamil":"ta", "Telugu":"te", "Thai":"th", "Turkish":"tr", "Ukrainian":"uk", "Urdu":"ur", "Uzbek":"uz", "Vietnamese":"vi", "Welsh":"cy", "Xhosa":"xh", "Yiddish":"yi", "Yoruba":"yo", "Zulu":"zu", } + + +def gettranslation(words, lf, lt) : + import urllib2 + import json + agent = {'User-Agent': + "Mozilla/4.0 (\ + compatible;\ + MSIE 6.0;\ + Windows NT 5.1;\ + SV1;\ + .NET CLR 1.1.4322;\ + .NET CLR 2.0.50727;\ + .NET CLR 3.0.04506.30\ + )"} + base_link = "https://translate.googleapis.com/translate_a/single?client=gtx&sl=%s&tl=%s&dt=t&q=%s" + totrans = urllib2.quote('\n'.join(words).encode('utf8')) + link = base_link % (lf, lt, totrans) + request = urllib2.Request(link, headers=agent) + raw_data = urllib2.urlopen(request).read() + data = json.loads(raw_data) + return [line[0].decode('utf8', errors='replace').replace(u"'", u'_').replace(u' | ', u'|').replace(u' ', u'_').replace(u'-',u'_').replace(u'\n','') for line in data[0]] + +def makenprof(prof, trans, deb=0) : + nprof=[] + if deb == 0 : + nprof.append(prof[0]) + for i, val in enumerate(trans) : + line = prof[deb+i+1][:] + line[6] = val + nprof.append(line) + return nprof + +def treatempty(val) : + if val.strip() == '' : + return '_' + else : + return val + +def translateprofile(corpus, dictprofile, lf='it', lt='fr', maxword = 20) : + nprof = {} + lems = {} + for i in range(len(dictprofile)) : + prof = dictprofile[`i+1`] + try : + lenact = prof.index([u'*****', u'*', u'*', u'*', u'*', u'*', '', '']) + lensup = -1 + except ValueError: + try : + lenact = prof.index([u'*', u'*', u'*', u'*', u'*', u'*', '', '']) + lensup = 0 + except ValueError: + lenact = len(prof) + lensup = 0 + try : + lensup += prof.index([u'*', u'*', u'*', u'*', u'*', u'*', '', '']) + lensup = lensup - lenact + except ValueError: + lensup += len(prof) - lenact + if lenact != 0 : + if lenact > maxword : + nlenact = maxword + else : + nlenact = lenact + actori = [line[6] for line in prof[1:nlenact]] + act = [val.replace(u'_', u' ') for val in actori] + act = gettranslation(act, lf, lt) + for j, val in enumerate(actori) : + if act[j] not in lems : + lems[act[j]] = val + else : + while act[j] in lems : + act[j] = act[j] + u"+" + lems[act[j]] = val + nprof[`i+1`] = makenprof(prof, act) + + if lensup != 0 : + if lensup > maxword : + nlensup = maxword + else : + nlensup = lensup + supori = [line[6] for line in prof[(1+lenact):(lenact+nlensup+1)]] + sup = [val.replace(u'_', u' ') for val in supori] + sup = [treatempty(val) for val in sup] + sup = gettranslation(sup, lf, lt) + for j, val in enumerate(supori) : + if sup[j] not in lems : + lems[sup[j]] = val + else : + while sup[j] in lems : + sup[j] = sup[j] + u"+" + lems[sup[j]] = val + nprof[`i+1`].append([u'*****', u'*', u'*', u'*', u'*', u'*', '', '']) + nprof[`i+1`] += makenprof(prof, sup, deb=lenact) + + try : + lenet = prof.index([u'*', u'*', u'*', u'*', u'*', u'*', '', '']) + nprof[`i+1`].append([u'*', u'*', u'*', u'*', u'*', u'*', '', '']) + nprof[`i+1`] += prof[(lenet+1):] + except : + pass + return nprof, lems + +def write_translation_profile(prof, lems, language, dictpathout) : + if os.path.exists(dictpathout['translations.txt']) : + with codecs.open(dictpathout['translations.txt'], 'r', 'utf8') as f : + translist = f.read() + translist = [line.split('\t') for line in translist.splitlines()] + else : + translist = [] + toprint = [] + toprint.append(['','','','','','']) + toprint.append([u'***', u'nb classes', `len(prof)`, u'***', '', '']) + for i in range(len(prof)) : + toprint.append([u'**', u'classe', `i+1`, u'**', '', '']) + toprint.append([u'****'] + prof[`i+1`][0] + [u'****']) + rest = [[`line[1]`, `line[2]`, `line[3]`, `line[4]`, line[6], line[7].replace('< 0,0001', '0.00009').replace('NS (','').replace(')','')] for line in prof[`i+1`][1:]] + for i, line in enumerate(prof[`i+1`][1:]) : + if line[0] == u'*' : + rest[i] = [u'*', u'*', u'*', u'*', u'*', u'*'] + elif line[0] == u'*****' : + rest[i] = [u'*****',u'*',u'*', u'*', u'*', u'*'] + toprint += rest + with open(dictpathout['translation_profile_%s.csv' % language], 'w') as f : + f.write('\n'.join([';'.join(line) for line in toprint]).encode('utf8')) + with open(dictpathout['translation_words_%s.csv' % language], 'w') as f : + f.write('\n'.join(['\t'.join([val, lems[val]]) for val in lems]).encode('utf8')) + if 'translation_profile_%s.csv' % language not in [val[0] for val in translist] : + translist.append(['translation_profile_%s.csv' % language, 'translation_words_%s.csv' % language]) + with open(dictpathout['translations.txt'], 'w') as f : + f.write('\n'.join(['\t'.join(line) for line in translist]).encode('utf8')) + +def makesentidict(infile, language) : + #'/home/pierre/workspace/iramuteq/dev/langues/NRC/NRC-Emotion-Lexicon.csv' + with codecs.open(infile,'r', 'utf8') as f : + content = f.read() + content = [line.split('\t') for line in content.splitlines()] + titles = content.pop(0) + senti = ['Positive', 'Negative', 'Anger', 'Anticipation', 'Disgust', 'Fear', 'Joy', 'Sadness', 'Surprise', 'Trust'] + sentid = {} + for sent in senti : + sentid[sent] = titles.index(sent) + frtitle = [val for val in titles if '(fr)' in val] + frid = titles.index(frtitle[0]) + sentidict = [[line[frid].lower(), [line[sentid[sent]] for sent in senti]] for line in content] + pos = ['positive'] + [line[0] for line in sentidict if line[1][0] == '1'] + neg = ['negative'] + [line[0] for line in sentidict if line[1][1] == '1'] + anger = ['anger'] + [line[0] for line in sentidict if line[1][2] == '1'] + anticipation = ['anticipation'] + [line[0] for line in sentidict if line[1][3] == '1'] + disgust = ['disgust'] + [line[0] for line in sentidict if line[1][4] == '1'] + fear = ['fear'] + [line[0] for line in sentidict if line[1][5] == '1'] + joy = ['joy'] + [line[0] for line in sentidict if line[1][6] == '1'] + sadness = ['sadness'] + [line[0] for line in sentidict if line[1][7] == '1'] + surprise = ['surprise'] + [line[0] for line in sentidict if line[1][8] == '1'] + trust = ['trust'] + [line[0] for line in sentidict if line[1][9] == '1'] + with open('/tmp/tgenemo.csv', 'w') as f : + for val in [pos, neg, anger, anticipation, disgust, fear, joy, sadness, surprise, trust] : + f.write('\t'.join(val).encode('utf8') + '\n') + +def countsentfromprof(prof, encoding, sentidict) : + with codecs.open(prof, 'r', encoding) as f : + content = f.read() + content = [line.split(';') for line in content.splitlines()] + print content + content = [[line[0], [int(val) for val in line[1:]]] for line in content] + print content + content = dict(content) + print content + +def iratolexico(infile, outfile, encoding) : + with codecs.open(infile, 'r', encoding) as f : + for line in f : + if line.startswith(u'**** ') : + line = line.split() +