X-Git-Url: http://iramuteq.org/git?p=iramuteq;a=blobdiff_plain;f=functions.py;h=62f669ce24d0ee52a58780313a183b7d0c1aff53;hp=8a10f674e926bceaea110dd272334b322335437c;hb=aae91ab48172a83c49bb502ac737bcc1b3a6685c;hpb=ba7e45a167c8cd7a796efe332a4bf61dc864b373 diff --git a/functions.py b/functions.py index 8a10f67..62f669c 100644 --- a/functions.py +++ b/functions.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- #Author: Pierre Ratinaud #Copyright (c) 2008-2012 Pierre Ratinaud -#Lisense: GNU/GPL +#License: GNU/GPL import wx import re @@ -30,12 +30,72 @@ log = logging.getLogger('iramuteq') indices_simi = [u'cooccurrence' ,'pourcentage de cooccurrence',u'Russel',u'Jaccard', 'Kulczynski1', 'Kulczynski2', 'Mountford', 'Fager', 'simple matching', 'Hamman', 'Faith', 'Tanimoto', 'Dice', 'Phi', 'Stiles', 'Michael', 'Mozley', 'Yule', 'Yule2', 'Ochiai', 'Simpson', 'Braun-Blanquet','Chi-squared', 'Phi-squared', 'Tschuprow', 'Cramer', 'Pearson', 'binomial'] + +def open_folder(folder): + if sys.platform == "win32": + os.startfile(folder) + else: + opener ="open" if sys.platform == "darwin" else "xdg-open" + call([opener, folder]) + +def normpath_win32(path) : + if not sys.platform == 'win32' : + return path + while '\\\\' in path : + path = path.replace('\\\\', '\\') + if path.startswith('\\') and not path.startswith('\\\\') : + path = '\\' + path + return path + +class TGen : + def __init__(self, path = None, encoding = 'utf8'): + self.path = path + self.tgen = {} + self.encoding = encoding + + def __getitem__(self, key): + return self.tgen[key] + + def read(self, path = None): + if path is None : + path = self.path + with codecs.open(path, 'r', self.encoding) as f : + tgen = f.read() + tgen = [line.split('\t') for line in tgen.splitlines()] + tgen = dict([[line[0], line[1:]] for line in tgen]) + self.tgen = tgen + self.path = path + + def write(self, path = None): + if path is None : + path = self.path + with open(path, 'w') as f : + f.write('\n'.join(['\t'.join([val] + self.tgen[val]) for val in self.tgen]).encode(self.encoding)) + + def writetable(self, pathout, tgens, totocc): + etoiles = totocc.keys() + etoiles.sort() + with open(pathout, 'w') as f : + line = '\t'.join([u'tgens'] + etoiles) + '\n' + f.write(line.encode(self.encoding)) + for t in tgens : + line = '\t'.join([t] + [`tgens[t][et]` for et in etoiles]) + '\n' + f.write(line.encode(self.encoding)) + i = 0 + totname = 'total' + while totname + `i` in tgens : + i += 1 + totname = totname + `i` + line = '\t'.join([totname] + [`totocc[et]` for et in etoiles]) + '\n' + f.write(line.encode(self.encoding)) + class History : def __init__(self, filein, syscoding = 'utf8') : self.filein = filein self.syscoding = syscoding - self.corpora = {} + self.corpus = {} self.openedcorpus = {} + self.openedmatrix = {} self.orph = [] self.analyses = {} self.history = [] @@ -45,19 +105,26 @@ class History : def read(self) : d = shelve.open(self.filein) self.history = d.get('history', []) + self.matrix = d.get('matrix', []) self.ordercorpus = dict([[corpus['uuid'], i] for i, corpus in enumerate(self.history)]) - self.corpus = dict([[corpus['uuid'], corpus] for i, corpus in enumerate(self.history)]) + self.corpus = dict([[corpus['uuid'], corpus] for corpus in self.history]) self.analyses = dict([[analyse['uuid'], analyse] for corpus in self.history for analyse in corpus.get('analyses', [])]) + self.matrixanalyse = dict([[mat['uuid'], mat] for mat in self.matrix]) + self.ordermatrix = dict([[matrix['uuid'], i] for i, matrix in enumerate(self.matrix)]) d.close() def write(self) : d = shelve.open(self.filein) d['history'] = self.history + d['matrix'] = self.matrix d.close() def add(self, analyse) : log.info('add to history %s' % analyse.get('corpus_name', 'pas un corpus')) tosave = {'uuid' : analyse['uuid'], 'ira': analyse['ira'], 'type' : analyse['type']} + if tosave['uuid'] in self.corpus : + log.info('problem : this uuid is already in history : %s' % tosave['uuid']) + return if analyse.get('corpus', False) : if analyse['uuid'] in self.analyses : return @@ -73,14 +140,33 @@ class History : self.orph.append(tosave) else : tosave['corpus_name'] = analyse['corpus_name'] + #self.ordercorpus[tosave['uuid']] = len(history) + #self.corpus[tosave['uuid']] = analyse self.history.append(tosave) self.write() self.read() + def addMatrix(self, analyse) : + tosave = analyse + #tosave['matrix_name'] = analyse['matrix_name'] + tosave['analyses'] = [] + self.matrix.append(tosave) + self.write() + self.read() + + def addMatrixAnalyse(self, analyse) : + tosave = {'uuid' : analyse['uuid'], 'ira': analyse['ira'], 'type' : analyse['type'], 'matrix' : analyse['matrix']} + tosave['name'] = analyse['name'] + if tosave['matrix'] in self.ordermatrix : + self.matrix[self.ordermatrix[tosave['matrix']]]['analyses'].append(tosave) + self.write() + self.read() + def addmultiple(self, analyses) : + log.info('add multiple') for analyse in analyses : tosave = {'uuid' : analyse['uuid'], 'ira': analyse['ira'], 'type' : analyse['type']} - corpus = analyse['uuid'] + corpus = analyse['corpus'] tosave['corpus'] = corpus tosave['name'] = analyse['name'] if corpus in self.corpus : @@ -92,13 +178,17 @@ class History : self.read() def delete(self, analyse, corpus = False) : + log.info('delete %s' % analyse.get('name', 'noname')) if corpus : self.history.pop(self.ordercorpus[analyse['uuid']]) if analyse['uuid'] in self.openedcorpus : del self.openedcorpus[analyse['uuid']] - else : + log.info('delete corpus : %s' % analyse['uuid']) + elif analyse['uuid'] in self.analyses : todel = [i for i, ana in enumerate(self.corpus[analyse['corpus']]['analyses']) if ana['uuid'] == analyse['uuid']][0] self.history[self.ordercorpus[analyse['corpus']]]['analyses'].pop(todel) + elif analyse['uuid'] in self.matrixanalyse : + self.matrix = [mat for mat in self.matrix if mat['uuid'] != analyse['uuid']] self.write() self.read() @@ -108,6 +198,30 @@ class History : def rmtab(self, analyse) : del self.opened[analyse['uuid']] + def update(self, analyse) : + if 'matrix_name' in analyse : + self.matrixanalyse[analyse['uuid']].update(analyse) + elif 'corpus_name' in analyse : + self.corpus[analyse['uuid']].update(analyse) + elif 'corpus' in analyse : + self.analyses[analyse['uuid']].update(analyse) + else : + toupdate = [an for an in self.matrixanalyse[analyse['matrix']]['analyses'] if an['uuid'] == analyse['uuid']] + toupdate[0].update(analyse) + self.write() + self.read() + + def clean(self) : + corpustodel = [corpus for corpus in self.history if not os.path.exists(corpus['ira'])] + print corpustodel + for corpus in corpustodel : + print 'cleaning :', corpus['corpus_name'] + self.delete(corpus, corpus = True) + anatodel = [analyse for corpus in self.history for analyse in corpus.get('analyses', []) if not os.path.exists(analyse.get('ira', '/'))] + for analyse in anatodel : + print 'cleaning :', analyse['name'] + self.delete(analyse) + def __str__(self) : return str(self.history) @@ -115,7 +229,9 @@ class DoConf : def __init__(self, configfile=None, diff = None, parametres = None) : self.configfile = configfile self.conf = ConfigParser() + if configfile is not None : + configfile = normpath_win32(configfile) self.conf.readfp(codecs.open(configfile, 'r', 'utf8')) self.parametres = {} if parametres is not None : @@ -171,8 +287,9 @@ class DoConf : txt += '%s = %s\n' % (option, `parametres[i][option]`) if outfile is None : outfile = self.configfile - with codecs.open(outfile, 'w', 'utf8') as f : - f.write(txt) + outfile = normpath_win32(outfile) + with open(outfile, 'w') as f : + f.write(txt.encode('utf8')) #self.conf.write(f) def totext(self, parametres) : @@ -197,10 +314,12 @@ def write_tab(tab, fileout) : class BugDialog(wx.Dialog): def __init__(self, *args, **kwds): # begin wxGlade: MyDialog.__init__ - kwds["style"] = wx.DEFAULT_DIALOG_STYLE + kwds["style"] = wx.DEFAULT_DIALOG_STYLE | wx.STAY_ON_TOP kwds["size"] = wx.Size(500, 200) wx.Dialog.__init__(self, *args, **kwds) + self.SetTitle(kwds['title']) self.text_ctrl_1 = wx.TextCtrl(self, -1, "", style=wx.TE_MULTILINE) + self.text_ctrl_1.SetBackgroundColour('#DDE8EB') self.button_1 = wx.Button(self, wx.ID_OK, "") self.__set_properties() @@ -209,7 +328,6 @@ class BugDialog(wx.Dialog): def __set_properties(self): # begin wxGlade: MyDialog.__set_properties - self.SetTitle("Bug") self.SetMinSize(wx.Size(500, 200)) self.text_ctrl_1.SetMinSize(wx.Size(500, 200)) @@ -281,21 +399,15 @@ def treat_line_alceste(i, line) : return [i, int(line[0]), int(line[1]), float(line[2]), float(line[3]), line[6], line[4], line[5]] def ReadProfileAsDico(File, Alceste=False, encoding = sys.getdefaultencoding()): - #print 'lecture des profils : ReadProfileAsDico' - #if Alceste : - # print 'lecture du dictionnaire de type' - # dictlem = {} - # for line in parent.corpus.lem_type_list : - # dictlem[line[0]] = line[1] dictlem = {} print 'lecture des profiles' - #encoding = sys.getdefaultencoding() FileReader = codecs.open(File, 'r', encoding) Filecontent = FileReader.readlines() FileReader.close() DictProfile = {} count = 0 - rows = [row.replace('\n', '').replace("'", '').replace('\"', '').replace(',', '.').replace('\r','').split(';') for row in Filecontent] + #rows = [row.replace('\n', '').replace("'", '').replace('\"', '').replace(',', '.').replace('\r','').split(';') for row in Filecontent] + rows = [row.replace('\n', '').replace("'", '').replace('\"', '').replace('\r','').split(';') for row in Filecontent] rows.pop(0) ClusterNb = rows[0][2] rows.pop(0) @@ -322,14 +434,17 @@ def GetTxtProfile(dictprofile, cluster_size) : return '\n\n'.join(proflist) def formatExceptionInfo(maxTBlevel=5): - cla, exc, trbk = sys.exc_info() - excName = cla.__name__ - try: - excArgs = exc.args[0] - except : - excArgs = "" - excTb = traceback.format_tb(trbk, maxTBlevel) - return (excName, excArgs, excTb) + cla, exc, trbk = sys.exc_info() + try : + excName = cla.__name__ + except : + excName = 'None' + try: + excArgs = exc.args[0] + except : + excArgs = "" + excTb = traceback.format_tb(trbk, maxTBlevel) + return (excName, excArgs, excTb) #fonction des etudiants de l'iut @@ -389,21 +504,42 @@ def decoupercharact(chaine, longueur, longueurOptimale, separateurs = None) : # si on a rien trouvé return False, chaine.split(), '' + +exceptions = {'paragrapheOT' : u"Un problème de formatage (présence d'un marqueur de paragraphe (-*) en dehors d'un texte) est survenu à la ligne ", + 'EmptyText' : u"Texte vide (probablement un problème de formatage du corpus). Le problème est apparu à la ligne ", + 'CorpusEncoding' : u"Problème d'encodage.", + 'TextBeforeTextMark' : u"Problème de formatage : du texte avant le premier marqueur de texte (****). Le problème est survenu à la ligne ", + 'MissingAnalyse' : u'Aucun fichier à cet emplacement :\n', +} + def BugReport(parent, error = None): for ch in parent.GetChildren(): if "" == str(type(ch)): ch.Destroy() excName, exc, excTb = formatExceptionInfo() if excName == 'Exception' : - txt = 'Message :\n' + print exc + if len(exc.split()) == 2 : + mss, linenb = exc.split() + if mss in exceptions : + txt = exceptions[mss] + linenb + else : + txt = exc + else : + if exc in exceptions : + txt = exceptions[exc] + else : + txt = exc + title = "Information" else : txt = u' !== BUG ==! \n' txt += u'*************************************\n' txt += '\n'.join(excTb).replace(' ', ' ') txt += excName + '\n' - txt += exc + txt += `exc` + title = "Bug" - dial = BugDialog(parent) + dial = BugDialog(parent, **{'title' : title}) if 'Rerror' in dir(parent) : txt += parent.Rerror parent.Rerror = '' @@ -427,12 +563,8 @@ def PlaySound(parent): def ReadDicoAsDico(dicopath): with codecs.open(dicopath, 'r', 'UTF8') as f: content = f.readlines() - dico = {} - for line in content : - if line[0] != u'': - line = line.replace(u'\n', '').replace('"', '').split('\t') - dico[line[0]] = line[1:] - return dico + lines = [line.rstrip('\n\r').replace(u'\n', '').replace('"', '').split('\t') for line in content if line != u''] + return dict([[line[0], line[1:]] for line in lines]) def ReadLexique(parent, lang = 'french', filein = None): if lang != 'other' : @@ -441,20 +573,26 @@ def ReadLexique(parent, lang = 'french', filein = None): else : parent.lexique = ReadDicoAsDico(filein) else : - parent.lexique = {} + if filein is None : + parent.lexique = {} + else : + parent.lexique = ReadDicoAsDico(filein) -def ReadList(filein, encoding = sys.getdefaultencoding()): +def ReadList(filein, encoding = sys.getdefaultencoding(), sep = ';'): #file = open(filein) - file = codecs.open(filein, 'r', encoding) - content = file.readlines() - file.close() + with codecs.open(filein, 'r', encoding) as f : + content = f.read() + content = [line.replace('\n', '').replace('\r','').replace('\"', '').replace(',', '.').split(sep) for line in content.splitlines()] + #file = codecs.open(filein, 'r', encoding) + #content = file.readlines() + #file.close() first = content.pop(0) - first = first.replace('\n', '').replace('\r','').replace('\"', '').split(';') + #first = first.replace('\n', '').replace('\r','').replace('\"', '').split(sep) dict = {} i = 0 for line in content: - line = line.replace('\n', '').replace('\r','').replace('\"', '').replace(',', '.') - line = line.split(';') + #line = line.replace('\n', '').replace('\r','').replace('\"', '').replace(',', '.') + #line = line.split(';') nline = [line[0]] for val in line[1:]: if val == u'NA' : @@ -481,7 +619,6 @@ def exec_rcode(rpath, rcode, wait = True, graph = False): if sys.platform == 'darwin' : try : macversion = platform.mac_ver()[0].split('.') - print macversion if int(macversion[1]) < 5 : needX11 = True else : @@ -490,18 +627,21 @@ def exec_rcode(rpath, rcode, wait = True, graph = False): needX11 = False rpath = rpath.replace('\\','\\\\') + env = os.environ.copy() + if sys.platform == 'darwin' and 'LC_ALL' not in env: + env['LC_ALL'] = 'en_US.UTF-8' if not graph : if wait : if sys.platform == 'win32': error = call(["%s" % rpath, "--vanilla","--slave","-f", "%s" % rcode]) else : - error = call([rpath, '--vanilla','--slave',"-f %s" % rcode]) + error = call([rpath, '--slave', "--vanilla", "-f %s" % rcode, "--encoding=UTF-8"], env = env) return error else : if sys.platform == 'win32': pid = Popen(["%s" % rpath, '--vanilla','--slave','-f', "%s" % rcode]) else : - pid = Popen([rpath, '--vanilla','--slave',"-f %s" % rcode], stderr = PIPE) + pid = Popen([rpath, '--slave', "--vanilla", "-f %s" % rcode, "--encoding=UTF-8"], stderr = PIPE, env = env) return pid else : if wait : @@ -509,43 +649,42 @@ def exec_rcode(rpath, rcode, wait = True, graph = False): error = call(["%s" % rpath, '--vanilla','--slave','-f', "%s" % rcode]) elif sys.platform == 'darwin' and needX11: os.environ['DISPLAY'] = ':0.0' - error = call([rpath, '--vanilla','--slave',"-f %s" % rcode]) + error = call([rpath, '--vanilla','--slave',"-f %s" % rcode, "--encoding=UTF-8"], env = env) else : - error = call([rpath, '--vanilla','--slave',"-f %s" % rcode]) + error = call([rpath, '--vanilla','--slave',"-f %s" % rcode, "--encoding=UTF-8"], env = env) return error else : if sys.platform == 'win32': pid = Popen(["%s" % rpath, '--vanilla','--slave','-f', "%s" % rcode]) elif sys.platform == 'darwin' and needX11: os.environ['DISPLAY'] = ':0.0' - pid = Popen([rpath, '--vanilla','--slave',"-f %s" % rcode], stderr = PIPE) + pid = Popen([rpath, '--vanilla','--slave',"-f %s" % rcode, "--encoding=UTF-8"], stderr = PIPE, env = env) else : - pid = Popen([rpath, '--vanilla','--slave',"-f %s" % rcode], stderr = PIPE) + pid = Popen([rpath, '--vanilla','--slave',"-f %s" % rcode, "--encoding=UTF-8"], stderr = PIPE, env = env) return pid def check_Rresult(parent, pid) : if isinstance(pid, Popen) : if pid.returncode != 0 : error = pid.communicate() - print error error = [str(error[0]), error[1]] if error[1] is None : error[1] = 'None' parent.Rerror = '\n'.join([str(pid.returncode), '\n'.join(error)]) - #try : - raise Exception('\n'.join([u'Erreur R', '\n'.join(error[1:])])) + try : + raise Exception('\n'.join([u'Erreur R', '\n'.join(error[1:])])) + except : + BugReport(parent) return False - #except : - # BugReport(parent) else : return True else : if pid != 0 : - #try : - raise Exception(u'Erreur R') + try : + raise Exception(u'Erreur R') + except : + BugReport(parent) return False - #except : - # BugReport(parent) else : return True @@ -560,79 +699,71 @@ def read_list_file(filename, encoding = sys.getdefaultencoding()): ncontent=[line.replace('\n','').split(';') for line in content if line.strip() != ''] return ncontent -class MessageImage(wx.Frame): - def __init__(self, parent,title, size): - wx.Frame.__init__ ( self, parent, id = wx.ID_ANY, title = title, pos = wx.DefaultPosition, size = size, style = wx.DEFAULT_FRAME_STYLE ) - self.SetSizeHintsSz( wx.DefaultSize, wx.DefaultSize ) - self.imageFile = False - self.imagename = u"chi_classe.png" - self.HtmlPage = wx.html.HtmlWindow(self, -1) - self.HtmlPage.SetMinSize(size) - if "gtk2" in wx.PlatformInfo: - self.HtmlPage.SetStandardFonts() - self.HtmlPage.SetFonts('Courier', 'Courier') - - self.button_1 = wx.Button(self, wx.ID_CANCEL) - self.button_2 = wx.Button(self, wx.ID_SAVE) - self.Bind(wx.EVT_BUTTON, self.OnCloseMe, self.button_1) - self.Bind(wx.EVT_BUTTON, self.OnSaveImage, self.button_2) - self.do_layout() - - def do_layout(self): - self.sizer_1 = wx.BoxSizer(wx.VERTICAL) - self.sizer_2 = wx.BoxSizer(wx.HORIZONTAL) - self.sizer_1.Add(self.HtmlPage, 2, wx.EXPAND, 0) - - self.m_sdbSizer1 = wx.StdDialogButtonSizer() - self.m_sdbSizer1.AddButton( self.button_2 ) - self.m_sdbSizer1.AddButton( self.button_1 ) - self.m_sdbSizer1.Realize() - self.sizer_1.Add(self.m_sdbSizer1, 0, wx.EXPAND, 5) - self.SetSizer(self.sizer_1) - self.Layout() - self.sizer_1.Fit( self ) - def addsaveimage(self, imageFile) : - self.imageFile = imageFile - - def OnCloseMe(self, event): - self.Destroy() - - def OnSaveImage(self, event) : - dlg = wx.FileDialog( - self, message="Enregistrer sous...", defaultDir=os.getcwd(), - defaultFile= self.imagename, wildcard="png|*.png", style=wx.SAVE | wx.OVERWRITE_PROMPT - ) - dlg.SetFilterIndex(2) - dlg.CenterOnParent() - if dlg.ShowModal() == wx.ID_OK: - path = dlg.GetPath() - copyfile(self.imageFile, path) def progressbar(self, maxi) : - if 'parent' in dir(self) : - parent = self.parent - else : - parent = self - return wx.ProgressDialog("Traitements", + ira = wx.GetApp().GetTopWindow() + parent = ira + try : + maxi = int(maxi) + except : + maxi = 1 + prog = wx.ProgressDialog("Traitements", "Veuillez patienter...", maximum=maxi, parent=parent, style=wx.PD_APP_MODAL | wx.PD_AUTO_HIDE | wx.PD_ELAPSED_TIME | wx.PD_CAN_ABORT ) - + prog.SetSize((400,150)) + prog.SetIcon(ira._icon) + return prog def treat_var_mod(variables) : var_mod = {} - for variable in variables : - if u'_' in variable : - forme = variable.split(u'_') - var = forme[0] - mod = forme[1] - if not var in var_mod : - var_mod[var] = [variable] - else : - if not mod in var_mod[var] : - var_mod[var].append(variable) + variables = list(set(variables)) + varmod = [variable.split('_') for variable in variables] + vars = list(set([var[0] for var in varmod if len(var) >=2])) + for var in vars : + mods = ['_'.join(v) for v in varmod if v[0] == var] + var_mod[var] = mods + +# for variable in variables : +# if u'_' in variable : +# forme = variable.split(u'_') +# var = forme[0] +# mod = forme[1] +# if not var in var_mod : +# var_mod[var] = [variable] +# else : +# if not mod in var_mod[var] : +# var_mod[var].append(variable) return var_mod + +def doconcorde(corpus, uces, mots, uci = False) : + if not uci : + ucestxt1 = [row for row in corpus.getconcorde(uces)] + else : + ucestxt1 = [row for row in corpus.getuciconcorde(uces)] + ucestxt1 = dict(ucestxt1) + ucestxt = [] + ucis_txt = [] + listmot = [corpus.getlems()[lem].formes for lem in mots] + listmot = [corpus.getforme(fid).forme for lem in listmot for fid in lem] + mothtml = ['%s' % mot for mot in listmot] + dmots = dict(zip(listmot, mothtml)) + for uce in uces : + ucetxt = ucestxt1[uce].split() + ucetxt = ' '.join([dmots.get(mot, mot) for mot in ucetxt]) + if not uci : + ucis_txt.append('

' + ' '.join(corpus.ucis[corpus.getucefromid(uce).uci].etoiles) + '

') + else : + ucis_txt.append('

' + ' '.join(corpus.ucis[uce].etoiles) + '

') + ucestxt.append(ucetxt) + return ucis_txt, ucestxt + + +def getallstcarac(corpus, analyse) : + pathout = PathOut(analyse['ira']) + profils = ReadProfileAsDico(pathout['PROFILE_OUT'], Alceste, self.encoding) + print profils