2 # -*- coding: utf-8 -*-
3 #Author: Pierre Ratinaud
4 #Copyright (c) 2008-2012 Pierre Ratinaud
9 from ConfigParser import ConfigParser
10 from subprocess import Popen, call, PIPE
22 from shutil import copyfile
24 #from dialog import BugDialog
27 log = logging.getLogger('iramuteq')
30 indices_simi = [u'cooccurrence' ,'pourcentage de cooccurrence',u'Russel',u'Jaccard', 'Kulczynski1', 'Kulczynski2', 'Mountford', 'Fager', 'simple matching', 'Hamman', 'Faith', 'Tanimoto', 'Dice', 'Phi', 'Stiles', 'Michael', 'Mozley', 'Yule', 'Yule2', 'Ochiai', 'Simpson', 'Braun-Blanquet','Chi-squared', 'Phi-squared', 'Tschuprow', 'Cramer', 'Pearson', 'binomial']
34 def __init__(self, filein, syscoding = 'utf8') :
36 self.syscoding = syscoding
38 self.openedcorpus = {}
46 d = shelve.open(self.filein)
47 self.history = d.get('history', [])
48 self.matrix = d.get('matrix', [])
49 self.ordercorpus = dict([[corpus['uuid'], i] for i, corpus in enumerate(self.history)])
50 self.corpus = dict([[corpus['uuid'], corpus] for corpus in self.history])
51 self.analyses = dict([[analyse['uuid'], analyse] for corpus in self.history for analyse in corpus.get('analyses', [])])
52 self.matrixanalyse = dict([[mat['uuid'], mat] for mat in self.matrix])
56 d = shelve.open(self.filein)
57 d['history'] = self.history
58 d['matrix'] = self.matrix
61 def add(self, analyse) :
62 log.info('add to history %s' % analyse.get('corpus_name', 'pas un corpus'))
63 tosave = {'uuid' : analyse['uuid'], 'ira': analyse['ira'], 'type' : analyse['type']}
64 if analyse.get('corpus', False) :
65 if analyse['uuid'] in self.analyses :
67 tosave['corpus'] = analyse['corpus']
68 tosave['name'] = analyse['name']
69 acorpus_uuid = analyse['corpus']
70 if acorpus_uuid in self.corpus :
71 if 'analyses' in self.history[self.ordercorpus[acorpus_uuid]] :
72 self.history[self.ordercorpus[acorpus_uuid]]['analyses'].append(tosave)
74 self.history[self.ordercorpus[acorpus_uuid]]['analyses'] = [tosave]
76 self.orph.append(tosave)
78 tosave['corpus_name'] = analyse['corpus_name']
79 self.history.append(tosave)
83 def addMatrix(self, analyse) :
84 tosave = {'uuid' : analyse['uuid'], 'ira': analyse['ira'], 'type' : analyse['type']}
85 tosave['name'] = analyse['name']
86 self.matrix.append(tosave)
90 def addmultiple(self, analyses) :
91 log.info('add multiple')
92 for analyse in analyses :
93 tosave = {'uuid' : analyse['uuid'], 'ira': analyse['ira'], 'type' : analyse['type']}
94 corpus = analyse['corpus']
95 tosave['corpus'] = corpus
96 tosave['name'] = analyse['name']
97 if corpus in self.corpus :
98 if 'analyses' in self.history[self.ordercorpus[corpus]] :
99 self.history[self.ordercorpus[corpus]]['analyses'].append(tosave)
101 self.history[self.ordercorpus[corpus]]['analyses'] = [tosave]
105 def delete(self, analyse, corpus = False) :
106 log.info('delete %s' % analyse.get('name', 'noname'))
108 self.history.pop(self.ordercorpus[analyse['uuid']])
109 if analyse['uuid'] in self.openedcorpus :
110 del self.openedcorpus[analyse['uuid']]
111 elif analyse['uuid'] in self.analyses :
112 todel = [i for i, ana in enumerate(self.corpus[analyse['corpus']]['analyses']) if ana['uuid'] == analyse['uuid']][0]
113 self.history[self.ordercorpus[analyse['corpus']]]['analyses'].pop(todel)
114 elif analyse['uuid'] in self.matrixanalyse :
115 self.matrix = [mat for mat in self.matrix if mat['uuid'] != analyse['uuid']]
119 def addtab(self, analyse) :
120 self.opened[analyse['uuid']] = analyse
122 def rmtab(self, analyse) :
123 del self.opened[analyse['uuid']]
126 corpustodel = [corpus for corpus in self.history if not os.path.exists(corpus['ira'])]
127 for corpus in corpustodel :
128 self.delete(corpus, corpus = True)
131 return str(self.history)
134 def __init__(self, configfile=None, diff = None, parametres = None) :
135 self.configfile = configfile
136 self.conf = ConfigParser()
137 if configfile is not None :
138 self.conf.readfp(codecs.open(configfile, 'r', 'utf8'))
140 if parametres is not None :
141 self.doparametres(parametres)
143 def doparametres(self, parametres) :
146 def getsections(self) :
147 return self.conf.sections()
149 def getoptions(self, section = None, diff = None):
152 section = self.conf.sections()[0]
153 for option in self.conf.options(section) :
154 if self.conf.get(section, option).isdigit() :
155 parametres[option] = int(self.conf.get(section, option))
156 elif self.conf.get(section, option) == 'False' :
157 parametres[option] = False
158 elif self.conf.get(section, option) == 'True' :
159 parametres[option] = True
160 elif self.conf.get(section, option).startswith('(') and self.conf.get(section, option).endswith(')') :
161 parametres[option] = ast.literal_eval(self.conf.get(section, option))
162 elif self.conf.get(section, option).startswith('[') and self.conf.get(section, option).endswith(']') :
163 parametres[option] = ast.literal_eval(self.conf.get(section, option))
165 parametres[option] = self.conf.get(section, option)
166 if 'type' not in parametres :
167 parametres['type'] = section
170 def makeoptions(self, sections, parametres, outfile = None) :
172 for i, section in enumerate(sections) :
173 txt += '[%s]\n' % section
174 if not self.conf.has_section(section) :
175 self.conf.add_section(section)
176 for option in parametres[i] :
177 if isinstance(parametres[i][option], int) :
178 self.conf.set(section, option, `parametres[i][option]`)
179 txt += '%s = %i\n' % (option, parametres[i][option])
180 elif isinstance(parametres[i][option], basestring) :
181 self.conf.set(section, option, parametres[i][option].encode('utf8'))
182 txt += '%s = %s\n' % (option, parametres[i][option])
183 elif isinstance(parametres[i][option], wx.Colour) :
184 self.conf.set(section, option, str(parametres[i][option]))
185 txt += '%s = %s\n' % (option, str(parametres[i][option]))
186 elif option == 'analyses' :
189 self.conf.set(section, option, `parametres[i][option]`)
190 txt += '%s = %s\n' % (option, `parametres[i][option]`)
192 outfile = self.configfile
193 with codecs.open(outfile, 'w', 'utf8') as f :
197 def totext(self, parametres) :
200 for val in parametres :
201 if isinstance(parametres[val], int) :
202 txt.append(' \t\t: '.join([val, `parametres[val]`]))
203 elif isinstance(parametres[val], basestring) :
204 txt.append(' \t\t: '.join([val, parametres[val]]))
205 elif val in ['listet', 'stars'] :
208 txt.append(' \t\t: '.join([val, `parametres[val]`]))
209 return '\n'.join(txt)
212 def write_tab(tab, fileout) :
213 writer = csv.writer(open(fileout, 'wb'), delimiter=';', quoting = csv.QUOTE_NONNUMERIC)
214 writer.writerows(tab)
216 class BugDialog(wx.Dialog):
217 def __init__(self, *args, **kwds):
218 # begin wxGlade: MyDialog.__init__
219 kwds["style"] = wx.DEFAULT_DIALOG_STYLE
220 kwds["size"] = wx.Size(500, 200)
221 wx.Dialog.__init__(self, *args, **kwds)
222 self.SetTitle(kwds['title'])
223 self.text_ctrl_1 = wx.TextCtrl(self, -1, "", style=wx.TE_MULTILINE)
224 self.text_ctrl_1.SetBackgroundColour('#DDE8EB')
225 self.button_1 = wx.Button(self, wx.ID_OK, "")
227 self.__set_properties()
231 def __set_properties(self):
232 # begin wxGlade: MyDialog.__set_properties
233 self.SetMinSize(wx.Size(500, 200))
234 self.text_ctrl_1.SetMinSize(wx.Size(500, 200))
238 def __do_layout(self):
239 # begin wxGlade: MyDialog.__do_layout
240 sizer_1 = wx.BoxSizer(wx.VERTICAL)
241 sizer_1.Add(self.text_ctrl_1, 1, wx.EXPAND, 0)
242 sizer_1.Add(self.button_1, 0, wx.ALIGN_CENTER_HORIZONTAL, 0)
243 self.SetSizer(sizer_1)
248 def CreateIraFile(DictPathOut, clusternb, corpname='corpus_name', section = 'analyse'):
249 AnalyseConf = ConfigParser()
250 AnalyseConf.read(DictPathOut['ira'])
251 AnalyseConf.add_section(section)
252 date = datetime.datetime.now().ctime()
253 AnalyseConf.set(section, 'date', str(date))
254 AnalyseConf.set(section, 'clusternb', clusternb)
255 AnalyseConf.set(section, 'corpus_name', corpname)
257 fileout = open(DictPathOut['ira'], 'w')
258 AnalyseConf.write(fileout)
261 def sortedby(list, direct, *indices):
264 sortedby: sort a list of lists (e.g. a table) by one or more indices
265 (columns of the table) and return the sorted list
268 for list = [[2,3],[1,2],[3,1]]:
269 sortedby(list,1) will return [[3, 1], [1, 2], [2, 3]],
270 sortedby(list,0) will return [[1, 2], [2, 3], [3, 1]]
273 nlist = map(lambda x, indices=indices:
274 map(lambda i, x=x: x[i], indices) + [x],
279 nlist.sort(reverse=True)
280 return map(lambda l: l[-1], nlist)
282 def add_type(line, dictlem):
283 if line[4] in dictlem:
284 line.append(dictlem[line[4]])
289 def treat_line_alceste(i, line) :
290 if line[0] == '*' or line[0] == '*****' :
295 elif float(line[5].replace(',', '.')) < 0.0001:
297 elif float(line[5].replace(',', '.')) > 0.05:
298 line[5] = 'NS (%s)' % str(float(line[5].replace(',', '.')))[0:7]
300 line[5] = str(float(line[5].replace(',', '.')))[0:7]
301 return [i, int(line[0]), int(line[1]), float(line[2]), float(line[3]), line[6], line[4], line[5]]
303 def ReadProfileAsDico(File, Alceste=False, encoding = sys.getdefaultencoding()):
304 #print 'lecture des profils : ReadProfileAsDico'
306 # print 'lecture du dictionnaire de type'
308 # for line in parent.corpus.lem_type_list :
309 # dictlem[line[0]] = line[1]
311 print 'lecture des profiles'
312 #encoding = sys.getdefaultencoding()
313 FileReader = codecs.open(File, 'r', encoding)
314 Filecontent = FileReader.readlines()
318 rows = [row.replace('\n', '').replace("'", '').replace('\"', '').replace(',', '.').replace('\r','').split(';') for row in Filecontent]
320 ClusterNb = rows[0][2]
322 clusters = [row[2] for row in rows if row[0] == u'**']
323 valclusters = [row[1:4] for row in rows if row[0] == u'****']
324 lp = [i for i, line in enumerate(rows) if line[0] == u'****']
325 prof = [rows[lp[i] + 1:lp[i+1] - 1] for i in range(0, len(lp)-1)] + [rows[lp[-1] + 1:len(rows)]]
327 prof = [[add_type(row, dictlem) for row in pr] for pr in prof]
328 prof = [[treat_line_alceste(i,line) for i, line in enumerate(pr)] for pr in prof]
330 prof = [[line + [''] for line in pr] for pr in prof]
331 prof = [[treat_line_alceste(i,line) for i, line in enumerate(pr)] for pr in prof]
332 for i, cluster in enumerate(clusters):
333 DictProfile[cluster] = [valclusters[i]] + prof[i]
336 def GetTxtProfile(dictprofile, cluster_size) :
338 for classe in range(0, len(dictprofile)) :
339 prof = dictprofile[str(classe + 1)]
340 clinfo = cluster_size[classe]
341 proflist.append('\n'.join([' '.join(['classe %i' % (classe + 1), '-', '%s uce sur %s - %s%%' % (clinfo[0], clinfo[1], clinfo[2])]), '\n'.join(['%5s|%5s|%6s|%6s|%8s|%8s|%20s\t%10s' % tuple([str(val) for val in line]) for line in prof if len(line)==8])]))
342 return '\n\n'.join(proflist)
344 def formatExceptionInfo(maxTBlevel=5):
345 cla, exc, trbk = sys.exc_info()
347 excName = cla.__name__
351 excArgs = exc.args[0]
353 excArgs = "<no args>"
354 excTb = traceback.format_tb(trbk, maxTBlevel)
355 return (excName, excArgs, excTb)
358 #fonction des etudiants de l'iut
359 def decoupercharact(chaine, longueur, longueurOptimale, separateurs = None) :
361 on part du dernier caractère, et on recule jusqu'au début de la chaîne.
362 Si on trouve un '$', c'est fini.
363 Sinon, on cherche le meilleur candidat. C'est-Ã -dire le rapport poids/distance le plus important.
365 separateurs = [[u'.', 60.0], [u'?', 60.0], [u'!', 60.0], [u'£$£', 60], [u':', 50.0], [u';', 40.0], [u',', 10.0], [u' ', 0.1]]
366 trouve = False # si on a trouvé un bon séparateur
367 iDecoupe = 0 # indice du caractere ou il faut decouper
369 # on découpe la chaine pour avoir au maximum 240 caractères
370 longueur = min(longueur, len(chaine) - 1)
371 chaineTravail = chaine[:longueur + 1]
373 meilleur = ['', 0, 0] # type, poids et position du meilleur separateur
375 # on vérifie si on ne trouve pas un '$'
376 indice = chaineTravail.find(u'$')
381 # si on ne trouve rien, on cherche le meilleur séparateur
384 caractere = chaineTravail[nbCar]
385 distance = abs(longueurOptimale - nbCar) + 1
386 meilleureDistance = abs(longueurOptimale - meilleur[2]) + 1
388 # on vérifie si le caractére courant est une marque de ponctuation
389 for s in separateurs:
390 if caractere == s[0]:
391 # si c'est une ponctuation
393 if s[1] / distance > float(meilleur[1]) / meilleureDistance:
401 # et on termine la recherche
404 # on passe au caractère précédant
409 fin = chaine[iDecoupe + 1:]
410 retour = chaineTravail[:iDecoupe]
411 return len(retour) > 0, retour.split(), fin
412 # si on a rien trouvé
413 return False, chaine.split(), ''
416 exceptions = {'paragrapheOT' : u"Un problème de formatage (présence d'un marqueur de paragraphe (-*) en dehors d'un texte) est survenu à la ligne ",
417 'EmptyText' : u"Texte vide (probablement un problème de formatage du corpus). Le problème est apparu à la ligne ",
418 'CorpusEncoding' : u"Problème d'encodage.",
419 'TextBeforeTextMark' : u"Problème de formatage : du texte avant le premier marqueur de texte (****). Le problème est survenu à la ligne ",
422 def BugReport(parent, error = None):
423 for ch in parent.GetChildren():
424 if "<class 'wx._windows.ProgressDialog'>" == str(type(ch)):
426 excName, exc, excTb = formatExceptionInfo()
427 if excName == 'Exception' :
429 if len(exc.split()) == 2 :
430 mss, linenb = exc.split()
431 if mss in exceptions :
432 txt = exceptions[mss] + linenb
436 if exc in exceptions :
437 txt = exceptions[exc]
440 title = "Information"
442 txt = u' !== BUG ==! \n'
443 txt += u'*************************************\n'
444 txt += '\n'.join(excTb).replace(' ', ' ')
445 txt += excName + '\n'
449 dial = BugDialog(parent, **{'title' : title})
450 if 'Rerror' in dir(parent) :
454 dial.text_ctrl_1.write(txt)
455 dial.CenterOnParent()
459 def PlaySound(parent):
460 if parent.pref.getboolean('iramuteq', 'sound') :
462 if "gtk2" in wx.PlatformInfo:
463 error = Popen(['aplay','-q',os.path.join(parent.AppliPath,'son_fin.wav')])
465 sound = wx.Sound(os.path.join(parent.AppliPath, 'son_fin.wav'))
466 sound.Play(wx.SOUND_SYNC)
470 def ReadDicoAsDico(dicopath):
471 with codecs.open(dicopath, 'r', 'UTF8') as f:
472 content = f.readlines()
474 for line in content :
476 line = line.rstrip('\n\r').replace(u'\n', '').replace('"', '').split('\t')
477 dico[line[0]] = line[1:]
480 def ReadLexique(parent, lang = 'french', filein = None):
483 parent.lexique = ReadDicoAsDico(parent.DictPath.get(lang, 'french'))
485 parent.lexique = ReadDicoAsDico(filein)
489 def ReadList(filein, encoding = sys.getdefaultencoding()):
491 file = codecs.open(filein, 'r', encoding)
492 content = file.readlines()
494 first = content.pop(0)
495 first = first.replace('\n', '').replace('\r','').replace('\"', '').split(';')
499 line = line.replace('\n', '').replace('\r','').replace('\"', '').replace(',', '.')
500 line = line.split(';')
509 don = float('%.5f' % float(val))
515 def exec_RCMD(rpath, command) :
516 log.info('R CMD INSTALL %s' % command)
517 rpath = rpath.replace('\\','\\\\')
518 error = call(["%s" % rpath, 'CMD', 'INSTALL', "%s" % command])
521 def exec_rcode(rpath, rcode, wait = True, graph = False):
522 log.info("R Script : %s" % rcode)
524 if sys.platform == 'darwin' :
526 macversion = platform.mac_ver()[0].split('.')
527 if int(macversion[1]) < 5 :
534 rpath = rpath.replace('\\','\\\\')
535 env = os.environ.copy()
536 if sys.platform == 'darwin' and 'LC_ALL' not in env:
537 env['LC_ALL'] = 'en_US.UTF-8'
540 if sys.platform == 'win32':
541 error = call(["%s" % rpath, "--vanilla","--slave","-f", "%s" % rcode])
543 error = call([rpath, '--slave', "--vanilla", "-f %s" % rcode, "--encoding=UTF-8"], env = env)
546 if sys.platform == 'win32':
547 pid = Popen(["%s" % rpath, '--vanilla','--slave','-f', "%s" % rcode])
549 pid = Popen([rpath, '--slave', "--vanilla", "-f %s" % rcode, "--encoding=UTF-8"], stderr = PIPE, env = env)
553 if sys.platform == 'win32':
554 error = call(["%s" % rpath, '--vanilla','--slave','-f', "%s" % rcode])
555 elif sys.platform == 'darwin' and needX11:
556 os.environ['DISPLAY'] = ':0.0'
557 error = call([rpath, '--vanilla','--slave',"-f %s" % rcode, "--encoding=UTF-8"], env = env)
559 error = call([rpath, '--vanilla','--slave',"-f %s" % rcode, "--encoding=UTF-8"], env = env)
562 if sys.platform == 'win32':
563 pid = Popen(["%s" % rpath, '--vanilla','--slave','-f', "%s" % rcode])
564 elif sys.platform == 'darwin' and needX11:
565 os.environ['DISPLAY'] = ':0.0'
566 pid = Popen([rpath, '--vanilla','--slave',"-f %s" % rcode, "--encoding=UTF-8"], stderr = PIPE, env = env)
568 pid = Popen([rpath, '--vanilla','--slave',"-f %s" % rcode, "--encoding=UTF-8"], stderr = PIPE, env = env)
571 def check_Rresult(parent, pid) :
572 if isinstance(pid, Popen) :
573 if pid.returncode != 0 :
574 error = pid.communicate()
575 error = [str(error[0]), error[1]]
576 if error[1] is None :
578 parent.Rerror = '\n'.join([str(pid.returncode), '\n'.join(error)])
580 raise Exception('\n'.join([u'Erreur R', '\n'.join(error[1:])]))
589 raise Exception(u'Erreur R')
596 def print_liste(filename,liste):
597 with open(filename,'w') as f :
599 f.write(';'.join(graph)+'\n')
601 def read_list_file(filename, encoding = sys.getdefaultencoding()):
602 with codecs.open(filename,'rU', encoding) as f :
603 content=f.readlines()
604 ncontent=[line.replace('\n','').split(';') for line in content if line.strip() != '']
607 class MessageImage(wx.Frame):
608 def __init__(self, parent,title, size):
609 wx.Frame.__init__ ( self, parent, id = wx.ID_ANY, title = title, pos = wx.DefaultPosition, size = size, style = wx.DEFAULT_FRAME_STYLE )
610 self.SetSizeHintsSz( wx.DefaultSize, wx.DefaultSize )
611 self.imageFile = False
612 self.imagename = u"chi_classe.png"
613 self.HtmlPage = wx.html.HtmlWindow(self, -1)
614 self.HtmlPage.SetMinSize(size)
615 if "gtk2" in wx.PlatformInfo:
616 self.HtmlPage.SetStandardFonts()
617 self.HtmlPage.SetFonts('Courier', 'Courier')
619 self.button_1 = wx.Button(self, wx.ID_CANCEL)
620 self.button_2 = wx.Button(self, wx.ID_SAVE)
621 self.Bind(wx.EVT_BUTTON, self.OnCloseMe, self.button_1)
622 self.Bind(wx.EVT_BUTTON, self.OnSaveImage, self.button_2)
626 self.sizer_1 = wx.BoxSizer(wx.VERTICAL)
627 self.sizer_2 = wx.BoxSizer(wx.HORIZONTAL)
628 self.sizer_1.Add(self.HtmlPage, 2, wx.EXPAND, 0)
630 self.m_sdbSizer1 = wx.StdDialogButtonSizer()
631 self.m_sdbSizer1.AddButton( self.button_2 )
632 self.m_sdbSizer1.AddButton( self.button_1 )
633 self.m_sdbSizer1.Realize()
634 self.sizer_1.Add(self.m_sdbSizer1, 0, wx.EXPAND, 5)
635 self.SetSizer(self.sizer_1)
637 self.sizer_1.Fit( self )
639 def addsaveimage(self, imageFile) :
640 self.imageFile = imageFile
642 def OnCloseMe(self, event):
645 def OnSaveImage(self, event) :
647 self, message="Enregistrer sous...", defaultDir=os.getcwd(),
648 defaultFile= self.imagename, wildcard="png|*.png", style=wx.SAVE | wx.OVERWRITE_PROMPT
650 dlg.SetFilterIndex(2)
652 if dlg.ShowModal() == wx.ID_OK:
654 copyfile(self.imageFile, path)
657 def progressbar(self, maxi) :
658 if 'parent' in dir(self) :
662 return wx.ProgressDialog("Traitements",
663 "Veuillez patienter...",
666 style=wx.PD_APP_MODAL | wx.PD_AUTO_HIDE | wx.PD_ELAPSED_TIME | wx.PD_CAN_ABORT
670 def treat_var_mod(variables) :
672 for variable in variables :
673 if u'_' in variable :
674 forme = variable.split(u'_')
677 if not var in var_mod :
678 var_mod[var] = [variable]
680 if not mod in var_mod[var] :
681 var_mod[var].append(variable)
684 def doconcorde(corpus, uces, mots) :
685 ucestxt1 = [row for row in corpus.getconcorde(uces)]
686 ucestxt1 = dict(ucestxt1)
689 listmot = [corpus.getlems()[lem].formes for lem in mots]
690 listmot = [corpus.getforme(fid).forme for lem in listmot for fid in lem]
691 mothtml = ['<font color=red><b>'+mot+'</b></font>' for mot in listmot]
692 dmots = dict(zip(listmot, mothtml))
694 ucetxt = ucestxt1[uce].split()
695 ucetxt = ' '.join([dmots.get(mot, mot) for mot in ucetxt])
696 ucis_txt.append('<p><b>' + ' '.join(corpus.ucis[corpus.getucefromid(uce).uci].etoiles) + '</b></p>')
697 ucestxt.append(ucetxt)
698 return ucis_txt, ucestxt