2 # -*- coding: utf-8 -*-
3 #Author: Pierre Ratinaud
4 #Copyright (c) 2008-2012 Pierre Ratinaud
9 from ConfigParser import ConfigParser
10 from subprocess import Popen, call, PIPE
22 from shutil import copyfile
23 #from dialog import BugDialog
26 log = logging.getLogger('iramuteq')
29 indices_simi = [u'cooccurrence' ,'pourcentage de cooccurrence',u'Russel',u'Jaccard', 'Kulczynski1', 'Kulczynski2', 'Mountford', 'Fager', 'simple matching', 'Hamman', 'Faith', 'Tanimoto', 'Dice', 'Phi', 'Stiles', 'Michael', 'Mozley', 'Yule', 'Yule2', 'Ochiai', 'Simpson', 'Braun-Blanquet','Chi-squared', 'Phi-squared', 'Tschuprow', 'Cramer', 'Pearson', 'binomial']
33 def __init__(self, filein, syscoding = 'utf8') :
35 self.syscoding = syscoding
37 self.openedcorpus = {}
44 self.conf = DoConf(self.filein)
47 for i, section in enumerate(self.conf.conf.sections()) :
48 if self.conf.conf.has_option(section, 'corpus_name') :
49 self.corpora[section] = self.conf.getoptions(section)
50 self.order[len(self.order)] = section
52 self.analyses[section] = self.conf.getoptions(section)
53 self.ordera[len(self.ordera)] = section
55 for corpus in self.corpora :
56 self.history[corpus] = copy(self.corpora[corpus])
57 for analyse in self.analyses :
58 if self.analyses[analyse]['corpus'] in self.corpora :
59 if 'analyses' in self.history[self.analyses[analyse]['corpus']] :
60 self.history[self.analyses[analyse]['corpus']]['analyses'].append(self.analyses[analyse])
63 self.history[self.analyses[analyse]['corpus']]['analyses'] = [self.analyses[analyse]]
66 self.history[analyse] = self.analyses[analyse]
67 #for analyse in todel :
68 # del self.analyses[analyse]
71 sections = self.corpora.keys() + self.analyses.keys()
72 parametres = [self.corpora[key] for key in self.corpora.keys()] + [self.analyses[key] for key in self.analyses.keys()]
73 self.conf.makeoptions(sections, parametres)
74 log.info('write history')
76 def add(self, analyse) :
77 tosave = {'uuid' : analyse['uuid'], 'ira': analyse['ira'], 'type' : analyse['type']}
78 if analyse.get('corpus', False) :
79 tosave['corpus'] = analyse['corpus']
80 tosave['name'] = analyse['name']
81 acorpus_uuid = analyse['corpus']
82 if acorpus_uuid in self.corpora :
83 if 'analyses' in self.history[acorpus_uuid] :
84 self.history[acorpus_uuid]['analyses'].append(tosave)
86 self.history[acorpus_uuid]['analyses'] = [tosave]
87 self.analyses[analyse['uuid']] = tosave
89 self.analyses[analyse['uuid']] = tosave
90 elif 'corpus_name' in analyse :
91 tosave['corpus_name'] = analyse['corpus_name']
92 self.history[analyse['uuid']] = tosave
93 self.corpora[analyse['uuid']] = tosave
96 def delete(self, uuid, corpus = False) :
98 del self.corpora[uuid]
99 self.conf.conf.remove_section(uuid)
100 for analyse in self.history[uuid].get('analyses', [False]) :
102 del self.analyses[analyse['uuid']]
103 self.conf.conf.remove_section(analyse['uuid'])
105 del self.analyses[uuid]
106 self.conf.conf.remove_section(uuid)
109 def addtab(self, analyse) :
110 self.opened[analyse['uuid']] = analyse
112 def rmtab(self, analyse) :
113 del self.opened[analyse['uuid']]
116 return str(self.history)
119 def __init__(self, configfile=None, diff = None, parametres = None) :
120 self.configfile = configfile
121 self.conf = ConfigParser()
122 if configfile is not None :
123 self.conf.readfp(codecs.open(configfile, 'r', 'utf8'))
125 if parametres is not None :
126 self.doparametres(parametres)
128 def doparametres(self, parametres) :
131 def getsections(self) :
132 return self.conf.sections()
134 def getoptions(self, section = None, diff = None):
137 section = self.conf.sections()[0]
138 for option in self.conf.options(section) :
139 if self.conf.get(section, option).isdigit() :
140 parametres[option] = int(self.conf.get(section, option))
141 elif self.conf.get(section, option) == 'False' :
142 parametres[option] = False
143 elif self.conf.get(section, option) == 'True' :
144 parametres[option] = True
145 elif self.conf.get(section, option).startswith('(') and self.conf.get(section, option).endswith(')') :
146 parametres[option] = ast.literal_eval(self.conf.get(section, option))
148 parametres[option] = self.conf.get(section, option)
149 if 'type' not in parametres :
150 parametres['type'] = section
153 def makeoptions(self, sections, parametres, outfile = None) :
154 for i, section in enumerate(sections) :
155 if not self.conf.has_section(section) :
156 self.conf.add_section(section)
157 for option in parametres[i] :
158 if isinstance(parametres[i][option], int) :
159 self.conf.set(section, option, `parametres[i][option]`)
160 elif isinstance(parametres[i][option], basestring) :
161 self.conf.set(section, option, parametres[i][option].encode('utf8'))
162 elif isinstance(parametres[i][option], wx.Colour) :
163 self.conf.set(section, option, str(parametres[i][option]))
164 elif option == 'analyses' :
167 self.conf.set(section, option, `parametres[i][option]`)
169 outfile = self.configfile
170 with codecs.open(outfile, 'w', 'utf8') as f :
173 def totext(self, parametres) :
175 for val in parametres :
176 if isinstance(parametres[val], int) :
177 txt.append(' \t\t: '.join([val, `parametres[val]`]))
179 txt.append(' \t\t: '.join([val, parametres[val]]))
180 return '\n'.join(txt)
183 def write_tab(tab, fileout) :
184 writer = csv.writer(open(fileout, 'wb'), delimiter=';', quoting = csv.QUOTE_NONNUMERIC)
185 writer.writerows(tab)
187 class BugDialog(wx.Dialog):
188 def __init__(self, *args, **kwds):
189 # begin wxGlade: MyDialog.__init__
190 kwds["style"] = wx.DEFAULT_DIALOG_STYLE
191 kwds["size"] = wx.Size(500, 200)
192 wx.Dialog.__init__(self, *args, **kwds)
193 self.text_ctrl_1 = wx.TextCtrl(self, -1, "", style=wx.TE_MULTILINE)
194 self.button_1 = wx.Button(self, wx.ID_OK, "")
196 self.__set_properties()
200 def __set_properties(self):
201 # begin wxGlade: MyDialog.__set_properties
203 self.SetMinSize(wx.Size(500, 200))
204 self.text_ctrl_1.SetMinSize(wx.Size(500, 200))
208 def __do_layout(self):
209 # begin wxGlade: MyDialog.__do_layout
210 sizer_1 = wx.BoxSizer(wx.VERTICAL)
211 sizer_1.Add(self.text_ctrl_1, 1, wx.EXPAND, 0)
212 sizer_1.Add(self.button_1, 0, wx.ALIGN_CENTER_HORIZONTAL, 0)
213 self.SetSizer(sizer_1)
218 def CreateIraFile(DictPathOut, clusternb, corpname='corpus_name', section = 'analyse'):
219 AnalyseConf = ConfigParser()
220 AnalyseConf.read(DictPathOut['ira'])
221 AnalyseConf.add_section(section)
222 date = datetime.datetime.now().ctime()
223 AnalyseConf.set(section, 'date', str(date))
224 AnalyseConf.set(section, 'clusternb', clusternb)
225 AnalyseConf.set(section, 'corpus_name', corpname)
227 fileout = open(DictPathOut['ira'], 'w')
228 AnalyseConf.write(fileout)
231 def sortedby(list, direct, *indices):
234 sortedby: sort a list of lists (e.g. a table) by one or more indices
235 (columns of the table) and return the sorted list
238 for list = [[2,3],[1,2],[3,1]]:
239 sortedby(list,1) will return [[3, 1], [1, 2], [2, 3]],
240 sortedby(list,0) will return [[1, 2], [2, 3], [3, 1]]
243 nlist = map(lambda x, indices=indices:
244 map(lambda i, x=x: x[i], indices) + [x],
249 nlist.sort(reverse=True)
250 return map(lambda l: l[-1], nlist)
252 def add_type(line, dictlem):
253 if line[4] in dictlem:
254 line.append(dictlem[line[4]])
259 def treat_line_alceste(i, line) :
260 if line[0] == '*' or line[0] == '*****' :
265 elif float(line[5].replace(',', '.')) < 0.0001:
267 elif float(line[5].replace(',', '.')) > 0.05:
268 line[5] = 'NS (%s)' % str(float(line[5].replace(',', '.')))[0:7]
270 line[5] = str(float(line[5].replace(',', '.')))[0:7]
271 return [i, int(line[0]), int(line[1]), float(line[2]), float(line[3]), line[6], line[4], line[5]]
273 def ReadProfileAsDico(parent, File, Alceste=False, encoding = sys.getdefaultencoding()):
274 #print 'lecture des profils : ReadProfileAsDico'
276 # print 'lecture du dictionnaire de type'
278 # for line in parent.corpus.lem_type_list :
279 # dictlem[line[0]] = line[1]
281 print 'lecture des profiles'
282 #encoding = sys.getdefaultencoding()
284 FileReader = codecs.open(File, 'r', encoding)
285 Filecontent = FileReader.readlines()
289 rows = [row.replace('\n', '').replace("'", '').replace('\"', '').replace(',', '.').replace('\r','').split(';') for row in Filecontent]
291 ClusterNb = rows[0][2]
293 clusters = [row[2] for row in rows if row[0] == u'**']
294 valclusters = [row[1:4] for row in rows if row[0] == u'****']
295 lp = [i for i, line in enumerate(rows) if line[0] == u'****']
296 prof = [rows[lp[i] + 1:lp[i+1] - 1] for i in range(0, len(lp)-1)] + [rows[lp[-1] + 1:len(rows)]]
298 prof = [[add_type(row, dictlem) for row in pr] for pr in prof]
299 prof = [[treat_line_alceste(i,line) for i, line in enumerate(pr)] for pr in prof]
301 prof = [[line + [''] for line in pr] for pr in prof]
302 prof = [[treat_line_alceste(i,line) for i, line in enumerate(pr)] for pr in prof]
303 for i, cluster in enumerate(clusters):
304 DictProfile[cluster] = [valclusters[i]] + prof[i]
307 def GetTxtProfile(dictprofile) :
309 for classe in range(0, len(dictprofile)) :
310 prof = dictprofile[str(classe + 1)]
312 proflist.append('\n'.join([' '.join(['classe %i' % (classe + 1), '-', '%s uce sur %s - %s%%' % (clinfo[0], clinfo[1], clinfo[2])]), '\n'.join(['%5s|%5s|%6s|%6s|%8s|%8s|%20s\t%10s' % tuple([str(val) for val in line]) for line in prof if len(line)==8])]))
313 return '\n\n'.join(proflist)
315 def formatExceptionInfo(maxTBlevel=5):
316 cla, exc, trbk = sys.exc_info()
317 excName = cla.__name__
319 excArgs = exc.__dict__["args"]
321 excArgs = "<no args>"
322 excTb = traceback.format_tb(trbk, maxTBlevel)
323 return (excName, excArgs, excTb)
326 #fonction des etudiants de l'iut
327 def decoupercharact(chaine, longueur, longueurOptimale, separateurs = None) :
329 on part du dernier caractère, et on recule jusqu'au début de la chaîne.
330 Si on trouve un '$', c'est fini.
331 Sinon, on cherche le meilleur candidat. C'est-Ã -dire le rapport poids/distance le plus important.
333 separateurs = [[u'.', 60.0], [u'?', 60.0], [u'!', 60.0], [u'£', 60], [u':', 50.0], [u';', 40.0], [u',', 10.0], [u' ', 0.1]]
334 trouve = False # si on a trouvé un bon séparateur
335 iDecoupe = 0 # indice du caractere ou il faut decouper
337 # on découpe la chaine pour avoir au maximum 240 caractères
338 longueur = min(longueur, len(chaine) - 1)
339 chaineTravail = chaine[:longueur + 1]
341 meilleur = ['', 0, 0] # type, poids et position du meilleur separateur
343 # on vérifie si on ne trouve pas un '$'
344 indice = chaineTravail.find(u'$')
349 # si on ne trouve rien, on cherche le meilleur séparateur
352 caractere = chaineTravail[nbCar]
353 distance = abs(longueurOptimale - nbCar) + 1
354 meilleureDistance = abs(longueurOptimale - meilleur[2]) + 1
356 # on vérifie si le caractére courant est une marque de ponctuation
357 for s in separateurs:
358 if caractere == s[0]:
359 # si c'est une ponctuation
361 if s[1] / distance > float(meilleur[1]) / meilleureDistance:
369 # et on termine la recherche
372 # on passe au caractère précédant
377 fin = chaine[iDecoupe + 1:]
378 retour = chaineTravail[:iDecoupe]
379 return len(retour) > 0, retour.split(), fin
380 # si on a rien trouvé
381 return False, chaine.split(), ''
383 def BugReport(parent):
384 for ch in parent.GetChildren():
385 if "<class 'wx._windows.ProgressDialog'>" == str(type(ch)):
387 dial = BugDialog(parent)
388 txt = u' !== BUG ==! \n'
389 txt += u'*************************************\n'
390 for line in formatExceptionInfo():
391 if type(line) == type([]):
393 txt += don.replace(' ', ' ')
396 if 'Rerror' in dir(parent) :
399 print formatExceptionInfo()
401 dial.text_ctrl_1.write(txt)
402 dial.CenterOnParent()
404 raise NameError('Bug')
406 def PlaySound(parent):
407 if parent.pref.getboolean('iramuteq', 'sound') :
409 if "gtk2" in wx.PlatformInfo:
410 error = Popen(['aplay','-q',os.path.join(parent.AppliPath,'son_fin.wav')])
412 sound = wx.Sound(os.path.join(parent.AppliPath, 'son_fin.wav'))
413 sound.Play(wx.SOUND_SYNC)
417 def ReadDicoAsDico(dicopath):
418 with codecs.open(dicopath, 'r', 'UTF8') as f:
419 content = f.readlines()
421 for line in content :
423 line = line.replace(u'\n', '').replace('"', '').split('\t')
424 dico[line[0]] = line[1:]
427 def ReadLexique(parent, lang = 'french'):
428 parent.lexique = ReadDicoAsDico(parent.DictPath.get(lang, 'french'))
430 def ReadList(filein, encoding = sys.getdefaultencoding()):
432 file = codecs.open(filein, 'r', encoding)
433 content = file.readlines()
435 first = content.pop(0)
436 first = first.replace('\n', '').replace('\r','').replace('\"', '').split(';')
440 line = line.replace('\n', '').replace('\r','').replace('\"', '').replace(',', '.')
441 line = line.split(';')
450 don = float('%.5f' % float(val))
456 def exec_rcode(rpath, rcode, wait = True, graph = False):
459 if sys.platform == 'darwin' :
461 macversion = platform.mac_ver()[0].split('.')
463 if int(macversion[1]) < 5 :
470 rpath = rpath.replace('\\','\\\\')
473 if sys.platform == 'win32':
474 error = call(["%s" % rpath, "--vanilla","--slave","-f", "%s" % rcode])
476 error = call([rpath, '--vanilla','--slave',"-f %s" % rcode])
479 if sys.platform == 'win32':
480 pid = Popen(["%s" % rpath, '--vanilla','--slave','-f', "%s" % rcode])
482 pid = Popen([rpath, '--vanilla','--slave',"-f %s" % rcode], stderr = PIPE)
486 if sys.platform == 'win32':
487 error = call(["%s" % rpath, '--vanilla','--slave','-f', "%s" % rcode])
488 elif sys.platform == 'darwin' and needX11:
489 os.environ['DISPLAY'] = ':0.0'
490 error = call([rpath, '--vanilla','--slave',"-f %s" % rcode])
492 error = call([rpath, '--vanilla','--slave',"-f %s" % rcode])
495 if sys.platform == 'win32':
496 pid = Popen(["%s" % rpath, '--vanilla','--slave','-f', "%s" % rcode])
497 elif sys.platform == 'darwin' and needX11:
498 os.environ['DISPLAY'] = ':0.0'
499 pid = Popen([rpath, '--vanilla','--slave',"-f %s" % rcode], stderr = PIPE)
501 pid = Popen([rpath, '--vanilla','--slave',"-f %s" % rcode], stderr = PIPE)
504 def check_Rresult(parent, pid) :
505 if isinstance(pid, Popen) :
506 if pid.returncode != 0 :
507 error = pid.communicate()
508 error = [str(error[0]), error[1]]
509 if error[1] is None :
511 parent.Rerror = '\n'.join([str(pid.returncode), '\n'.join(error)])
513 raise Exception('\n'.join(u'Erreur R', '\n'.join(error[1:])))
519 raise Exception(u'Erreur R')
523 def print_liste(filename,liste):
524 with open(filename,'w') as f :
526 f.write(';'.join(graph)+'\n')
528 def read_list_file(filename, encoding = sys.getdefaultencoding()):
529 with codecs.open(filename,'rU', encoding) as f :
530 content=f.readlines()
531 ncontent=[line.replace('\n','').split(';') for line in content if line.strip() != '']
534 class MessageImage(wx.Frame):
535 def __init__(self, *args, **kwds):
536 # begin wxGlade: MyFrame.__init__
537 kwds["style"] = wx.DEFAULT_FRAME_STYLE
538 wx.Frame.__init__(self, *args, **kwds)
539 #self.text_ctrl_1 = wx.TextCtrl(self, -1, "", style=wx.TE_MULTILINE)
540 self.imageFile = False
541 self.imagename = u"chi_classe.png"
542 self.HtmlPage = wx.html.HtmlWindow(self, -1)
543 if "gtk2" in wx.PlatformInfo:
544 self.HtmlPage.SetStandardFonts()
545 self.HtmlPage.SetFonts('Courier', 'Courier')
547 self.button_1 = wx.Button(self, -1, u"Fermer")
548 self.Bind(wx.EVT_BUTTON, self.OnCloseMe, self.button_1)
549 self.Bind(wx.EVT_CLOSE, self.OnCloseWindow)
553 def __do_layout(self):
554 # begin wxGlade: MyFrame.__do_layout
555 sizer_1 = wx.BoxSizer(wx.VERTICAL)
556 self.sizer_2 = wx.BoxSizer(wx.VERTICAL)
557 self.sizer_2.Add(self.HtmlPage, 1, wx.EXPAND | wx.ADJUST_MINSIZE, 0)
558 self.sizer_2.Add(self.button_1, 0, wx.ALIGN_CENTER_HORIZONTAL | wx.ADJUST_MINSIZE, 0)
559 sizer_1.Add(self.sizer_2, 1, wx.EXPAND, 0)
560 self.SetAutoLayout(True)
561 self.SetSizer(sizer_1)
564 def addsaveimage(self, imageFile) :
565 self.imageFile = imageFile
566 self.button_2 = wx.Button(self, -1, u"Enregistrer l'image...")
567 self.Bind(wx.EVT_BUTTON, self.OnSaveImage, self.button_2)
568 self.sizer_2.Add(self.button_2, 0, wx.ALIGN_CENTER_HORIZONTAL | wx.ADJUST_MINSIZE, 0)
571 def OnCloseMe(self, event):
574 def OnCloseWindow(self, event):
577 def OnSaveImage(self, event) :
579 self, message="Enregistrer sous...", defaultDir=os.getcwd(),
580 defaultFile= self.imagename, wildcard="png|*.png", style=wx.SAVE | wx.OVERWRITE_PROMPT
582 dlg.SetFilterIndex(2)
584 if dlg.ShowModal() == wx.ID_OK:
586 copyfile(self.imageFile, path)
589 def progressbar(self, maxi) :
590 if 'parent' in dir(self) :
594 return wx.ProgressDialog("Traitements",
595 "Veuillez patienter...",
598 style=wx.PD_APP_MODAL | wx.PD_AUTO_HIDE | wx.PD_ELAPSED_TIME | wx.PD_CAN_ABORT
602 def treat_var_mod(variables) :
604 for variable in variables :
605 if u'_' in variable :
606 forme = variable.split(u'_')
609 if not var in var_mod :
610 var_mod[var] = [variable]
612 if not mod in var_mod[var] :
613 var_mod[var].append(variable)