X-Git-Url: http://iramuteq.org/git?p=iramuteq;a=blobdiff_plain;f=functions.py;h=b41c483545574ab9670bb31f9f4dfb00a1c9284d;hp=62f669ce24d0ee52a58780313a183b7d0c1aff53;hb=b4ab59a79dbc62d4234778e793c05718648c6775;hpb=aae91ab48172a83c49bb502ac737bcc1b3a6685c diff --git a/functions.py b/functions.py index 62f669c..b41c483 100644 --- a/functions.py +++ b/functions.py @@ -21,6 +21,7 @@ import datetime from copy import copy from shutil import copyfile import shelve +import json #from dialog import BugDialog import logging @@ -118,7 +119,7 @@ class History : d['history'] = self.history d['matrix'] = self.matrix d.close() - + def add(self, analyse) : log.info('add to history %s' % analyse.get('corpus_name', 'pas un corpus')) tosave = {'uuid' : analyse['uuid'], 'ira': analyse['ira'], 'type' : analyse['type']} @@ -161,7 +162,7 @@ class History : self.matrix[self.ordermatrix[tosave['matrix']]]['analyses'].append(tosave) self.write() self.read() - + def addmultiple(self, analyses) : log.info('add multiple') for analyse in analyses : @@ -197,7 +198,7 @@ class History : def rmtab(self, analyse) : del self.opened[analyse['uuid']] - + def update(self, analyse) : if 'matrix_name' in analyse : self.matrixanalyse[analyse['uuid']].update(analyse) @@ -221,7 +222,7 @@ class History : for analyse in anatodel : print 'cleaning :', analyse['name'] self.delete(analyse) - + def __str__(self) : return str(self.history) @@ -229,7 +230,7 @@ class DoConf : def __init__(self, configfile=None, diff = None, parametres = None) : self.configfile = configfile self.conf = ConfigParser() - + if configfile is not None : configfile = normpath_win32(configfile) self.conf.readfp(codecs.open(configfile, 'r', 'utf8')) @@ -263,7 +264,7 @@ class DoConf : if 'type' not in parametres : parametres['type'] = section return parametres - + def makeoptions(self, sections, parametres, outfile = None) : txt = '' for i, section in enumerate(sections) : @@ -330,7 +331,7 @@ class BugDialog(wx.Dialog): # begin wxGlade: MyDialog.__set_properties self.SetMinSize(wx.Size(500, 200)) self.text_ctrl_1.SetMinSize(wx.Size(500, 200)) - + # end wxGlade def __do_layout(self): @@ -688,19 +689,20 @@ def check_Rresult(parent, pid) : else : return True + +def launchcommand(mycommand): + Popen(mycommand) + def print_liste(filename,liste): with open(filename,'w') as f : for graph in liste : - f.write(';'.join(graph)+'\n') + f.write(';'.join(graph).encode(sys.getdefaultencoding())+'\n') def read_list_file(filename, encoding = sys.getdefaultencoding()): with codecs.open(filename,'rU', encoding) as f : content=f.readlines() ncontent=[line.replace('\n','').split(';') for line in content if line.strip() != ''] return ncontent - - - def progressbar(self, maxi) : ira = wx.GetApp().GetTopWindow() @@ -716,7 +718,7 @@ def progressbar(self, maxi) : style=wx.PD_APP_MODAL | wx.PD_AUTO_HIDE | wx.PD_ELAPSED_TIME | wx.PD_CAN_ABORT ) prog.SetSize((400,150)) - prog.SetIcon(ira._icon) + #prog.SetIcon(ira._icon) return prog def treat_var_mod(variables) : @@ -727,7 +729,7 @@ def treat_var_mod(variables) : for var in vars : mods = ['_'.join(v) for v in varmod if v[0] == var] var_mod[var] = mods - + # for variable in variables : # if u'_' in variable : # forme = variable.split(u'_') @@ -761,9 +763,170 @@ def doconcorde(corpus, uces, mots, uci = False) : ucis_txt.append('

' + ' '.join(corpus.ucis[uce].etoiles) + '

') ucestxt.append(ucetxt) return ucis_txt, ucestxt - + def getallstcarac(corpus, analyse) : pathout = PathOut(analyse['ira']) profils = ReadProfileAsDico(pathout['PROFILE_OUT'], Alceste, self.encoding) print profils + +def read_chd(filein, fileout): + with open(filein, 'r') as f : + content = f.read() + #content = [line[3:].replace('"',"").replace(' ','') for line in content.splitlines()] + content = [line.split('\t') for line in content.splitlines()] + chd = {'name':1, 'children':[]} + mere={} + for i, line in enumerate(content) : + if i == 0 : + chd['children'] = [{'name': line[1],'size' : content[i+1][0]}, {'name':line[2], 'size': content[i+1][1]}] + mere[line[1]] = chd['children'][0] + mere[line[2]] = chd['children'][1] + elif not i % 2 : + if 'children' in mere[line[0]]: + mere[line[0]]['children'].append({'name': line[1],'size' : content[i+1][0]}) + mere[line[1]] = mere[line[0]]['children'][-1] + mere[line[0]]['children'].append({'name': line[2],'size' : content[i+1][1]}) + mere[line[2]] = mere[line[0]]['children'][-1] + else : + mere[line[0]]['children'] = [{'name': line[1],'size' : content[i+1][0]}, {'name':line[2], 'size': content[i+1][1]}] + mere[line[1]] = mere[line[0]]['children'][-2] + mere[line[2]] = mere[line[0]]['children'][-1] + with open(fileout, 'w') as f : + f.write(json.dumps(chd)) + + +translation_languages = {"Afrikaans":"af", "Albanian":"sq", "Amharic":"am", "Arabic":"ar", "Armenian":"hy", "Azeerbaijani":"az", "Basque":"eu", "Belarusian":"be", "Bengali":"bn", "Bosnian":"bs", "Bulgarian":"bg", "Catalan":"ca", "Cebuano":"ceb", "Chichewa":"ny", "Chinese (Simplified)":"zh-CN", "Chinese (Traditional)":"zh-TW", "Corsican":"co", "Croatian":"hr", "Czech":"cs", "Danish":"da", "Dutch":"nl", "English":"en", "Esperanto":"eo", "Estonian":"et", "Filipino":"tl", "Finnish":"fi", "French":"fr", "Frisian":"fy", "Galician":"gl", "Georgian":"ka", "German":"de", "Greek":"el", "Gujarati":"gu", "Haitian Creole":"ht", "Hausa":"ha", "Hawaiian":"haw", "Hebrew":"iw", "Hindi":"hi", "Hmong":"hmn ", "Hungarian":"hu", "Icelandic":"is", "Igbo":"ig", "Indonesian":"id", "Irish":"ga", "Italian":"it", "Japanese":"ja", "Javanese":"jw", "Kannada":"kn", "Kazakh":"kk", "Khmer":"km", "Korean":"ko", "Kurdish":"ku", "Kyrgyz":"ky", "Lao":"lo", "Latin":"la", "Latvian":"lv", "Lithuanian":"lt", "Luxembourgish":"lb", "Macedonian":"mk", "Malagasy":"mg", "Malay":"ms", "Malayalam":"ml", "Maltese":"mt", "Maori":"mi", "Marathi":"mr", "Mongolian":"mn", "Burmese":"my", "Nepali":"ne", "Norwegian":"no", "Pashto":"ps", "Persian":"fa", "Polish":"pl", "Portuguese":"pt", "Punjabi":"ma", "Romanian":"ro", "Russian":"ru", "Samoan":"sm", "Scots Gaelic":"gd", "Serbian":"sr", "Sesotho":"st", "Shona":"sn", "Sindhi":"sd", "Sinhala":"si", "Slovak":"sk", "Slovenian":"sl", "Somali":"so", "Spanish":"es", "Sundanese":"su", "Swahili":"sw", "Swedish":"sv", "Tajik":"tg", "Tamil":"ta", "Telugu":"te", "Thai":"th", "Turkish":"tr", "Ukrainian":"uk", "Urdu":"ur", "Uzbek":"uz", "Vietnamese":"vi", "Welsh":"cy", "Xhosa":"xh", "Yiddish":"yi", "Yoruba":"yo", "Zulu":"zu", } + + +def gettranslation(words, lf, lt) : + import urllib2 + import json + agent = {'User-Agent': + "Mozilla/4.0 (\ + compatible;\ + MSIE 6.0;\ + Windows NT 5.1;\ + SV1;\ + .NET CLR 1.1.4322;\ + .NET CLR 2.0.50727;\ + .NET CLR 3.0.04506.30\ + )"} + base_link = "https://translate.googleapis.com/translate_a/single?client=gtx&sl=%s&tl=%s&dt=t&q=%s" + print len(words) + totrans = urllib2.quote('\n'.join(words).encode('utf8')) + link = base_link % (lf, lt, totrans) + request = urllib2.Request(link, headers=agent) + raw_data = urllib2.urlopen(request).read() + data = json.loads(raw_data) + return [line[0].decode('utf8').replace(u"'", u'_').replace(u' | ', u'|').replace(u' ', u'_').replace(u'-',u'_').replace(u'\n','') for line in data[0]] + +def makenprof(prof, trans, deb=0) : + nprof=[] + if deb == 0 : + nprof.append(prof[0]) + for i, val in enumerate(trans) : + line = prof[deb+i+1][:] + line[6] = val + nprof.append(line) + return nprof + +def treatempty(val) : + if val.strip() == '' : + return '_' + else : + return val + +def translateprofile(corpus, dictprofile, lf='it', lt='fr') : + nprof = {} + lems = {} + for i in range(len(dictprofile)) : + prof = dictprofile[`i+1`] + try : + lenact = prof.index([u'*****', u'*', u'*', u'*', u'*', u'*', '', '']) + lensup = -1 + except ValueError: + try : + lenact = prof.index([u'*', u'*', u'*', u'*', u'*', u'*', '', '']) + lensup = 0 + except ValueError: + lenact = len(prof) + lensup = 0 + try : + lensup += prof.index([u'*', u'*', u'*', u'*', u'*', u'*', '', '']) + lensup = lensup - lenact + except ValueError: + lensup += len(prof) - lenact + if lenact != 0 : + if lenact > 400 : + nlenact = 400 + else : + nlenact = lenact + actori = [line[6] for line in prof[1:nlenact]] + act = [val.replace(u'_', u' ') for val in actori] + act = gettranslation(act, lf, lt) + for j, val in enumerate(actori) : + if act[j] not in lems : + lems[act[j]] = val + else : + while act[j] in lems : + act[j] = act[j] + u"+" + lems[act[j]] = val + nprof[`i+1`] = makenprof(prof, act) + + if lensup != 0 : + if lensup > 400 : + nlensup = 400 + else : + nlensup = lensup + supori = [line[6] for line in prof[(1+lenact):(lenact+nlensup)]] + sup = [val.replace(u'_', u' ') for val in supori] + sup = [treatempty(val) for val in sup] + sup = gettranslation(sup, lf, lt) + for j, val in enumerate(supori) : + if sup[j] not in lems : + lems[sup[j]] = val + else : + while sup[j] in lems : + sup[j] = sup[j] + u"+" + lems[sup[j]] = val + nprof[`i+1`].append([u'*****', u'*', u'*', u'*', u'*', u'*', '', '']) + nprof[`i+1`] += makenprof(prof, sup, deb=lenact) + + try : + lenet = prof.index([u'*', u'*', u'*', u'*', u'*', u'*', '', '']) + nprof[`i+1`].append([u'*', u'*', u'*', u'*', u'*', u'*', '', '']) + nprof[`i+1`] += prof[(lenet+1):] + except : + pass + return nprof, lems + + +def write_translation_profile(prof, lems, language, dictpathout) : + if os.path.exists(dictpathout['translations.txt']) : + with codecs.open(dictpathout['translations.txt'], 'r', 'utf8') as f : + translist = f.read() + translist = [line.split('\t') for line in translist.splitlines()] + else : + translist = [] + toprint = [] + toprint.append(['','','','','','']) + toprint.append([u'***', u'nb classes', `len(prof)`, u'***', '', '']) + for i in range(len(prof)) : + toprint.append([u'**', u'classe', `i+1`, u'**', '', '']) + toprint.append([u'****'] + prof[`i+1`][0] + [u'****']) + rest = [[`line[1]`, `line[2]`, `line[3]`, `line[4]`, line[6], line[7].replace('< 0,0001', '0.00009').replace('NS (','').replace(')','')] for line in prof[`i+1`][1:]] + for i, line in enumerate(prof[`i+1`][1:]) : + if line[0] == u'*' : + rest[i] = [u'*', u'*', u'*', u'*', u'*', u'*'] + elif line[0] == u'*****' : + rest[i] = [u'*****',u'*',u'*', u'*', u'*', u'*'] + toprint += rest + with open(dictpathout['translation_profile_%s.csv' % language], 'w') as f : + f.write('\n'.join([';'.join(line) for line in toprint]).encode('utf8')) + with open(dictpathout['translation_words_%s.csv' % language], 'w') as f : + f.write('\n'.join(['\t'.join([val, lems[val]]) for val in lems]).encode('utf8')) + if 'translation_profile_%s.csv' % language not in [val[0] for val in translist] : + translist.append(['translation_profile_%s.csv' % language, 'translation_words_%s.csv' % language]) + with open(dictpathout['translations.txt'], 'w') as f : + f.write('\n'.join(['\t'.join(line) for line in translist]).encode('utf8'))