1 # -*- coding: utf-8 -*-
2 #Author: Pierre Ratinaud
3 #Copyright (c) 2010 Pierre Ratinaud
15 from uuid import uuid4
18 log = logging.getLogger('iramuteq.tableau')
21 # Removes HTML or XML character references and entities from a text string.
23 # @param text The HTML (or XML) source text.
24 # @return The plain text, as a Unicode string, if necessary.
28 #apos is not in the dictionnary
29 htmlentitydefs.name2codepoint['apos'] = ord("'")
35 return unichr(int(text[3:-1], 16))
37 return unichr(int(text[2:-1]))
42 text = unichr(htmlentitydefs.name2codepoint[text[1:-1]])
45 return text # leave as is
46 return re.sub("&#?\w+;", fixup, text)
48 def UpdateDico(Dico, word, line):
51 Dico[word][1].append(line)
53 Dico[word] = [1, [line]]
56 def __init__(self, parent, filename = '', filetype = 'csv', encodage = 'utf-8') :
58 self.parametre = {'filename' : filename}
59 self.parametre['filetype'] = filetype
60 self.parametre['encodage'] = encodage
61 self.parametre['pathout'] = os.path.dirname(os.path.abspath(filename))
62 self.parametre['mineff'] = 3
63 self.parametre['syscoding'] = sys.getdefaultencoding()
64 self.parametre['type'] = 'matrix'
65 self.parametre['name'] = 'unNOm'
68 self.listactives = None
73 self.firstrowiscolnames = True
75 self.firstcolisrownames = True
80 self.parametres = self.parametre
82 def read_tableau(self, fileout) :
83 d=shelve.open(fileout)
84 self.parametre = d['parametre']
85 if 'syscoding' not in self.parametre :
86 self.parametre['syscoding'] = sys.getdefaultencoding()
87 self.actives = d['actives']
89 self.classes = d['classes']
90 self.listactives = d['listactives']
92 self.listet = d['listet']
93 if 'selected_col' in d :
94 self.selected_col = d['selected_col']
96 self.datas = d['datas']
101 def save_tableau(self, fileout) :
102 d=shelve.open(fileout)
103 d['parametre'] = self.parametre
104 d['actives'] = self.actives
105 d['sups'] = self.sups
106 d['classes'] = self.classes
107 d['listactives'] = self.listactives
108 if 'listet' in dir(self) :
109 d['listet'] = self.listet
110 if 'selected_col' in dir(self) :
111 d['selected_col'] = self.selected_col
112 if 'datas' in dir(self) :
113 d['datas'] = self.datas
114 if 'lchi' in dir(self) :
115 d['lchi'] = self.lchi
118 def make_content(self) :
119 if self.parametre['filetype'] == 'csv' :
121 elif self.parametre['filetype'] == 'xls' :
123 elif self.parametre['filetype'] == 'ods' :
125 self.parametre['csvfile'] = tempfile.mktemp(dir=self.parent.TEMPDIR)
130 #print '############## ENCODING IN EXCEL #######################'
131 #datafile = xlrd.open_workbook(self.parametre['filename'], encoding_override="azerazerazer")
132 datafile = xlrd.open_workbook(self.parametre['filename'])
133 datatable = datafile.sheet_by_index(self.parametre['sheetnb']-1)
134 self.linecontent = [[str(datatable.cell_value(rowx = i, colx = j)) for j in range(datatable.ncols)] for i in range(datatable.nrows)]
137 doc = ooolib.Calc(opendoc=self.parametre['filename'])
138 doc.set_sheet_index(0)
139 (cols, rows) = doc.get_sheet_dimensions()
141 for row in range(1, rows + 1):
143 for col in range(1, cols + 1):
144 data = doc.get_cell_value(col, row)
145 if data is not None :
146 ligne.append(unescape(data[1]))
149 self.linecontent.append(ligne)
152 with codecs.open(self.parametre['filename'], 'r', self.parametre['encodage']) as f :
154 self.linecontent = [line.replace('"','').split(self.parametre['colsep']) for line in content.splitlines()]
156 def write_csvfile(self) :
157 with open(self.parametre['csvfile'], 'w') as f :
158 f.write('\n'.join([';'.join(line) for line in self.csvtable]))
160 def make_tmpfile(self) :
161 self.rownb = len(self.linecontent)
162 self.colnb = len(self.linecontent[0])
163 if self.firstrowiscolnames :
164 self.colnames = self.linecontent[0]
165 self.linecontent.pop(0)
168 self.colnames = ['_'.join([u'colonne', `i`]) for i in range(self.colnb)]
169 if self.firstcolisrownames :
170 self.rownames = [row[0] for row in self.linecontent]
171 self.linecontent = [row[1:] for row in self.linecontent]
173 self.idname = self.colnames[0]
175 self.check_rownames()
177 self.rownames = [`i` for i in range(self.rownb)]
178 self.idname = u'identifiant'
179 self.csvtable = [[self.idname] + self.colnames] + [[self.rownames[i]] + self.linecontent[i] for i in range(len(self.rownames))]
183 self.parent.content = self.csvtable
184 self.parent.ShowMenu(_("View"))
185 self.parent.ShowMenu(_("Spreadsheet analysis"))
186 self.parent.ShowMenu(_("Text analysis"), False)
187 self.parent.type = "Data"
188 self.parent.DataPop = False
189 self.parent.OnViewData('')
191 def check_rownames(self) :
192 if len(self.rownames) == len(list(set(self.rownames))) :
193 print u'row names ok'
195 print u'les noms de lignes ne sont pas uniques, ils sont remplaces'
196 self.rownames = [`i` for i in range(self.rownb)]
198 def make_unique_list(self) :
199 return list(set([val for line in self.linecontent for val in line if val.strip() != '']))
201 def make_dico(self, linecontent) :
203 for i, line in enumerate(linecontent) :
205 if forme.strip() != '' :
206 UpdateDico(dico, forme, i)
209 def select_col(self, listcol) :
210 dc = dict(zip(listcol, listcol))
211 selcol = [[val for i, val in enumerate(row) if i in dc] for row in self.linecontent]
214 def write01(self, fileout, dico, linecontent) :
215 self.listactives = [val for val in dico if val != 'NA' and dico[val] >= self.parametre['mineff']]
216 out = [['0' for forme in self.listactives] for line in linecontent]
217 for i, forme in enumerate(self.listactives) :
218 for line in dico[forme][1] :
220 #out = [[self.rownames[i]] + out[i] for i in range(len(linecontent))]
221 #out.insert(0,[self.idname] + self.listactives)
222 out.insert(0, self.listactives)
223 with open(fileout, 'w') as f :
224 f.write('\n'.join([';'.join(line) for line in out]))
226 def make_01_from_selection(self, listact, listsup = None, dowrite = True) :
227 selcol = self.select_col(listact)
228 self.actives = self.make_dico(selcol)
229 self.write01(self.dictpathout['mat01'], self.actives, selcol)
230 if listsup is not None :
231 selcol = self.select_col(listsup)
232 self.sups = self.make_dico(selcol)
234 def make_01_alc_format(self, fileout) :
235 for i, ligne in enumerate(self.linecontent) :
239 UpdateDico(self.sups, forme, i)
241 UpdateDico(self.actives, forme, i)
242 self.listactives = [val for val in self.actives if self.actives[val][0] >= self.parametre['mineff']]
243 table = [['0' for i in range(len(self.listactives))] for j in range(self.rownb)]
244 for i, val in enumerate(self.listactives) :
245 for j, line in enumerate(self.linecontent) :
248 #table = [[self.rownames[i]] + table[i] for i in range(len(self.rownames))]
249 #table.insert(0, [self.idname] + self.listactives)
250 table.insert(0, self.listactives)
251 with open(fileout, 'w') as f:
252 f.write('\n'.join([';'.join(line) for line in table]))
254 def printtable(self, filename, Table):
255 with open(filename, 'w') as f :
256 f.write('\n'.join([';'.join(line) for line in Table]))
258 def buildprofil(self) :
259 with open(self.dictpathout['uce'], 'rU') as filein :
260 content = filein.readlines()
264 for i, line in enumerate(content) :
265 line = line.replace('\n', '').replace('"', '').split(';')
266 UpdateDico(dicocl, line[1], i)
267 lsucecl.append([int(line[0]) - 1, int(line[1])])
268 self.classes = lsucecl
269 nlist = [[nbuce, cl] for nbuce, cl in lsucecl if cl != 0]
270 self.ucecla = len(nlist)
272 self.clnb = len(dicocl) - 1
274 self.clnb = len(dicocl)
277 for active in self.listactives :
279 line0 = [0] * self.clnb
281 for i in range(0, self.clnb) :
282 for uce, cl in nlist:
284 if active in self.linecontent[uce]:
286 if sum(line[1:]) > self.parametre['mineff']:
287 tablecont.append([line[0]] + [`don` for don in line if type(don) == type(1)])
290 for sup in self.sups :
292 line0 = [0] * self.clnb
294 for i in range(0, self.clnb) :
295 for uce, cl in nlist:
297 if sup in self.linecontent[uce]:
299 tablecontet.append([line[0]] + [`don` for don in line if type(don) == type(1)])
301 self.printtable(self.dictpathout['ContEtOut'], tablecontet)
302 self.printtable(self.dictpathout['Contout'], tablecont)
304 def get_colnames(self) :
305 return self.colnames[:]
307 def make_table_from_classe(self, cl, la) :
308 ln = [line[0] for line in self.classes if line[1] == cl]
309 out = [['0' for col in la] for line in ln]
311 for i, act in enumerate(la) :
312 for j, line in enumerate(ln) :
313 if line in self.actives[act][1] :
315 out.insert(0,[act for act in la])
320 #filename = 'corpus/cent3.csv'
321 #filename = 'corpus/agir2sortie.csv'
322 #tab = Tableau('',filename, encodage='utf-8')
323 #tab.parametre['csvfile'] = tab.parametre['filename']
324 #tab.parametre['sep'] = '\t'
325 #tab.firstrowiscolnames = True
326 #tab.firstcolisrownames = False
328 #tab.make_01('corpus/matrice01.csv')