1 # -*- coding: utf-8 -*-
2 #Author: Pierre Ratinaud
3 #Copyright (c) 2010 Pierre Ratinaud
18 # Removes HTML or XML character references and entities from a text string.
20 # @param text The HTML (or XML) source text.
21 # @return The plain text, as a Unicode string, if necessary.
25 #apos is not in the dictionnary
26 htmlentitydefs.name2codepoint['apos'] = ord("'")
32 return unichr(int(text[3:-1], 16))
34 return unichr(int(text[2:-1]))
39 text = unichr(htmlentitydefs.name2codepoint[text[1:-1]])
42 return text # leave as is
43 return re.sub("&#?\w+;", fixup, text)
45 def UpdateDico(Dico, word, line):
48 Dico[word][1].append(line)
50 Dico[word] = [1, [line]]
53 def __init__(self, parent, filename = '', filetype = 'csv', encodage = 'utf-8') :
55 self.parametre = {'filename' : filename}
56 self.parametre['filetype'] = filetype
57 self.parametre['encodage'] = encodage
58 self.parametre['mineff'] = 3
59 self.parametre['syscoding'] = sys.getdefaultencoding()
62 self.listactives = None
67 self.firstrowiscolnames = True
69 self.firstcolisrownames = True
75 def read_tableau(self, fileout) :
76 d=shelve.open(fileout)
77 self.parametre = d['parametre']
78 if 'syscoding' not in self.parametre :
79 self.parametre['syscoding'] = sys.getdefaultencoding()
80 self.actives = d['actives']
82 self.classes = d['classes']
83 self.listactives = d['listactives']
85 self.listet = d['listet']
86 if 'selected_col' in d :
87 self.selected_col = d['selected_col']
89 self.datas = d['datas']
94 def save_tableau(self, fileout) :
95 d=shelve.open(fileout)
96 d['parametre'] = self.parametre
97 d['actives'] = self.actives
99 d['classes'] = self.classes
100 d['listactives'] = self.listactives
101 if 'listet' in dir(self) :
102 d['listet'] = self.listet
103 if 'selected_col' in dir(self) :
104 d['selected_col'] = self.selected_col
105 if 'datas' in dir(self) :
106 d['datas'] = self.datas
107 if 'lchi' in dir(self) :
108 d['lchi'] = self.lchi
111 def make_content(self) :
112 if self.parametre['filetype'] == 'csv' :
114 elif self.parametre['filetype'] == 'xls' :
116 elif self.parametre['filetype'] == 'ods' :
118 self.parametre['csvfile'] = tempfile.mktemp(dir=self.parent.TEMPDIR)
123 #print '############## ENCODING IN EXCEL #######################'
124 #datafile = xlrd.open_workbook(self.parametre['filename'], encoding_override="azerazerazer")
125 datafile = xlrd.open_workbook(self.parametre['filename'])
126 datatable = datafile.sheet_by_index(self.parametre['sheetnb']-1)
127 self.linecontent = [[str(datatable.cell_value(rowx = i, colx = j)) for j in range(datatable.ncols)] for i in range(datatable.nrows)]
130 doc = ooolib.Calc(opendoc=self.parametre['filename'])
131 doc.set_sheet_index(0)
132 (cols, rows) = doc.get_sheet_dimensions()
134 for row in range(1, rows + 1):
136 for col in range(1, cols + 1):
137 data = doc.get_cell_value(col, row)
138 if data is not None :
139 ligne.append(unescape(data[1]))
142 self.linecontent.append(ligne)
145 with codecs.open(self.parametre['filename'], 'r', self.parametre['encodage']) as f :
147 self.linecontent = [line.replace('"','').split(self.parametre['colsep']) for line in content.splitlines()]
149 def write_csvfile(self) :
150 with open(self.parametre['csvfile'], 'w') as f :
151 f.write('\n'.join([';'.join(line) for line in self.csvtable]))
153 def make_tmpfile(self) :
154 self.rownb = len(self.linecontent)
155 self.colnb = len(self.linecontent[0])
156 if self.firstrowiscolnames :
157 self.colnames = self.linecontent[0]
158 self.linecontent.pop(0)
161 self.colnames = ['_'.join([u'colonne', `i`]) for i in range(self.colnb)]
162 if self.firstcolisrownames :
163 self.rownames = [row[0] for row in self.linecontent]
164 self.linecontent = [row[1:] for row in self.linecontent]
166 self.idname = self.colnames[0]
168 self.check_rownames()
170 self.rownames = [`i` for i in range(self.rownb)]
171 self.idname = u'identifiant'
172 self.csvtable = [[self.idname] + self.colnames] + [[self.rownames[i]] + self.linecontent[i] for i in range(len(self.rownames))]
176 self.parent.content = self.csvtable
177 self.parent.ShowMenu(_("View"))
178 self.parent.ShowMenu(_("Spreadsheet analysis"))
179 self.parent.ShowMenu(_("Text analysis"), False)
180 self.parent.type = "Data"
181 self.parent.DataPop = False
182 self.parent.OnViewData('')
184 def check_rownames(self) :
185 if len(self.rownames) == len(list(set(self.rownames))) :
186 print u'row names ok'
188 print u'les noms de lignes ne sont pas uniques, ils sont remplaces'
189 self.rownames = [`i` for i in range(self.rownb)]
191 def make_unique_list(self) :
192 return list(set([val for line in self.linecontent for val in line if val.strip() != '']))
194 def make_dico(self, linecontent) :
196 for i, line in enumerate(linecontent) :
198 if forme.strip() != '' :
199 UpdateDico(dico, forme, i)
202 def select_col(self, listcol) :
203 ArTable = array(self.linecontent)
204 selcol = ArTable[:, listcol]
205 selcol = selcol.tolist()
208 def write01(self, fileout, dico, linecontent) :
209 self.listactives = [val for val in dico if val != 'NA' and dico[val] >= self.parametre['mineff']]
210 out = [['0' for forme in self.listactives] for line in linecontent]
211 for i, forme in enumerate(self.listactives) :
212 for line in dico[forme][1] :
214 #out = [[self.rownames[i]] + out[i] for i in range(len(linecontent))]
215 #out.insert(0,[self.idname] + self.listactives)
216 out.insert(0, self.listactives)
217 with open(fileout, 'w') as f :
218 f.write('\n'.join([';'.join(line) for line in out]))
220 def make_01_from_selection(self, listact, listsup = None, dowrite = True) :
221 selcol = self.select_col(listact)
222 self.actives = self.make_dico(selcol)
223 if 'Act01' in self.dictpathout and dowrite:
224 self.write01(self.dictpathout['Act01'], self.actives, selcol)
225 elif 'mat01' in self.dictpathout and dowrite:
226 self.write01(self.dictpathout['mat01'], self.actives, selcol)
227 if listsup is not None :
228 selcol = self.select_col(listsup)
229 self.sups = self.make_dico(selcol)
231 def make_01_alc_format(self, fileout) :
232 for i, ligne in enumerate(self.linecontent) :
236 UpdateDico(self.sups, forme, i)
238 UpdateDico(self.actives, forme, i)
239 self.listactives = [val for val in self.actives if self.actives[val][0] >= self.parametre['mineff']]
240 table = [['0' for i in range(len(self.listactives))] for j in range(self.rownb)]
241 for i, val in enumerate(self.listactives) :
242 for j, line in enumerate(self.linecontent) :
245 #table = [[self.rownames[i]] + table[i] for i in range(len(self.rownames))]
246 #table.insert(0, [self.idname] + self.listactives)
247 table.insert(0, self.listactives)
248 with open(fileout, 'w') as f:
249 f.write('\n'.join([';'.join(line) for line in table]))
251 def printtable(self, filename, Table):
252 with open(filename, 'w') as f :
253 f.write('\n'.join([';'.join(line) for line in Table]))
255 def buildprofil(self) :
256 with open(self.dictpathout['uce'], 'rU') as filein :
257 content = filein.readlines()
261 for i, line in enumerate(content) :
262 line = line.replace('\n', '').replace('"', '').split(';')
263 UpdateDico(dicocl, line[1], i)
264 lsucecl.append([int(line[0]) - 1, int(line[1])])
265 self.classes = lsucecl
266 nlist = [[nbuce, cl] for nbuce, cl in lsucecl if cl != 0]
267 self.ucecla = len(nlist)
269 self.clnb = len(dicocl) - 1
271 self.clnb = len(dicocl)
274 for active in self.listactives :
276 line0 = [0] * self.clnb
278 for i in range(0, self.clnb) :
279 for uce, cl in nlist:
281 if active in self.linecontent[uce]:
283 if sum(line[1:]) > self.parametre['mineff']:
284 tablecont.append([line[0]] + [`don` for don in line if type(don) == type(1)])
287 for sup in self.sups :
289 line0 = [0] * self.clnb
291 for i in range(0, self.clnb) :
292 for uce, cl in nlist:
294 if sup in self.linecontent[uce]:
296 tablecontet.append([line[0]] + [`don` for don in line if type(don) == type(1)])
298 self.printtable(self.dictpathout['ContEtOut'], tablecontet)
299 self.printtable(self.dictpathout['Contout'], tablecont)
301 def get_colnames(self) :
302 return self.colnames[:]
304 def make_table_from_classe(self, cl, la) :
305 ln = [line[0] for line in self.classes if line[1] == cl]
306 out = [['0' for col in la] for line in ln]
308 for i, act in enumerate(la) :
309 for j, line in enumerate(ln) :
310 if line in self.actives[act][1] :
312 out.insert(0,[act for act in la])
317 #filename = 'corpus/cent3.csv'
318 #filename = 'corpus/agir2sortie.csv'
319 #tab = Tableau('',filename, encodage='utf-8')
320 #tab.parametre['csvfile'] = tab.parametre['filename']
321 #tab.parametre['sep'] = '\t'
322 #tab.firstrowiscolnames = True
323 #tab.firstcolisrownames = False
325 #tab.make_01('corpus/matrice01.csv')