1 # -*- coding: utf-8 -*-
2 #Author: Pierre Ratinaud
3 #Copyright (c) 2008-2020 Pierre Ratinaud
4 #modification pour python 3 : Laurent Mérat, 6x7 - mai 2020
7 #------------------------------------
8 # import des modules python
9 #------------------------------------
10 import xml.dom.minidom
15 #------------------------------------
16 # import des modules wx
17 #------------------------------------
19 import wx.lib.filebrowsebutton as filebrowse
21 #------------------------------------
22 # import des fichiers du projet
23 #------------------------------------
24 from parse_factiva_mail import ParseFactivaMail
25 from parse_factiva_txt import ParseFactivaPaste
26 from parse_europress import ParseEuropress
27 from import_txm import TXM2IRA
28 from functions import BugReport
31 def ParseDocument(filename) :
32 with codecs.open(filename, 'r', 'utf-8') as f :
34 content = content.replace('<hlt>', ' ').replace('</hlt>', ' ')
35 dom = xml.dom.minidom.parseString(content)
37 articles = dom.getElementsByTagName("article")
38 for article in articles :
39 headline = article.getElementsByTagName("headline")
41 para_headline = headline[0].getElementsByTagName("paragraph")
42 val_headline = [val.firstChild.nodeValue.replace('\n', ' ') for val in para_headline]
45 leadParagraph = article.getElementsByTagName("leadParagraph")
46 if leadParagraph != [] :
47 para_leadParagraph = leadParagraph[0].getElementsByTagName("paragraph")
48 val_leadParagraph = [val.firstChild.nodeValue.replace('\n', ' ') for val in para_leadParagraph]
50 val_leadParagraph = []
51 publicationDate = article.getElementsByTagName("publicationDate")
52 if publicationDate != [] :
53 para_publicationDate = publicationDate[0].getElementsByTagName("date")
54 if para_publicationDate == [] :
55 para_publicationDate = publicationDate[0].getElementsByTagName("dateTime")
56 val_publicationDate = [val.firstChild.nodeValue.replace('\n', ' ') for val in para_publicationDate]
58 val_publicationDate = []
59 sourceName = article.getElementsByTagName("sourceName")
61 val_sourceName = sourceName[0].firstChild.nodeValue.replace('\n', ' ')
63 val_sourceName = 'INCONNU'
64 tailParagraphs = article.getElementsByTagName("tailParagraphs")
65 if tailParagraphs != [] :
66 para_tailParagraphs = tailParagraphs[0].getElementsByTagName("paragraph")
67 val_tailParagraphs = [val.firstChild.nodeValue.replace('\n', ' ') for val in para_tailParagraphs]
69 val_tailParagraphs = []
70 inter = [' '.join(val_headline), val_sourceName,' '.join(val_publicationDate), ' '.join(val_leadParagraph), ' '.join(val_tailParagraphs)]
71 inter = [re.sub(r'[ "\n\r]+', ' ', val).replace('"',' ').replace('\n', ' ').replace('\r', ' ') for val in inter]
72 #inter = ['"' + val +'"' for val in inter]
76 def getcorpus_from_xml(xmldir, corpus_out):
77 files = os.listdir(xmldir)
78 files = [os.path.join(xmldir,f) for f in files if os.path.splitext(f)[1] == '.xml']
81 fileout = codecs.open(corpus_out, 'w', 'utf-8')
84 #dates = [row[2].split('-') for row in rs]
85 #dates = [[date[0],date[1],date[2].split('T')[0]] for date in dates]
86 #txt = '\n'.join(['\n'.join([' '.join([u'****', '*%s' % row[1].replace(' ','_').replace('\'','_'), '*%s' % row[2].replace('-','_')]), row[3], row[4]]) for row in rs])
87 #avec la date decompose
88 txt = '\n'.join(['\n'.join([' '.join(['****', '*s_%s' % row[1].replace(' ','').replace('\'',''), '*annee_%s' % row[2].split('-')[0], '*mois_%s' % row[2].split('-')[1], '*jour_%s' % row[2].split('-')[2].split('T')[0]]), row[3], row[4]]) for row in rs])
89 fileout.write(txt+'\n\n')
94 class PrefImport(wx.Dialog):
96 def __init__(self, parent, size=wx.DefaultSize, pos=wx.DefaultPosition, style=wx.DEFAULT_DIALOG_STYLE, methode = 'mail'):
97 wx.Dialog.__init__(self) # 1
98 self.SetExtraStyle(wx.DIALOG_EX_CONTEXTHELP) # 2
99 self.Create(parent, -1, '') # 3
100 self.methode = methode
101 if methode in ['xml', 'txm'] :
102 txt = _('Select a directory of xml files')
103 elif methode == 'euro' :
104 txt = _('Select a directory of html files')
105 elif methode == 'dmi' :
106 txt = _('Select a csv file')
108 txt = _('Select a directory of txt files')
110 self.txt1 = wx.StaticText(self, -1, txt)
111 if methode != 'dmi' :
112 self.dbb = filebrowse.DirBrowseButton(self, -1, size=(450, -1), changeCallback = self.fbbCallback)
114 self.dbb = filebrowse.FileBrowseButton(self, -1, size=(450, -1), fileMode = 2, changeCallback = self.fbbCallback)
115 self.dbb.SetLabel("")
116 self.txt2 = wx.StaticText(self, -1, _('Output file'))
117 self.fbb = filebrowse.FileBrowseButton(self, -1, size=(450, -1), fileMode = 2)
118 self.fbb.SetLabel("")
119 self.btnsizer = wx.StdDialogButtonSizer()
120 btn_ok = wx.Button(self, wx.ID_OK)
121 btn = wx.Button(self, wx.ID_CANCEL)
122 self.btnsizer.AddButton(btn_ok)
123 self.btnsizer.AddButton(btn)
124 self.btnsizer.Realize()
125 self.Bind(wx.EVT_BUTTON, self.checkfile, btn_ok)
126 #self.SetButtonSizer(self.CreateStdDialogButtonSizer(wx.OK | wx.CANCEL))
127 self.Bind(wx.EVT_BUTTON, self.checkfile)
130 self.SetMinSize(self.GetSize())
132 def __do_layout(self):
133 sizer = wx.BoxSizer(wx.VERTICAL)
134 grid_sizer_1 = wx.BoxSizer(wx.HORIZONTAL)
135 grid_sizer_2 = wx.BoxSizer(wx.HORIZONTAL)
136 grid_sizer_1.Add(self.txt1, 0, wx.ALIGN_LEFT | wx.ALIGN_CENTER_VERTICAL, 0)
137 grid_sizer_1.Add(self.dbb, 2, wx.ALIGN_LEFT | wx.ALIGN_CENTER_VERTICAL, 0)
138 grid_sizer_2.Add(self.txt2, 0, wx.ALIGN_LEFT | wx.ALIGN_CENTER_VERTICAL, 0)
139 grid_sizer_2.Add(self.fbb, 2, wx.ALIGN_LEFT | wx.ALIGN_CENTER_VERTICAL, 0)
140 sizer.Add(grid_sizer_1, 0, wx.EXPAND, 0)
141 sizer.Add(grid_sizer_2, 0, wx.EXPAND, 0)
142 sizer.Add(self.btnsizer, 0, wx.EXPAND, 0)
147 def fbbCallback(self, evt):
148 if self.fbb.GetValue() == "" :
149 if self.methode != 'dmi' :
150 self.fbb.SetValue(os.path.join(self.dbb.GetValue(), 'corpus.txt'))
152 self.fbb.SetValue(os.path.join(os.path.dirname(self.dbb.GetValue()), 'corpus.txt'))
153 #self.log.write('FileBrowseButton: %s\n' % evt.GetString())
155 def checkfile(self, evt) :
156 if evt.GetId() == wx.ID_OK :
157 if self.dbb.GetValue() != "" :
158 if self.methode == 'dmi' :
159 if not os.path.exists(self.dbb.GetValue()) :
160 dlg = wx.MessageDialog(self,
161 ' : '.join([self.dbb.GetValue(), _("this file doesn't exist")]), 'ATTENTION', wx.NO | wx.YES | wx.ICON_WARNING)
163 if dlg.ShowModal() not in [wx.ID_NO, wx.ID_CANCEL]:
164 self.EndModal(wx.ID_OK)
165 if os.path.exists(self.fbb.GetValue()):
166 dlg = wx.MessageDialog(self,
167 "%s\nCe fichier existe, continuer quand même ?" % self.fbb.GetValue(), 'ATTENTION', wx.NO | wx.YES | wx.ICON_WARNING)
169 if dlg.ShowModal() not in [wx.ID_NO, wx.ID_CANCEL]:
170 self.EndModal(wx.ID_OK)
172 self.EndModal(wx.ID_OK)
174 dlg = wx.MessageDialog(self, "Vous devez choisir le répertoire contenant le ou les fichier(s) xml", 'ATTENTION', wx.OK | wx.ICON_WARNING)
178 self.EndModal(wx.ID_CANCEL)
181 class ImportFactiva():
183 def __init__(self, parent, methode):
184 self.dial = PrefImport(parent, methode=methode)
185 self.dial.CenterOnParent()
186 val = self.dial.ShowModal()
188 xmldir = self.dial.dbb.GetValue()
189 corp_out = self.dial.fbb.GetValue()
191 busy = wx.BusyInfo(_("Please wait..."))
194 if methode == 'xml' :
195 res = getcorpus_from_xml(xmldir, corp_out)
196 elif methode == 'mail' :
197 res = ParseFactivaMail(xmldir, corp_out, 'utf8', parent.syscoding)
198 elif methode == 'txt' :
199 res = ParseFactivaPaste(xmldir, corp_out, 'utf8', parent.syscoding)
200 elif methode == 'txm' :
201 res = TXM2IRA(xmldir, corp_out, 'utf8', parent.syscoding)
202 elif methode == 'euro' :
203 res = ParseEuropress(xmldir, corp_out, 'utf8', 'utf8')
206 dlg = wx.MessageDialog(parent, "Pas de fichiers dans %s" % xmldir, 'ATTENTION', wx.OK | wx.ICON_WARNING)
211 msg = '\n'.join([_("Corpus created :"), corp_out, _("Do you want to open it in IRaMuTeQ ?")])
212 dlg = wx.MessageDialog(parent, msg, _('Information'), wx.YES_NO | wx.ICON_INFORMATION | wx.STAY_ON_TOP)
214 val = dlg.ShowModal()
215 if val == wx.ID_YES :
217 parent.filename = os.path.abspath(corp_out)