X-Git-Url: http://iramuteq.org/git?p=iramuteq;a=blobdiff_plain;f=parse_factiva_xml.py;h=e127ae33138932157c6507041457db9d11cad4c9;hp=8d9b8a7778b160400055f33f1a25847cb0f100b3;hb=5249c446f605b92824c1b12fe390563450d90f3f;hpb=773e317db20590ffaa926c9e7f51dbcdc64377f2 diff --git a/parse_factiva_xml.py b/parse_factiva_xml.py index 8d9b8a7..e127ae3 100644 --- a/parse_factiva_xml.py +++ b/parse_factiva_xml.py @@ -2,28 +2,27 @@ # -*- coding: utf-8 -*- #Author: Pierre Ratinaud #Copyright (c) 2008-2010 Pierre Ratinaud -#Lisense: GNU/GPL +#License: GNU/GPL import xml.dom.minidom -import wx.lib.sized_controls as sc import wx.lib.filebrowsebutton as filebrowse import os import codecs -import sys import re import wx from parse_factiva_mail import ParseFactivaMail from parse_factiva_txt import ParseFactivaPaste +from parse_europress import ParseEuropress +from import_txm import TXM2IRA +from functions import BugReport def ParseDocument(filename) : - print filename with codecs.open(filename, 'r', 'utf-8') as f : - content = f.read() + content = f.read() content = content.replace('', ' ').replace('', ' ') dom = xml.dom.minidom.parseString(content.encode("utf-8")) result = [] articles = dom.getElementsByTagName("article") - i = 0 for article in articles : headline = article.getElementsByTagName("headline") if headline != [] : @@ -49,7 +48,7 @@ def ParseDocument(filename) : if sourceName != [] : val_sourceName = sourceName[0].firstChild.nodeValue.replace('\n', ' ') else : - val_sourceName = INCONNU + val_sourceName = 'INCONNU' tailParagraphs = article.getElementsByTagName("tailParagraphs") if tailParagraphs != [] : para_tailParagraphs = tailParagraphs[0].getElementsByTagName("paragraph") @@ -64,13 +63,12 @@ def ParseDocument(filename) : def getcorpus_from_xml(xmldir, corpus_out): files = os.listdir(xmldir) - files = [os.path.join(xmldir,file) for file in files if os.path.splitext(file)[1] == '.xml'] + files = [os.path.join(xmldir,f) for f in files if os.path.splitext(f)[1] == '.xml'] if len(files) == 0 : return 'nofile' - result = [] fileout = codecs.open(corpus_out, 'w', 'utf-8') - for file in files : - rs = ParseDocument(file) + for f in files : + rs = ParseDocument(f) #dates = [row[2].split('-') for row in rs] #dates = [[date[0],date[1],date[2].split('T')[0]] for date in dates] #txt = '\n'.join(['\n'.join([' '.join([u'****', '*%s' % row[1].replace(' ','_').replace('\'','_'), '*%s' % row[2].replace('-','_')]), row[3], row[4]]) for row in rs]) @@ -86,15 +84,23 @@ class PrefImport(wx.Dialog): pre.SetExtraStyle(wx.DIALOG_EX_CONTEXTHELP) pre.Create(parent, -1, '', pos, size, style) self.PostCreate(pre) - if methode == 'xml' : - txt = _(u'Select a directory of xml files') + self.methode = methode + if methode in ['xml', 'txm'] : + txt = _(u'Select a directory of xml files').decode('utf8') + elif methode == 'euro' : + txt = _(u'Select a directory of html files').decode('utf8') + elif methode == 'dmi' : + txt = _(u'Select a csv file').decode('utf8') else : - txt = _(u'Select a directory of txt files') + txt = _(u'Select a directory of txt files').decode('utf8') self.parent = parent - self.txt1 = wx.StaticText(self, -1, txt.encode('utf8')) - self.dbb = filebrowse.DirBrowseButton(self, -1, size=(450, -1), changeCallback = self.fbbCallback) + self.txt1 = wx.StaticText(self, -1, txt) + if methode != 'dmi' : + self.dbb = filebrowse.DirBrowseButton(self, -1, size=(450, -1), changeCallback = self.fbbCallback) + else : + self.dbb = filebrowse.FileBrowseButton(self, -1, size=(450, -1), fileMode = 2, changeCallback = self.fbbCallback) self.dbb.SetLabel("") - self.txt2 = wx.StaticText(self, -1, _(u'Output file').encode('utf8')) + self.txt2 = wx.StaticText(self, -1, _(u'Output file').decode('utf8')) self.fbb = filebrowse.FileBrowseButton(self, -1, size=(450, -1), fileMode = 2) self.fbb.SetLabel("") @@ -133,12 +139,22 @@ class PrefImport(wx.Dialog): def fbbCallback(self, evt): if self.fbb.GetValue() == "" : - self.fbb.SetValue(os.path.join(self.dbb.GetValue(), 'corpus.txt')) + if self.methode != 'dmi' : + self.fbb.SetValue(os.path.join(self.dbb.GetValue(), 'corpus.txt')) + else : + self.fbb.SetValue(os.path.join(os.path.dirname(self.dbb.GetValue()), 'corpus.txt')) #self.log.write('FileBrowseButton: %s\n' % evt.GetString()) def checkfile(self, evt) : if evt.GetId() == wx.ID_OK : if self.dbb.GetValue() != "" : + if self.methode == 'dmi' : + if not os.path.exists(self.dbb.GetValue()) : + dlg = wx.MessageDialog(self, + ' : '.join([self.dbb.GetValue(), _(u"this file doesn't exist")]), 'ATTENTION', wx.NO | wx.YES | wx.ICON_WARNING) + dlg.CenterOnParent() + if dlg.ShowModal() not in [wx.ID_NO, wx.ID_CANCEL]: + self.EndModal(wx.ID_OK) if os.path.exists(self.fbb.GetValue()): dlg = wx.MessageDialog(self, u"%s\nCe fichier existe, continuer quand même ?" % self.fbb.GetValue(), 'ATTENTION', wx.NO | wx.YES | wx.ICON_WARNING) @@ -155,26 +171,48 @@ class PrefImport(wx.Dialog): else : self.EndModal(wx.ID_CANCEL) - - class ImportFactiva(): def __init__(self, parent, methode): - self.dial = PrefImport(parent, methode=methode) - self.dial.CenterOnParent() - val = self.dial.ShowModal() - if val == wx.ID_OK : - xmldir = self.dial.dbb.GetValue() - corp_out = self.dial.fbb.GetValue() - if methode == 'xml' : - res = getcorpus_from_xml(xmldir, corp_out) - elif methode == 'mail' : - res = ParseFactivaMail(xmldir, corp_out, 'utf8', parent.syscoding) - elif methode == 'txt' : - res = ParseFactivaPaste(xmldir, corp_out, 'utf8', parent.syscoding) - if res == 'nofile' : - dlg = wx.MessageDialog(parent, u"Pas de fichier \'.xml\' dans %s" % xmldir, 'ATTENTION', wx.OK | wx.NO_DEFAULT | wx.ICON_WARNING) - dlg.CenterOnParent() - dlg.ShowModal() - #else : - # parent.filename = corp_out - # parent.OpenText() + self.dial = PrefImport(parent, methode=methode) + self.dial.CenterOnParent() + val = self.dial.ShowModal() + if val == wx.ID_OK : + xmldir = self.dial.dbb.GetValue() + corp_out = self.dial.fbb.GetValue() + self.dial.Destroy() + busy = wx.BusyInfo(_("Please wait...").decode('utf8')) + wx.SafeYield() + try : + if methode == 'xml' : + res = getcorpus_from_xml(xmldir, corp_out) + elif methode == 'mail' : + res = ParseFactivaMail(xmldir, corp_out, 'utf8', parent.syscoding) + elif methode == 'txt' : + res = ParseFactivaPaste(xmldir, corp_out, 'utf8', parent.syscoding) + elif methode == 'txm' : + res = TXM2IRA(xmldir, corp_out, 'utf8', parent.syscoding) + elif methode == 'euro' : + res = ParseEuropress(xmldir, corp_out, 'utf8', 'utf8') + del busy + if res == 'nofile' : + dlg = wx.MessageDialog(parent, u"Pas de fichiers dans %s" % xmldir, 'ATTENTION', wx.OK | wx.ICON_WARNING) + dlg.CenterOnParent() + dlg.ShowModal() + dlg.Destroy() + else : + msg = '\n'.join([_(u"Corpus created :").decode('utf8'), corp_out, _(u"Do you want to open it in IRaMuTeQ ?").decode('utf8')]) + dlg = wx.MessageDialog(parent, msg, _(u'Information').decode('utf8'), wx.YES_NO | wx.ICON_INFORMATION | wx.STAY_ON_TOP) + dlg.CenterOnParent() + val = dlg.ShowModal() + if val == wx.ID_YES : + dlg.Destroy() + parent.filename = os.path.abspath(corp_out) + parent.OpenText() + else : + dlg.Destroy() + except : + del busy + BugReport(parent) + else : + self.dial.Destroy() +