2 # -*- coding: utf-8 -*-
3 #Author: Pierre Ratinaud
4 #Copyright (c) 2012-2013 Pierre Ratinaud
12 #txtdir = 'dev/factiva_txt'
13 #fileout = 'dev/factiva_txt_out.txt'
15 #encodage_out = 'utf8'
18 def parsetxtpaste(txt):
20 parser de texte pour factiva
21 à partir d'un copier/coller de la fenêtre de visualisation
22 merci à Lucie Loubère pour l'astuce :)
24 no = ['NS','RE','IPD','CO','IN'] # les balises qui signalent une fin
25 txt = txt.splitlines()
29 if line.startswith('Article') :
32 if lp[2] == 'Article' :
33 ucis.append([[u'****'],''])
35 if line.startswith('SN ') : #source
36 jsource = re.sub('[^A-Za-z0-9]', '', line[4:])
37 source = u'_'.join([u'*source', jsource]).lower()
38 #source = '*source_' + line[4:].replace(' ','').replace('\'','').replace(u'´','').replace(u'’','').replace('-','').lower()
39 ucis[-1][0].append(source)
40 elif line.startswith('PD ') : #date
41 mois_annee = '*ma_' + line[4:].split(' ')[1] + line[4:].split(' ')[2]
42 ucis[-1][0].append(mois_annee)
43 annee = u'*annee_' + line[4:].split(' ')[2]
44 ucis[-1][0].append(annee)
45 elif line.strip() in no : #fin
47 elif line.startswith('RF ') : #fin
49 elif line.strip() in ['LP', 'TD'] : #debut texte
53 if keepline and line.strip() not in ['LP', 'TD', ''] :
54 ucis[-1][1] = '\n'.join([ucis[-1][1],line])
58 def print_ucis(ucis, ofile, encodage) :
59 #elimination des articles vides
60 ucis = [uci for uci in ucis if uci[1].strip() != '']
61 toprint = '\n\n'.join(['\n'.join([' '.join(uci[0]),uci[1]]) for uci in ucis])
62 ofile.write(toprint.encode(encodage) + '\n')
64 class ParseFactivaPaste :
65 def __init__(self, txtdir, fileout, encodage_in, encodage_out) :
66 files = os.listdir(txtdir)
68 with open(fileout,'w') as outf :
71 f = os.path.join(txtdir, f)
72 with codecs.open(f, 'rU', encodage_in) as infile :
73 content = infile.read()
74 ucis = parsetxtpaste(content)
75 print_ucis(ucis, outf, encodage_out)
77 print 'ok', len(ucis), 'articles', ' - total : ', tot
79 #for dat in ['2001','2002','2003','2004', '2005','2006','2007','2008','2009','2010','2011'] :
80 # path = os.path.join(txtdir,dat)
81 # outfile = os.path.join(txtdir, 'corpus_' + dat + '.txt')
82 # doparse(path, outfile)
85 if __name__ == '__main__' :
86 doparse(txtdir, fileout, encodage_in, encodage_out)