1 # -*- coding: utf-8 -*-
2 #Author: Pierre Ratinaud
3 #Copyright (c) 2014 Pierre Ratinaud
7 #from BeautifulSoup import BeautifulSoup
10 from HTMLParser import HTMLParser
13 mois = {u'janvier' : '01',
48 return [`year`, month, '%02d' % day]
51 # create a subclass and override the handler methods
52 class MyHTMLParser(HTMLParser):
53 def handle_starttag(self, tag, attrs):
54 #print "Encountered a start tag:", tag
56 if attrs[0][1] == 'DocPublicationName' :
58 self.currentattr = 'DocPublicationName'
59 elif attrs[0][1] == 'DocHeader' :
61 self.currentattr = 'DocHeader'
62 elif attrs[0][1] == 'TitreArticleVisu' :
63 self.outfile.write('\n\n')
64 self.meta.append('\n')
65 self.outfile.write(' '.join(self.meta).encode('utf8', errors='replace'))
68 self.currentattr = 'TitreArticleVisu'
70 self.currentattr = None
71 def handle_endtag(self, tag):
73 #print "Encountered an end tag :", tag
74 def handle_data(self, data):
75 if self.currentattr == 'DocPublicationName' :
76 PublicationName = data.replace(' ', '_').replace('(','').replace(')','').replace('.','').replace('/','').replace("'",'').lower()
77 PublicationName = PublicationName.split(',')[0]
78 self.meta.append(u'*source_' + PublicationName)
79 elif self.currentattr == 'DocHeader' :
82 self.meta += [u'*date_' + '-'.join(date), u'*am_' + '-'.join(date[0:2]), u'*annee_' + date[0]]
83 elif self.currentattr == 'TitreArticleVisu' :
84 if data.startswith(u'©') :
85 self.currentattr = None
87 self.outfile.write(' '.join(data.replace('\n', ' ').split()).encode('utf8', errors='replace') + ' ')
89 def doinit(self, outfile):
90 self.currentattr = None
93 self.outfile = outfile
96 class ParseEuropress :
97 def __init__(self, txtdir, fileout, encodage_in, encodage_out) :
99 for root, subfolders, subfiles in os.walk(txtdir) :
100 nf = [os.path.join(root, f) for f in subfiles if f.split('.')[-1] == 'html']
104 parser = MyHTMLParser()
105 with open(fileout,'w') as outf :
109 with codecs.open(f, 'rU', encodage_in) as infile :
110 content = infile.read()
113 print 'ok', parser.nb, 'articles', ' - total : ', tot
115 #ParseEuropress('/home/pierre/fac/lerass/mariage/press', '/home/pierre/workspace/iramuteq/dev/europress/corpus_out.txt', 'utf8', 'utf8')
116 #print "Encountered some data :", data
117 # instantiate the parser and fed it some HTML
118 #outfile = '/home/pierre/workspace/iramuteq/dev/europress/corpus_out.txt'
119 #parser = MyHTMLParser()
120 #with open(outfile, 'w') as f :