diff --git a/import-feeds.py b/import-feeds.py index d6c2379..34e1ab8 100755 --- a/import-feeds.py +++ b/import-feeds.py @@ -1,44 +1,49 @@ -#!/usr/bin/python +#!/usr/bin/python3 from lxml import html import yaml import sys -import urllib2 -import urlparse +import urllib.request +import urllib.parse with open('bloggers.yml') as f: users = yaml.safe_load(f.read()) def fetch_links(url): - print >>sys.stderr, "Looking for feeds in %s" % (url,) - tree = html.fromstring(urllib2.urlopen(url).read()) - links = tree.xpath( - '//link[@rel="alternate"][contains(@type, "rss") or ' + - 'contains(@type, "atom") or contains(@type, "rdf")]') - candidates = [l for l in links if + print("Looking for feeds in %s" % (url,), file=sys.stderr) + try: + tree = html.document_fromstring(urllib.request.urlopen(url).read()) + links = tree.xpath( + '//link[@rel="alternate"][contains(@type, "rss") or ' + + 'contains(@type, "atom") or contains(@type, "rdf")]') + candidates = [l for l in links if 'atom' in l.attrib['type'] and 'comments' not in l.attrib['href'].lower() and 'comments' not in l.attrib.get('title','')] + except: + candidates = [] + links = [] if candidates: return candidates[0].attrib['href'] elif links: return links[0].attrib['href'] else: - print >>sys.stderr, "No link found for %s" % (url,) + print("No link found for %s" % (url,), file=sys.stderr) return None -for (name, u) in users.items(): +for (name, u) in list(users.items()): + print("Processing user %s" % (name,), file=sys.stderr) for e in u['links']: (title, url) = e[1:3] try: e[1] = e[1].strip() - except: + except: e[1] = e[1] if len(e) == 4: continue link = fetch_links(url) if link: if not link.startswith('http:'): - link = urlparse.urljoin(url, link) + link = urllib.parse.urljoin(url, link) e.append(link) with open('bloggers.yml', 'w') as f: diff --git a/render.py b/render.py index f371ccf..81dd186 100755 --- a/render.py +++ b/render.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # This Python file uses the following encoding: utf-8 import yaml from dateutil.parser import parse @@ -29,6 +29,7 @@ def get_balance(acct): def get_debts(): p = subprocess.Popen(['ledger', '-f', os.path.join(HERE, 'ledger'), '-n', 'balance', 'Pool:Owed:'], + universal_newlines=True, stdout=subprocess.PIPE) (out, _) = p.communicate() debts = [] @@ -48,7 +49,7 @@ def parse_skip(rec): out = [] for s in spec: if isinstance(s, list): - out.append(map(to_week_num, s)) + out.append(list(map(to_week_num, s))) else: out.append(to_week_num(s)) return out @@ -72,7 +73,7 @@ def render_template(path, week=None, **kwargs): else: week = START - week = (week - START).days / 7 + week = int( (week - START).days / 7) week_start = START + (week * datetime.timedelta(7)) week_end = START + ((week + 1) * datetime.timedelta(7)) @@ -84,7 +85,7 @@ def render_template(path, week=None, **kwargs): class User(object): pass - for (un, rec) in users.items(): + for (un, rec) in list(users.items()): u = User() u.username = un u.name = rec['name'] @@ -97,8 +98,8 @@ def render_template(path, week=None, **kwargs): u.stop = rec.get('stop') u.skip = parse_skip(rec) u.posts = report.get(un, {}) - u.goodblogs = [] - u.lameblogs = [] + u.goodblogs = [] + u.lameblogs = [] userlist.append(u) @@ -121,13 +122,13 @@ def render_template(path, week=None, **kwargs): continue if should_skip(u.skip, week): skipped_users.append(u) - continue + continue elif user_start > week_start: skip.append(u) - continue - for blog in u.links: - b=blog[0] - weeks=u.posts[b] + continue + for blog in u.links: + b=blog[0] + weeks=u.posts[b] if len(weeks) <= week or not weeks[week]: u.lameblogs.append(b) else: @@ -144,10 +145,10 @@ def render_template(path, week=None, **kwargs): if __name__ == '__main__': if len(sys.argv) < 2: - print >>sys.stderr, "Usage: %s TEMPLATE [WEEK]" + print("Usage: %s TEMPLATE [WEEK]", file=sys.stderr) sys.exit(1) template = sys.argv[1] week = None if len(sys.argv) > 2: week = sys.argv[2] - print render_template(template, week) + print(render_template(template, week)) diff --git a/scan-feeds.py b/scan-feeds.py index c9ea057..644ea6d 100755 --- a/scan-feeds.py +++ b/scan-feeds.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import yaml import feedparser import datetime @@ -37,26 +37,26 @@ def get_link(post): return post.link def parse_feeds(weeks, uri): - feedparser.USER_AGENT = "IronBloggerBot/0.1 +http://ironblogger.de/" + feedparser.USER_AGENT = "IronBloggerBot/0.2 +http://ironblogger.de/" feed = feedparser.parse(uri) if not feed.entries: - print >>sys.stderr, "WARN: no entries for ", uri + print("WARN: no entries for ", uri, file=sys.stderr) for post in feed.entries: date = parse_published(get_date(post)) if date < START: continue - wn = (date - START).days / 7 + wn = int ( (date - START).days / 7 ) while len(weeks) <= wn: weeks.append([]) - if post.has_key('title'): + if 'title' in post: post = dict(date=date, title=post.title, url=get_link(post)) - if not post.has_key('title'): + if 'title' not in post: post = dict(date=date, title="", url=get_link(post)) @@ -67,13 +67,13 @@ if len(sys.argv) > 1: for username in sys.argv[1:]: blogs = log.setdefault(username, {}) for l in users[username]['links']: - weeks = blogs.setdefault(l[0], []) + weeks = blogs.setdefault(l[0], []) parse_feeds(weeks, l[3]) else: - for (username, u) in users.items(): + for (username, u) in list(users.items()): blogs = log.setdefault(username, {}) for l in u['links']: - weeks = blogs.setdefault(l[0], []) + weeks = blogs.setdefault(l[0], []) parse_feeds(weeks, l[3]) with open('out/report.yml', 'w') as f: diff --git a/settings.py b/settings.py index 5ceda41..3d1abaf 100755 --- a/settings.py +++ b/settings.py @@ -1,8 +1,8 @@ #!/usr/bin/python -import ConfigParser, os +import configparser, os def load_settings(): - configfile = ConfigParser.ConfigParser() + configfile = configparser.ConfigParser() configfile.read('settings.cfg') config=dict() config['mail']=configfile.get("general","mail") diff --git a/update-participants.py b/update-participants.py index bdcb012..d020a5d 100755 --- a/update-participants.py +++ b/update-participants.py @@ -1,15 +1,15 @@ -#!/usr/bin/python +#!/usr/bin/python3 import render import os import sys -import xmlrpclib +import xmlrpc.client import subprocess import settings config=settings.load_settings() -x = xmlrpclib.ServerProxy(config['xmlrpc_endpoint']) +x = xmlrpc.client.ServerProxy(config['xmlrpc_endpoint']) page = x.wp.getPage(config['blog_id'], config['participants_page_id'], config['username'], config['password']) text = render.render_template('templates/users.tmpl') diff --git a/weekly-update.py b/weekly-update.py index 18bb1eb..934ec9a 100755 --- a/weekly-update.py +++ b/weekly-update.py @@ -1,9 +1,9 @@ -#!/usr/bin/python +#!/usr/bin/python3 # This Python file uses the following encoding: utf-8 import render import os import sys -import xmlrpclib +import xmlrpc.client import subprocess import datetime import yaml @@ -22,7 +22,7 @@ if len(args)>0: if args[0] == '-q': dry_run = True quick_view = True - send_mail = False + send_mail = False args = args[1:] if args[0] == '-r': @@ -32,14 +32,15 @@ if len(args)>0: if args[0] == '-n': dry_run = True - send_mail = False + send_mail = False args = args[1:] date = args[0] with open('ledger', 'a') as f: f.write("\n") - f.write(render.render_template('templates/ledger', date)) + # print(render.render_template('templates/ledger', date).decode("utf-8")) + f.write(render.render_template('templates/ledger', date).decode("utf-8")) if not dry_run: subprocess.check_call(["git", "commit", "ledger", @@ -61,7 +62,7 @@ with open('ledger', 'a') as f: if not dry_run: - text = render.render_template('templates/week.tmpl', date, punt=punt) + text = render.render_template('templates/week.tmpl', date, punt=punt).decode("utf-8") lines = text.split("\n") title = lines[0] @@ -69,16 +70,16 @@ if not dry_run: page = dict(title = title, description = body) - x = xmlrpclib.ServerProxy(config['xmlrpc_endpoint']) + x = xmlrpc.client.ServerProxy(config['xmlrpc_endpoint']) x.metaWeblog.newPost(config['blog_id'], config['username'], config['password'], page, True) if not reminder: email = render.render_template('templates/email.txt', date, punt=punt,mail=config['mail']) else: email = render.render_template('templates/reminder.txt', date, punt=punt,mail=config['mail']) if quick_view: - print(render.render_template('templates/quick_view.tmpl',date,punt=punt)) + print((render.render_template('templates/quick_view.tmpl',date,punt=punt))) if dry_run and not quick_view: - print email + print(email) if send_mail: # p = subprocess.Popen(['mutt', '-H', '/dev/stdin'], p = subprocess.Popen(['/usr/sbin/sendmail', '-oi', '-t'],