iron-blogger/iron-blogger
1
1
Fork 0
Code Issues Pull requests Releases Wiki Activity

Merge branch 'python3' of iron-blogger/iron-blogger into master

Browse source 
Python 2 is no longer able to connect to modern https-Servers.

Also it’s outdated. ;-)
This commit is contained in:
Thomas Renger 2019-03-23 11:10:23 +00:00 committed by Gitea
parent 8901fc53b3 9ff394df38
commit 6666300c93
7 changed files with 57 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
import-feeds.py
View file

@ -1,44 +1,49 @@
#!/usr/bin/python
#!/usr/bin/python3
from lxml import html
import yaml
import sys
import urllib2
import urlparse
import urllib.request
import urllib.parse
with open('bloggers.yml') as f:
users = yaml.safe_load(f.read())
def fetch_links(url):
print >>sys.stderr, "Looking for feeds in %s" % (url,)
tree = html.fromstring(urllib2.urlopen(url).read())
links = tree.xpath(
'//link[@rel="alternate"][contains(@type, "rss") or ' +
'contains(@type, "atom") or contains(@type, "rdf")]')
candidates = [l for l in links if
print("Looking for feeds in %s" % (url,), file=sys.stderr)
try:
tree = html.document_fromstring(urllib.request.urlopen(url).read())
links = tree.xpath(
'//link[@rel="alternate"][contains(@type, "rss") or ' +
'contains(@type, "atom") or contains(@type, "rdf")]')
candidates = [l for l in links if
'atom' in l.attrib['type'] and
'comments' not in l.attrib['href'].lower() and
'comments' not in l.attrib.get('title','')]
except:
candidates = []
links = []
if candidates:
return candidates[0].attrib['href']
elif links:
return links[0].attrib['href']
else:
print >>sys.stderr, "No link found for %s" % (url,)
print("No link found for %s" % (url,), file=sys.stderr)
return None
for (name, u) in users.items():
for (name, u) in list(users.items()):
print("Processing user %s" % (name,), file=sys.stderr)
for e in u['links']:
(title, url) = e[1:3]
try:
e[1] = e[1].strip()
except:
except:
e[1] = e[1]
if len(e) == 4:
continue
link = fetch_links(url)
if link:
if not link.startswith('http:'):
link = urlparse.urljoin(url, link)
link = urllib.parse.urljoin(url, link)
e.append(link)
with open('bloggers.yml', 'w') as f:

27
render.py
View file

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/python3
# This Python file uses the following encoding: utf-8
import yaml
from dateutil.parser import parse
@ -29,6 +29,7 @@ def get_balance(acct):
def get_debts():
p = subprocess.Popen(['ledger', '-f', os.path.join(HERE, 'ledger'),
'-n', 'balance', 'Pool:Owed:'],
universal_newlines=True,
stdout=subprocess.PIPE)
(out, _) = p.communicate()
debts = []
@ -48,7 +49,7 @@ def parse_skip(rec):
out = []
for s in spec:
if isinstance(s, list):
out.append(map(to_week_num, s))
out.append(list(map(to_week_num, s)))
else:
out.append(to_week_num(s))
return out
@ -72,7 +73,7 @@ def render_template(path, week=None, **kwargs):
else:
week = START
week = (week - START).days / 7
week = int( (week - START).days / 7)
week_start = START + (week * datetime.timedelta(7))
week_end = START + ((week + 1) * datetime.timedelta(7))
@ -84,7 +85,7 @@ def render_template(path, week=None, **kwargs):
class User(object):
pass
for (un, rec) in users.items():
for (un, rec) in list(users.items()):
u = User()
u.username = un
u.name = rec['name']
@ -97,8 +98,8 @@ def render_template(path, week=None, **kwargs):
u.stop = rec.get('stop')
u.skip = parse_skip(rec)
u.posts = report.get(un, {})
u.goodblogs = []
u.lameblogs = []
u.goodblogs = []
u.lameblogs = []
userlist.append(u)
@ -121,13 +122,13 @@ def render_template(path, week=None, **kwargs):
continue
if should_skip(u.skip, week):
skipped_users.append(u)
continue
continue
elif user_start > week_start:
skip.append(u)
continue
for blog in u.links:
b=blog[0]
weeks=u.posts[b]
continue
for blog in u.links:
b=blog[0]
weeks=u.posts[b]
if len(weeks) <= week or not weeks[week]:
u.lameblogs.append(b)
else:
@ -144,10 +145,10 @@ def render_template(path, week=None, **kwargs):
if __name__ == '__main__':
if len(sys.argv) < 2:
print >>sys.stderr, "Usage: %s TEMPLATE [WEEK]"
print("Usage: %s TEMPLATE [WEEK]", file=sys.stderr)
sys.exit(1)
template = sys.argv[1]
week = None
if len(sys.argv) > 2: week = sys.argv[2]
print render_template(template, week)
print(render_template(template, week))

18
scan-feeds.py
View file

@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/python3
import yaml
import feedparser
import datetime
@ -37,26 +37,26 @@ def get_link(post):
return post.link
def parse_feeds(weeks, uri):
feedparser.USER_AGENT = "IronBloggerBot/0.1 +http://ironblogger.de/"
feedparser.USER_AGENT = "IronBloggerBot/0.2 +http://ironblogger.de/"
feed = feedparser.parse(uri)
if not feed.entries:
print >>sys.stderr, "WARN: no entries for ", uri
print("WARN: no entries for ", uri, file=sys.stderr)
for post in feed.entries:
date = parse_published(get_date(post))
if date < START:
continue
wn = (date - START).days / 7
wn = int ( (date - START).days / 7 )
while len(weeks) <= wn:
weeks.append([])
if post.has_key('title'):
if 'title' in post:
post = dict(date=date,
title=post.title,
url=get_link(post))
if not post.has_key('title'):
if 'title' not in post:
post = dict(date=date,
title="",
url=get_link(post))
@ -67,13 +67,13 @@ if len(sys.argv) > 1:
for username in sys.argv[1:]:
blogs = log.setdefault(username, {})
for l in users[username]['links']:
weeks = blogs.setdefault(l[0], [])
weeks = blogs.setdefault(l[0], [])
parse_feeds(weeks, l[3])
else:
for (username, u) in users.items():
for (username, u) in list(users.items()):
blogs = log.setdefault(username, {})
for l in u['links']:
weeks = blogs.setdefault(l[0], [])
weeks = blogs.setdefault(l[0], [])
parse_feeds(weeks, l[3])
with open('out/report.yml', 'w') as f:

4
settings.py
View file

@ -1,8 +1,8 @@
#!/usr/bin/python
import ConfigParser, os
import configparser, os
def load_settings():
configfile = ConfigParser.ConfigParser()
configfile = configparser.ConfigParser()
configfile.read('settings.cfg')
config=dict()
config['mail']=configfile.get("general","mail")

2
templates/email.txt
View file

@ -53,7 +53,7 @@ Paid: € ${paid}
Events: € ${event}
Individual debts:
% for (u, v) in sorted(debts, key=lambda p:p[1], reverse=True):
${u"%20s %d \u20AC" % (u, v)}
${"%20s %d \u20AC" % (u, v)}
% endfor
PREVIOUSLY PUNTED (pay € 30 balance to return):

6
update-participants.py
View file

@ -1,15 +1,15 @@
#!/usr/bin/python
#!/usr/bin/python3
import render
import os
import sys
import xmlrpclib
import xmlrpc.client
import subprocess
import settings
config=settings.load_settings()
x = xmlrpclib.ServerProxy(config['xmlrpc_endpoint'])
x = xmlrpc.client.ServerProxy(config['xmlrpc_endpoint'])
page = x.wp.getPage(config['blog_id'], config['participants_page_id'], config['username'], config['password'])
text = render.render_template('templates/users.tmpl')

19
weekly-update.py
View file

@ -1,9 +1,9 @@
#!/usr/bin/python
#!/usr/bin/python3
# This Python file uses the following encoding: utf-8
import render
import os
import sys
import xmlrpclib
import xmlrpc.client
import subprocess
import datetime
import yaml
@ -22,7 +22,7 @@ if len(args)>0:
if args[0] == '-q':
dry_run = True
quick_view = True
send_mail = False
send_mail = False
args = args[1:]
if args[0] == '-r':
@ -32,14 +32,15 @@ if len(args)>0:
if args[0] == '-n':
dry_run = True
send_mail = False
send_mail = False
args = args[1:]
date = args[0]
with open('ledger', 'a') as f:
f.write("\n")
f.write(render.render_template('templates/ledger', date))
# print(render.render_template('templates/ledger', date).decode("utf-8"))
f.write(render.render_template('templates/ledger', date).decode("utf-8"))
if not dry_run:
subprocess.check_call(["git", "commit", "ledger",
@ -61,7 +62,7 @@ with open('ledger', 'a') as f:
if not dry_run:
text = render.render_template('templates/week.tmpl', date, punt=punt)
text = render.render_template('templates/week.tmpl', date, punt=punt).decode("utf-8")
lines = text.split("\n")
title = lines[0]
@ -69,16 +70,16 @@ if not dry_run:
page = dict(title = title, description = body)
x = xmlrpclib.ServerProxy(config['xmlrpc_endpoint'])
x = xmlrpc.client.ServerProxy(config['xmlrpc_endpoint'])
x.metaWeblog.newPost(config['blog_id'], config['username'], config['password'], page, True)
if not reminder:
email = render.render_template('templates/email.txt', date, punt=punt,mail=config['mail'])
else:
email = render.render_template('templates/reminder.txt', date, punt=punt,mail=config['mail'])
if quick_view:
print(render.render_template('templates/quick_view.tmpl',date,punt=punt))
print((render.render_template('templates/quick_view.tmpl',date,punt=punt)))
if dry_run and not quick_view:
print email
print(email)
if send_mail:
# p = subprocess.Popen(['mutt', '-H', '/dev/stdin'],
p = subprocess.Popen(['/usr/sbin/sendmail', '-oi', '-t'],