Compare commits

..

1 Commits

Author SHA1 Message Date
Thomas Renger 84de914c46 import (outdated) daily reporting script version 2023-12-28 22:27:47 +01:00
17 changed files with 354 additions and 648 deletions

1
.gitignore vendored
View File

@ -2,4 +2,3 @@
*.pyc
settings.cfg
out/
data/

View File

@ -1,9 +0,0 @@
FROM python:3
WORKDIR /usr/src/app
COPY requirements.txt ./
RUN pip install --no-cache-dir -r requirements.txt
COPY . .
CMD [ "python", "./blogbot.py" ]

View File

@ -1,140 +0,0 @@
#!/usr/bin/python3
import yaml
import feedparser
import datetime
import sys
import os
import shutil
import re
from dateutil.parser import parse
import dateutil.tz as tz
import settings
from git import Repo
def parse_published(pub):
try:
return parse(pub).astimezone(tz.tzlocal()).replace(tzinfo=None)
except:
return parse(pub).replace(tzinfo=None)
def get_date(post):
for k in ('published', 'created', 'updated'):
if k in post:
return post[k]
def get_link(post):
return post.link
def get_title(post):
if 'title' in post:
return post.title
else:
return ''
def remove_html_tags(txt):
p = re.compile(r'<[^<]*?/?>')
return p.sub('', txt)
def remove_extra_spaces(txt):
p = re.compile(r'\s+')
return p.sub(' ', txt)
def create_extract(txt):
stxt = remove_extra_spaces(remove_html_tags(txt))
if len(stxt) < 250:
return stxt
if stxt.rfind('. ',200,250)>0:
return stxt[:stxt.rfind('. ',200,250)+1]+" [...]"
if stxt.rfind('! ',200,250)>0:
return stxt[:stxt.rfind('! ',200,250)+1]+" [...]"
if stxt.rfind('? ',200,250)>0:
return stxt[:stxt.rfind('? ',200,250)+1]+" [...]"
if stxt.rfind(', ',200,250)>0:
return stxt[:stxt.rfind(', ',200,250)+1]+" [...]"
if stxt.rfind(' ',200,250)>0:
return stxt[:stxt.rfind(' ',200,250)]+" [...]"
return stxt[:250]+"[...]"
def parse_feeds(weeks, username, blog):
feedparser.USER_AGENT = "IronBloggerBot/0.2 +http://ironblogger.de/"
uri = blog['feed']
print("Retreiving ", uri)
feed = feedparser.parse(uri)
if not feed.entries:
print("WARN: no entries for ", uri, file=sys.stderr)
for post in feed.entries:
date = parse_published(get_date(post))
if date < START:
continue
key = date.strftime("%Y-%m-%d")
weeks.setdefault(key, [])
post = dict(date=date,
title=get_title(post),
url=get_link(post),
username=username,
blogname=blog[0],
description=create_extract(post.description))
if post['url'] not in [p['url'] for p in weeks[key]]:
weeks[key].append(post)
# -- main
config=settings.load_settings()
if os.path.exists('data'):
shutil.rmtree('data')
gitrepo = Repo.clone_from('https://git.wazong.de/iron-blogger/test.git', 'data')
try:
with open('data/blogs.yaml') as f:
users = yaml.safe_load(f.read())
except FileNotFoundError:
users = []
print(users)
if not os.path.exists('data/out'):
os.makedirs('data/out')
try:
with open('data/out/report.yaml') as f:
log = yaml.safe_load(f.read())
except FileNotFoundError:
log = {}
# START = datetime.datetime.strptime(config['start_date'],'%Y/%m/%d')
START = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) - datetime.timedelta(days=7)
if len(sys.argv) > 1:
for username in sys.argv[1:]:
blogs = log.setdefault(username, {})
for l in users[username]['links']:
parse_feeds(log, username, l)
else:
for (username, u) in list(users.items()):
if 'end' in u:
enddate = datetime.datetime.strptime(u['end'],'%Y/%m/%d')
if enddate < datetime.datetime.now():
print("User inactive: ", username)
continue
for l in u['blogs']:
parse_feeds(log, username, l)
with open('data/out/report.yaml', 'w') as f:
yaml.safe_dump(log, f)
gitrepo.index.add(['out/report.yaml'])
with open('data/blogs.yaml', 'w') as f:
yaml.safe_dump(users, f)
gitrepo.index.add(['blogs.yaml'])
print(gitrepo.index.diff(gitrepo.head.commit))
# gitrepo.index.commit('autocommit')
# gitrepo.remotes.origin.push()

View File

@ -1,170 +1,167 @@
BarbaraHoisl:
links:
- [0, barbara's blog, 'http://www.barbarahoisl.com/', 'http://www.barbarahoisl.com/feed/']
mail: barbara@barbarahoisl.com
name: Barbara Hoisl
start: 2012/10/08
ChemieEmma:
links:
- [dent, dent in the world (Englisch), 'http://dentintheworld.com/', 'http://dentintheworld.com/feed/atom/']
- [alpaka, the naked alpaca (Englisch), 'http://thenakedalpaca.com/', 'http://thenakedalpaca.com/feed/atom/']
mail: missp@dentintheworld.com
name: Paula Schramm
start: 2012/09/24
DVDtoday:
links:
- [0, JKdigital, 'http://jkdigital.de/', 'http://jkdigital.de/feed/']
mail: jk@jkplus.de
name: "J\xFCrgen Kaiser"
start: 2012/09/24
twitter: false
Der_Ideealist:
links:
- [0, Der Ideealist, 'http://der-ideealist.de/', 'http://der-ideealist.de/feed/']
mail: ideealist@steinhobelgruen.de
name: Jens Heim
EquilibriumBlog:
links:
- [0, Equilibrium, 'http://www.equilibriumblog.de', 'http://www.equilibriumblog.de/wordpress/feed/atom/']
name: Stefan Rybkowski
FlorianSchweer:
links:
- [0, blorts - wir bloggen sport!, 'http://www.blorts.de/', 'http://www.blorts.de/feed.xml']
mail: florian.schweer@blorts.de
name: Florian Schweer
twitter: false
Gedankenzirkus:
links:
- [0, Gedankenzirkus, 'http://www.gedankenzirkus.de/', 'http://www.gedankenzirkus.de/wordpress/feed/']
name: Oliver Koch
HeimBau:
links:
- [0, Teeren und Federn | HEIM Tiefbau Blog, 'http://blog.heim-tuttlingen.de/',
'http://blog.heim-tuttlingen.de/feed/']
mail: info@heim-tuttlingen.de
name: Jens Heim
HubertMayer:
links:
- [0, Hubert Mayer, 'http://hubert-mayer.de/', 'http://hubert-mayer.de/feed/']
- [testet, Hubert Testet, 'http://hubert-testet.de/', 'http://hubert-testet.de/feed/']
- [321blog, 321 Blog!, 'http://www.321blog.de/', 'http://www.321blog.de/author/dermayer/feed/']
mail: mayerwerbung@gmx.de
name: Hubert Mayer
start: 2012/09/24
TabTwo:
SMCST:
links:
- [0, Nerd Residenz, 'http://www.nerd-residenz.de/blog/', 'http://www.nerd-residenz.de/blog/index.atom']
mail: rmayer@nerd-residenz.de
name: Ralph Mayer
start: 2012/09/24
affiliteur:
- [0, Social Media Club Stuttgart, 'http://www.smcst.de/', 'http://www.smcst.de/feed/']
name: Markus Besch
SocialMediaInst:
links:
- [0, Affiliate auf Partytour, 'http://www.affiliteur.com/', 'http://www.affiliteur.com/feed/atom']
mail: info@affiliteur.com
name: Sascha Schilling
start: 2012/09/24
cmsfunk:
- [0, SocialMedia Institute, 'http://www.socialmedia-institute.com/', 'http://www.socialmedia-institute.com/feed/']
name: Markus Besch
_teecee:
links:
- [0, Stattmarketing, 'http://stattmarketing.wordpress.com/', 'http://stattmarketing.wordpress.com/feed/']
mail: christoph.funk28@gmail.com
name: Christoph Funk
start: 2012/09/24
- [0, Thomas Christinck, 'http://christinck.de/', 'http://christinck.de/feed/']
mail: thomas@christinck.de
name: Thomas Christinck
dentaku:
links:
- [0, Dentaku, 'http://dentaku.wazong.de/', 'http://dentaku.wazong.de/category/blog/feed']
- [1, Dentaku Blog, 'http://dentaku.wazong.de/', 'http://dentaku.wazong.de/feed']
mail: dentaku@wazong.de
name: Thomas Renger
start: 2012/09/24
dirkhaun:
hensch:
links:
- [0, Dirks Hirnableiter, 'http://hirnableiter.tinycities.de/', 'http://hirnableiter.tinycities.de/feeds/hirnableiter.rss']
- [1, The Mobile Presenter (Englisch), 'http://www.themobilepresenter.com/', 'http://www.themobilepresenter.com/feeds/themobilepresenter.rss']
mail: dirk@haun-online.de
name: Dirk Haun
start: 2012/09/24
erikschimmel:
links:
- [0, Erik Schimmel, 'http://erikschimmel.de/', 'http://erikschimmel.de/feed/']
mail: mail@erikschimmel.de
name: Erik Schimmel
start: 2012/10/01
gordongeisler:
links:
- [0, (blog folgt), 'http://staging.wazong.de/', 'http://staging.wazong.de/feed/']
mail: gordon.geisler@freiarbeiter.com
name: Gordon Geisler
start: 2012/12/31
- [0, "Blog von Henning Sch\xFCrig", 'http://www.henningschuerig.de/blog/', 'http://www.henningschuerig.de/blog/feed/']
mail: info@henningschuerig.de
name: "Henning Sch\xFCrig"
hirnrinde:
links:
- [0, Hirnrinde, 'http://www.hirnrinde.de/', 'http://www.hirnrinde.de/hirnrinde.php']
mail: stefan.evertz@gmail.com
name: Stefan Evertz
start: 2012/09/24
hoomygumb:
hubert_testet:
links:
- [0, it's a hoomygumb, 'http://hoomygumb.com/', 'http://hoomygumb.com/feed/']
mail: hoomygumb@gmail.com
name: Jay F. Kay
start: 2012/09/24
idrottning:
links:
- [0, (blog folgt), 'http://staging.wazong.de/', 'http://staging.wazong.de/feed/']
mail: v.frankenberg@gmx.de
name: Viola Frankenberg
start: 2012/12/31
- [testet, Hubert Testet, 'http://hubert-testet.de/', 'http://hubert-testet.de/feed/']
mail: mayerwerbung@gmx.de
name: Hubert Mayer
jantheofel:
links:
- [0, "Jans K\xFCchenleben", 'http://www.theofel.de/plog/', 'http://www.theofel.de/plog/index.xml']
- [1, Jans Technik-Blog, 'http://www.theofel.de/weblog/', 'http://www.theofel.de/weblog/index.xml']
- [2, Schokolade geht immer, 'http://www.schokolade-geht-immer.de/', 'http://www.schokolade-geht-immer.de/feed/atom/']
- [3, Contao-Anleitungen.de, 'http://www.contao-anleitungen.de/', 'http://www.contao-anleitungen.de/blog.xml']
- [4, Brettspiel-Blog, 'http://www.brettspiel-blog.de/', 'http://www.brettspiel-blog.de/atom.xml']
mail: jan@theofel.de
name: Jan Theofel
start: 2012/09/24
lakritzplanet:
kLAWtext:
links:
- [0, Lakritzplanet, 'http://www.lakritzplanet.de/', 'http://www.lakritzplanet.de/feed/']
mail: mail@lakritzplanet.de
name: Christian Kaufmann
start: 2012/09/24
momstagebuch:
- [0, kLAWtext, 'http://klawtext.blogspot.de/', 'http://klawtext.blogspot.com/feeds/posts/default']
name: Sebastian Dosch
ma_y:
links:
- [0, Mom's Tagebuch, 'http://momstagebuch.blogspot.de', 'http://momstagebuch.blogspot.com/feeds/posts/default']
mail: hahn-elke@gmx.de
name: Elke Hahn
start: 2012/10/08
musevg:
- [0, ma.y2, 'http://may2.wordpress.com/', 'http://may2.wordpress.com/feed/']
mail: info@frauschuetze.de
name: "Frau Sch\xFCtze"
mahrko:
links:
- [0, Abenteuer im Musenland, 'http://www.outsourcer.de/', 'http://www.outsourcer.de/feed/']
- [b2s, Brain 2 Shirt, 'http://www.2shirt.org', 'http://www.2shirt.org/feed/atom/']
- [allesso, Alles so, 'http://www.alles.so/', 'http://www.alles.so/feed/']
mail: iron@muse.vg
name: Michael Schommer
start: 2012/09/24
pNachhaltig:
- [0, blog.mahrko.de, 'http://blog.mahrko.de/', 'http://blog.mahrko.de/feed/']
name: Marco Bereth
micialmedia:
links:
- [0, praktisch Nachhaltig, 'http://praktisch-nachhaltig.de/news-und-presse/praktisch-nachhaltig-blog',
'http://staging.wazong.de/feed/']
mail: info@madiko.com
name: "Franziska K\xF6ppe"
start: 2012/12/31
poster4nature:
- [0, Micial Media, 'http://micialmedia.de/', 'http://micialmedia.de/feed/']
mail: info@michael-roth.de
name: Michael M. Roth
miradlo:
links:
- [0, poster4nature, 'http://www.poster4nature.com/category/blog/', 'http://www.poster4nature.com/category/blog/feed/']
mail: partner@poster4nature.com
name: Alexander Beck
start: 2012/09/24
reichepoet:
- [0, miradlo bloggt, 'http://www.miradlo.net/bloggt/', 'http://www.miradlo.net/bloggt/feeds/atom.xml']
- [1, uteles Blog, 'http://www.utele.eu/blog/', 'http://www.utele.eu/blog/feed']
mail: miradlo@steinhobelgruen.de
name: Ute Hauth
nezzform:
links:
- [0, Der Reiche Poet, 'http://reichepoet.blogspot.de/', 'http://reichepoet.blogspot.com/feeds/posts/default']
mail: ute.muendlein@10-o-clock.de
name: "Ute M\xFCndlein"
start: 2012/09/24
- [0, blog.nezzform, 'http://nezzform.blogspot.de/', 'http://nezzform.blogspot.com/feeds/posts/default']
- [1, Mama im Job, 'http://mamaimjob.blogspot.de/', 'http://mamaimjob.blogspot.com/feeds/posts/default']
mail: jastel@gmx.de
name: Jana Akyildiz
oliverg:
links:
- [0, OGOK oliver gassner online-kommunikation, 'http://www.ogok.de/', 'http://www.ogok.de/feeds/posts/default']
- [321blog, 321 Blog!, 'http://www.321blog.de/', 'http://www.321blog.de/author/oliverg/feed/']
mail: oliver@steinhobelgruen.de
name: Oliver Gassner
publizist:
links:
- [0, Publizist - Eigensinn verpflichtet!, 'http://publizist.wordpress.com/', 'http://publizist.wordpress.com/feed/']
name: Immo Sennewald
quilthexle:
links:
- [0, Quilthexle, 'http://www.quilthexle.blogspot.de/', 'http://quilthexle.blogspot.com/feeds/posts/default']
mail: quilthexle@steinhobelgruen.de
name: Frauke Schramm
ralphjschiel:
links:
- [0, das naturblau-blog, 'http://blog.naturblau.de/', 'http://blog.naturblau.de/feed/']
mail: null
name: Ralph J. Schiel
twitter: false
roadkill:
links:
- [0, Hinten beim Bier, 'http://www.hintenbeimbier.de/', 'http://www.hintenbeimbier.de/feed/']
mail: st.sommer@gmail.com
name: Stefan Sommer
start: 2012/09/24
schiri:
links:
- [0, Patrick Schneider, 'http://www.patrickschneider.net', 'http://www.patrickschneider.net/feed/']
mail: info@patrickschneider.net
name: Patrick Schneider
start: 2012/09/24
spacedani:
spaetzle_tweet:
links:
- [0, Danigee, 'http://www.danigee.de/', 'http://www.danigee.de/feed/']
mail: spacedani@gmail.com
name: Danijela Grgic
start: 2012/10/01
sympatexter:
- [0, "Sp\xE4tzle mit So\xDF", 'http://spaetzlemitsoss.de/', 'http://spaetzlemitsoss.de/feed']
mail: info@spaetzlemitsoss.de
name: Christian Mehler
startupstgt:
links:
- [0, Sympatexter, 'http://www.sympatexter.de/', 'http://www.sympatexter.de/feed']
mail: boehm.judith@gmail.com
name: "Judith B\xF6hm"
start: 2012/09/24
werkstatt:
- [0, Startup Stuttgart, 'http://startup-stuttgart.de/', 'http://startup-stuttgart.de/feed/']
mail: burn@startup-stuttgart.de
name: Team Startup Stuttgart
tilohensel:
links:
- [0, Wissenswerkstatt, 'http://www.wissenswerkstatt.net/', 'http://www.wissenswerkstatt.net/feed/']
mail: marc.scheloske@gmail.com
name: Marc Scheloske
start: 2012/09/24
wunschgeburt:
- [0, Tilo Hensel Blog, 'http://www.tilo-hensel.de/', 'http://www.tilo-hensel.de/feed']
mail: tilohensel@steinhobelgruen.de
name: Tilo Hensel
travellerblog:
links:
- [0, Wunschgeburt, 'http://www.wunschgeburt.de/weblog/', 'http://www.wunschgeburt.de/feed/']
mail: frau.kienberger@web.de
name: Mika Kienberger
start: 2012/10/08
- [0, Travellerblog, 'http://travellerblog.eu/', 'http://travellerblog.eu/feed/']
mail: mayerwerbung@gmx.de
name: Hubert Mayer
uliuli:
links:
- [0, dia-blog, 'http://www.dia-blog.de/', 'http://www.dia-blog.de/feed/']
- [1, Nichtsblog, 'http://www.nichtsblog.de/', 'http://www.nichtsblog.de/feed/atom/']
- [2, 'Wer ist dir lieber?', 'http://www.wer-ist-dir-lieber.de/', 'http://www.wer-ist-dir-lieber.de/feed/']
mail: uliuli@steinhobelgruen.de
name: Ulrich Eder

58
daily-update.py Executable file
View File

@ -0,0 +1,58 @@
#!/usr/bin/python
# This Python file uses the following encoding: utf-8
import render
import os
import sys
import xmlrpclib
import subprocess
import datetime
import yaml
import settings
config=settings.load_settings()
dry_run = False
send_mail = True
quick_view = False
args = sys.argv[1:]
if len(args)>0:
if args[0] == '-q':
dry_run = True
quick_view = True
send_mail = False
if args[0] == '-n':
dry_run = True
send_mail = False
START = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) - datetime.timedelta(days=1)
date = START.strftime("%Y-%m-%d")
email = render.render_template('templates/email.txt', date, start=START, mail=config['mail'])
if len(email)==0:
print("No posts.")
exit(0)
if not dry_run:
text = render.render_template('templates/week.tmpl', date, start=START)
lines = text.split("\n")
title = lines[0]
body = "\n".join(lines[1:])
page = dict(title = title, description = body)
x = xmlrpclib.ServerProxy(config['xmlrpc_endpoint'])
x.metaWeblog.newPost(config['blog_id'], config['username'], config['password'], page, True)
if quick_view:
print(render.render_template('templates/quick_view.tmpl',date,start=START))
if dry_run and not quick_view:
print email
if send_mail:
p = subprocess.Popen(['/usr/sbin/sendmail', '-oi', '-t'],
stdin=subprocess.PIPE)
p.communicate(email)

View File

@ -1,49 +1,43 @@
#!/usr/bin/python3
#!/usr/bin/python
from lxml import html
import yaml
import sys
import urllib.request
import urllib.parse
import urllib2
import urlparse
with open('bloggers.yml') as f:
users = yaml.safe_load(f.read())
def fetch_links(url):
print("Looking for feeds in %s" % (url,), file=sys.stderr)
try:
tree = html.document_fromstring(urllib.request.urlopen(url).read())
links = tree.xpath(
'//link[@rel="alternate"][contains(@type, "rss") or ' +
'contains(@type, "atom") or contains(@type, "rdf")]')
candidates = [l for l in links if
tree = html.fromstring(urllib2.urlopen(url).read())
links = tree.xpath(
'//link[@rel="alternate"][contains(@type, "rss") or ' +
'contains(@type, "atom") or contains(@type, "rdf")]')
candidates = [l for l in links if
'atom' in l.attrib['type'] and
'comments' not in l.attrib['href'].lower() and
'comments' not in l.attrib.get('title','')]
except:
candidates = []
links = []
if candidates:
return candidates[0].attrib['href']
elif links:
return links[0].attrib['href']
else:
print("No link found for %s" % (url,), file=sys.stderr)
print >>sys.stderr, "No link found for %s" % (url,)
return None
for (name, u) in list(users.items()):
print("Processing user %s" % (name,), file=sys.stderr)
for (name, u) in users.items():
for e in u['links']:
(title, url) = e[1:3]
try:
e[1] = e[1].strip()
except:
except:
e[1] = e[1]
if len(e) == 4:
continue
link = fetch_links(url)
if link:
if not link.startswith('http:'):
link = urllib.parse.urljoin(url, link)
link = urlparse.urljoin(url, link)
e.append(link)
with open('bloggers.yml', 'w') as f:

11
ledger
View File

@ -1,11 +0,0 @@

130
render.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/python3
#!/usr/bin/python
# This Python file uses the following encoding: utf-8
import yaml
from dateutil.parser import parse
@ -13,142 +13,30 @@ from mako.template import Template
config=settings.load_settings()
START = datetime.datetime.strptime(config['start_date'],"%Y/%m/%d")
HERE = os.path.dirname(__file__)
def get_balance(acct):
p = subprocess.Popen(['ledger', '-f', os.path.join(HERE,'ledger'),
'-n', 'balance', acct],
stdout=subprocess.PIPE)
(out, _) = p.communicate()
try:
return int(out.split()[0][1:])
except:
return 0
def get_debts():
p = subprocess.Popen(['ledger', '-f', os.path.join(HERE, 'ledger'),
'-n', 'balance', 'Pool:Owed:'],
universal_newlines=True,
stdout=subprocess.PIPE)
(out, _) = p.communicate()
debts = []
for line in out.split("\n"):
if not line: continue
(val, acct) = line.split()
user = acct[len("Pool:Owed:"):]
val = int(val[len("$"):])
debts.append((user, val))
return debts
def to_week_num(date):
return (parse(date, default=START) - START).days / 7
def parse_skip(rec):
spec = rec.get('skip', [])
out = []
for s in spec:
if isinstance(s, list):
out.append(list(map(to_week_num, s)))
else:
out.append(to_week_num(s))
return out
def should_skip(skips, week):
for e in skips:
if e == week:
return True
if isinstance(e, list) and e[0] <= week and e[1] > week:
return True
return False
def render_template(path, week=None, **kwargs):
def render_template(path, date, **kwargs):
with open('out/report.yml') as r:
report = yaml.safe_load(r)
with open('bloggers.yml') as f:
users = yaml.safe_load(f)
if week:
week = parse(week, default=START)
else:
week = START
week = int( (week - START).days / 7)
week_start = START + (week * datetime.timedelta(7))
week_end = START + ((week + 1) * datetime.timedelta(7))
posts = report.setdefault(date, [])
skip = []
skipped_users = []
userlist = []
punted = []
class User(object):
pass
for (un, rec) in list(users.items()):
u = User()
u.username = un
u.name = rec['name']
u.mail = rec['mail']
u.links = rec['links']
u.twitter = rec.get('twitter')
u.start_de = datetime.datetime.strptime(rec['start'],"%Y/%m/%d").strftime("%d.%m.%Y")
u.start = rec['start']
u.end = rec.get('end')
u.stop = rec.get('stop')
u.skip = parse_skip(rec)
u.posts = report.get(un, {})
u.goodblogs = []
u.lameblogs = []
userlist.append(u)
# create a subset of punted users
if u.end:
u.end_de = datetime.datetime.strptime(rec.get('end'),"%Y/%m/%d").strftime("%d.%m.%Y")
punted.append(u)
def user_key(u):
return (u.start, u.username)
userlist.sort(key=user_key)
punted.sort(key=user_key)
for u in userlist:
user_start = parse(u.start, default=START)
if u.stop:
continue
if u.end and parse(u.end, default=START) <= week_start:
continue
if should_skip(u.skip, week):
skipped_users.append(u)
continue
elif user_start > week_start:
skip.append(u)
continue
for blog in u.links:
b=blog[0]
weeks=u.posts[b]
if len(weeks) <= week or not weeks[week]:
u.lameblogs.append(b)
else:
u.goodblogs.append(b)
debts = get_debts()
if len(date)>0 and len(posts)==0:
return ""
return Template(filename=path, output_encoding='utf-8').render(
week=week, week_start=week_start,week_end=week_end,
skip=skip, skipped_users=skipped_users, userlist=userlist,
pool=(get_balance('Pool')-get_balance('Event')), paid=get_balance('Pool:Paid'),
event=get_balance('Pool:Event'),
debts=debts, punted=punted, **kwargs)
users=users, posts=posts, date=date,
**kwargs)
if __name__ == '__main__':
if len(sys.argv) < 2:
print("Usage: %s TEMPLATE [WEEK]", file=sys.stderr)
print >>sys.stderr, "Usage: %s TEMPLATE [WEEK]"
sys.exit(1)
template = sys.argv[1]
week = None
if len(sys.argv) > 2: week = sys.argv[2]
print(render_template(template, week))
print render_template(template, week)

View File

@ -1,4 +0,0 @@
pyyaml
feedparser
python-dateutil
GitPython

104
scan-feeds.py Executable file
View File

@ -0,0 +1,104 @@
#!/usr/bin/python
import yaml
import feedparser
import datetime
import sys
import re
import os
from dateutil.parser import parse
import dateutil.tz as tz
import settings
config=settings.load_settings()
with open('bloggers.yml') as f:
users = yaml.safe_load(f.read())
if not os.path.exists('out'):
os.makedirs('out')
try:
with open('out/report.yml') as f:
log = yaml.safe_load(f.read())
except IOError:
log = {}
START = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) - datetime.timedelta(days=7)
def parse_published(pub):
try:
return parse(pub).astimezone(tz.tzlocal()).replace(tzinfo=None)
except:
return parse(pub).replace(tzinfo=None)
def get_date(post):
for k in ('published', 'created', 'updated'):
if k in post:
return post[k]
def get_link(post):
return post.link
def remove_html_tags(txt):
p = re.compile(r'<[^<]*?/?>')
return p.sub('', txt)
def remove_extra_spaces(txt):
p = re.compile(r'\s+')
return p.sub(' ', txt)
def create_extract(txt):
stxt = remove_extra_spaces(remove_html_tags(txt))
if len(stxt) < 250:
return stxt
if stxt.rfind('. ',200,250)>0:
return stxt[:stxt.rfind('. ',200,250)+1]+" [...]"
if stxt.rfind('! ',200,250)>0:
return stxt[:stxt.rfind('! ',200,250)+1]+" [...]"
if stxt.rfind('? ',200,250)>0:
return stxt[:stxt.rfind('? ',200,250)+1]+" [...]"
if stxt.rfind(', ',200,250)>0:
return stxt[:stxt.rfind(', ',200,250)+1]+" [...]"
if stxt.rfind(' ',200,250)>0:
return stxt[:stxt.rfind(' ',200,250)]+" [...]"
return stxt[:250]+"[...]"
def parse_feeds(weeks, username, blog):
uri = blog[3]
# print >>sys.stderr, "Parsing ", uri
feed = feedparser.parse(uri)
if not feed.entries:
print >>sys.stderr, "WARN: no entries for ", uri
for post in feed.entries:
date = parse_published(get_date(post))
if date < START:
continue
key = date.strftime("%Y-%m-%d")
weeks.setdefault(key, [])
if post.has_key('title'):
post = dict(date=date,
title=post.title,
url=get_link(post),
username=username,
blogname=blog[0],
description=create_extract(post.description))
if not post.has_key('title'):
post = dict(date=date,
title="",
url=get_link(post),
username=username,
blogname=blog[0],
description=create_extract(post.description))
if post['url'] not in [p['url'] for p in weeks[key]]:
weeks[key].append(post)
for (username, u) in users.items():
for l in u['links']:
parse_feeds(log, username, l)
with open('out/report.yml', 'w') as f:
yaml.safe_dump(log, f)

View File

@ -1,17 +1,15 @@
#!/usr/bin/python
import configparser, os
import ConfigParser, os
def load_settings():
configfile = configparser.ConfigParser()
configfile = ConfigParser.ConfigParser()
configfile.read('settings.cfg')
config=dict()
config['mail']=configfile.get("general","mail")
config['report_interval']=configfile.get("general","report_interval", fallback="weekly")
config['username']=configfile.get("blogsettings","username")
config['password']=configfile.get("blogsettings","password", fallback="")
config['password']=configfile.get("blogsettings","password")
config['xmlrpc_endpoint']=configfile.get("blogsettings","xmlrpc_endpoint")
config['blog_id']=configfile.get("blogsettings","blog_id", fallback="0")
config['blog_id']=configfile.get("blogsettings","blog_id")
config['participants_page_id']=configfile.get("blogsettings","participants_page_id")
return config

View File

@ -1,62 +1,9 @@
## -*- coding: utf-8 -*-
From: ${mail}
Content-Type: text/plain; charset=utf-8
Subject: IRON BLOGGER results for the week beginning ${week_start.strftime("%F")}
Subject: Blogging BW on ${start.strftime("%F")}
To: ${mail}
SLACKERS:
<% lame=0 %>
% for u in sorted(userlist, key=lambda u:u.name[u.name.rfind(' '):].lower()):
% for b in u.lameblogs:
<% lame+=1 %> - ${u.name} in ${b}
% endfor
% endfor
% if punt:
PUNTED for balance ≥$30: ${", ".join(sorted(punt))}
% endif
People who posted:
% for u in sorted(userlist, key=lambda u:u.name[u.name.rfind(' '):].lower()):
% for b in u.goodblogs:
<% pc=0 %>
${u.name} in ${b} (${u.username}):
% for p in u.posts[b][week]:
% if pc < 10:
<% pc+=1 %> - ${p['url']}
% endif
% endfor
% if len(u.posts[b][week]) > 10:
(and ${len(u.posts[b][week])-10} more)
% endif
% endfor
% endfor
% if skip:
People who have not yet started:
% for u in sorted(skip, key=lambda u:u.username):
${u.username}
% endfor
% endif
% if skipped_users:
People who are currently skipped:
% for u in sorted(skipped_users, key=lambda u:u.username):
${u.username}
% endfor
% endif
Beer pool:
This Week: € ${5 * lame}
Total: € ${pool}
Paid: € ${paid}
Events: € ${event}
Individual debts:
% for (u, v) in sorted(debts, key=lambda p:p[1], reverse=True):
${"%20s %d \u20AC" % (u, v)}
% endfor
PREVIOUSLY PUNTED (pay € 30 balance to return):
% for (u) in sorted(punted, key=lambda p:p.username):
${"%20s (%s)" % (u.username, u.end)}
% for p in sorted(posts, key=lambda p:p['date']):
- ${p['title'] or "[ohne Titel]"} -- ${p['url']}
% endfor

View File

@ -2,8 +2,7 @@
From: ${mail}
Content-Type: text/plain; charset=utf-8
Subject: Iron Blogger Erinnerung: noch kein Beitrag in der Woche ab ${week_start.strftime("%d.%m.%Y")}:
To: ${mail}
Bcc: \
To: \
% for u in sorted(userlist, key=lambda u:u.name[u.name.rfind(' '):].lower()):
% if len(u.lameblogs)>0:
${u.mail}, \

View File

@ -1,39 +1,30 @@
<table id='participants'nowrap>
<table id='participants' nowrap>
<tr>
<th>Name</th>
<th>Twitter</th>
<th>Blog</th>
<th>Start</th>
<th>nicht gewertet</th>
<th>Autor</th>
<th>Twitter</th>
</tr>
% for u in sorted(userlist, key=lambda u:u.name[u.name.rfind(' '):].lower()):
<%
blogs=[]
for username in users:
u=users[username]
for l in u['links']:
if u.get('twitter') != False:
blogs.append([l[2], l[1], u['name'], username])
else:
blogs.append([l[2], l[1], u['name']])
%>
% for b in sorted(blogs, key=lambda b:b[1].lower()):
<tr>
<td align="left" valign="top">${u.name}</td>
<td align="left" valign="top">
% if u.twitter != False:
@<a href="http://twitter.com/${u.username}">${u.username}</a>
<a href="${b[0]}">${b[1]}</a>
</td>
<td align="left" valign="top">${b[2]}</td>
<td align="left" valign="top">
% if len(b)==4:
@<a href="http://twitter.com/${b[3]}">${b[3]}</a>
% endif
</td>
<td align="left" valign="top">
% if u.links:
% for a in u.links:
<a href="${a[2]}">${a[1]}</a>
% endfor
% else:
fehlt noch
% endif
</td>
<td align="left" valign="top"> ${u.start_de} </td>
<td align="left" valign="top">
% if u.end:
Ausgeschieden<br>
% endif
% if u.skip:
${u.skip}
% endif
</td>
</tr>
% endfor
</table>

View File

@ -1,122 +1,18 @@
Zusammenfassung der Woche ab ${week_start.strftime("%d.%m.%Y")}
<h3>Die Flei&szlig;igen:</h3>
<dl>
% for u in sorted(userlist, key=lambda u:u.name[u.name.rfind(' '):].lower()):
% for g in u.goodblogs:
% for b in u.links:
% if b[0] == g:
<dt><span class="user"><strong>${u.name} </strong> \
% if u.twitter != False:
(@<a href="http://twitter.com/${u.username}">${u.username}</a>) \
% endif
% if len(u.links) != 1:
in <a href="${b[2]}">${b[1]}</a> \
% endif
:</span></dt>
<dd>
<% pc=0 %><ul>
% for p in sorted(u.posts[g][week], key = lambda p:p['date']):
% if pc < 10 or len(u.posts[g][week]) == 11:
<% pc+=1 %><li><a href="${p['url']}">${p['title'] or "[ohne Titel]"}</a></li>
%endif
% endfor
% if len(u.posts[g][week]) > 11:
<li>und <a href="${b[2]}">${len(u.posts[g][week])-10} weitere Artikel</a>...</li>
% endif
</ul>
</dd>
% endif
% endfor
% endfor
% endfor
</dl>
<h3>Die Faulen:</h3> <% lame=0 %>
Blogging BW am ${start.strftime("%d.%m.%Y")}
<ul>
% for u in sorted(userlist, key=lambda u:u.name[u.name.rfind(' '):].lower()):
% for g in u.lameblogs:
% for b in u.links:
% if b[0] == g:
<li class="user"><strong>${u.name} </strong> <% lame+=1 %> \
% if u.twitter != False:
(@<a href="http://twitter.com/${u.username}">${u.username}</a>) \
% endif
% if len(u.links) != 1:
in <a href="${b[2]}">${b[1]}</a> \
% endif
% for p in sorted(posts, key=lambda p:p['date']):
<%
u=users[p['username']]
for l in u['links']:
if l[0] == p['blogname']:
b = l
%>
<li>${p['date'].strftime("%H:%M")}: <span class="user"><strong><a href="${p['url']}">${p['title'] or "[ohne Titel]"}</a></strong></span><br/>
${u['name']} \
% if u.get('twitter') != False:
(@<a href="http://twitter.com/${p['username']}">${p['username']}</a>) \
% endif
in <a href="${b[2]}">${b[1]}</a>:<blockquote>${p['description']}</blockquote>
</li>
% endif
% endfor
% endfor
% endfor
</ul>
% if punt:
<h3>Ausgeschieden wegen zu hoher Schulden:</h3>
<ul>
% for u in sorted(punt):
<li class="user">${u}</li>
% endfor
</ul>
% endif
% if skip:
<h3>Noch nicht dabei:</h3>
<ul>
% for u in sorted(skip, key=lambda u:u.name[u.name.rfind(' '):].lower()):
<li class="user"><strong>${u.name} </strong> \
% if u.twitter != False:
(@<a href="http://twitter.com/${u.username}">${u.username}</a>) \
% endif
</li>
% endfor
</ul>
% endif
% if skipped_users:
<h3>Gerade im Urlaub:</h3>
<ul>
% for u in sorted(skipped_users, key=lambda u:u.name[u.name.rfind(' '):].lower()):
<li class="user"><strong>${u.name} </strong> \
% if u.twitter != False:
(@<a href="http://twitter.com/${u.username}">${u.username}</a>) \
% endif
</li>
% endfor
</ul>
% endif
<h3>Kasse:</h3>
<table style="border-left-style:none; border-right-style:none;">
<tr> <td> diese Woche: </td> <td> ${5 * lame}&nbsp;&euro; </td> </tr>
<tr> <td> insgesamt: </td> <td> ${pool}&nbsp;&euro;</td> </tr>
<tr> <td> beglichen: </td> <td> ${paid}&nbsp;&euro;</td> </tr>
<tr> <td> verfeiert: </td> <td> ${event}&nbsp;&euro;</td> </tr>
</table>
<h3>Schulden:</h3>
<% i = 0 %>
<table class="debts" style="border-left-style:none; border-right-style:none;">
% for (u, v) in sorted(debts, key=lambda p:p[1], reverse=True):
% if i % 3 == 0:
<tr>\
% endif
<% i += 1 %>\
<td class="user">${u}</td> <td class="money">${v}&nbsp;&euro;</td>\
% if i % 3 == 0:
</tr>
%endif
% endfor
% if i % 3 != 0:
</tr>
%endif
</table>
% if punted:
<h3>Zuvor ausgeschieden (m&uuml;ssen 30&nbsp;&euro; f&uuml;r den Wiedereinstieg bezahlen):</h3>
<ul>
% for (u) in sorted(punted, key=lambda p:p.name[p.name.rfind(' '):].lower()):
<li>${u.name} (seit ${u.end_de})</li>
% endfor
</ul>
% endif

View File

@ -1,18 +1,18 @@
#!/usr/bin/python3
#!/usr/bin/python
import render
import os
import sys
import xmlrpc.client
import xmlrpclib
import subprocess
import settings
config=settings.load_settings()
x = xmlrpc.client.ServerProxy(config['xmlrpc_endpoint'])
x = xmlrpclib.ServerProxy(config['xmlrpc_endpoint'])
page = x.wp.getPage(config['blog_id'], config['participants_page_id'], config['username'], config['password'])
text = render.render_template('templates/users.tmpl').decode("utf-8")
text = render.render_template('templates/users.tmpl', '')
page['description'] = text
x.wp.editPage(config['blog_id'], config['participants_page_id'], config['username'], config['password'],page,True)

View File

@ -1,9 +1,9 @@
#!/usr/bin/python3
#!/usr/bin/python
# This Python file uses the following encoding: utf-8
import render
import os
import sys
import xmlrpc.client
import xmlrpclib
import subprocess
import datetime
import yaml
@ -22,7 +22,7 @@ if len(args)>0:
if args[0] == '-q':
dry_run = True
quick_view = True
send_mail = False
send_mail = False
args = args[1:]
if args[0] == '-r':
@ -32,15 +32,14 @@ if len(args)>0:
if args[0] == '-n':
dry_run = True
send_mail = False
send_mail = False
args = args[1:]
date = args[0]
with open('ledger', 'a') as f:
f.write("\n")
# print(render.render_template('templates/ledger', date).decode("utf-8"))
f.write(render.render_template('templates/ledger', date).decode("utf-8"))
f.write(render.render_template('templates/ledger', date))
if not dry_run:
subprocess.check_call(["git", "commit", "ledger",
@ -62,7 +61,7 @@ with open('ledger', 'a') as f:
if not dry_run:
text = render.render_template('templates/week.tmpl', date, punt=punt).decode("utf-8")
text = render.render_template('templates/week.tmpl', date, punt=punt)
lines = text.split("\n")
title = lines[0]
@ -70,16 +69,16 @@ if not dry_run:
page = dict(title = title, description = body)
x = xmlrpc.client.ServerProxy(config['xmlrpc_endpoint'])
x = xmlrpclib.ServerProxy(config['xmlrpc_endpoint'])
x.metaWeblog.newPost(config['blog_id'], config['username'], config['password'], page, True)
if not reminder:
email = render.render_template('templates/email.txt', date, punt=punt,mail=config['mail'])
else:
email = render.render_template('templates/reminder.txt', date, punt=punt,mail=config['mail'])
if quick_view:
print((render.render_template('templates/quick_view.tmpl',date,punt=punt)))
print(render.render_template('templates/quick_view.tmpl',date,punt=punt))
if dry_run and not quick_view:
print(email)
print email
if send_mail:
# p = subprocess.Popen(['mutt', '-H', '/dev/stdin'],
p = subprocess.Popen(['/usr/sbin/sendmail', '-oi', '-t'],