Start: load/save data dynamically from git (WIP)

Don't try to parse the feeds of bloggers with "end:" date in the past.

.gitignore

@@ -2,3 +2,4 @@
 *.pyc
 settings.cfg
 out/
+data/

Dockerfile

@@ -0,0 +1,9 @@
+FROM python:3
+WORKDIR /usr/src/app
+
+COPY requirements.txt ./
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+CMD [ "python", "./blogbot.py" ]

blogbot.py

@@ -0,0 +1,121 @@
+#!/usr/bin/python3
+import yaml
+import feedparser
+import datetime
+import sys
+import os
+import shutil
+import re
+from dateutil.parser import parse
+import dateutil.tz as tz
+import settings
+from git import Repo
+
+def parse_published(pub):
+    try:
+        return parse(pub).astimezone(tz.tzlocal()).replace(tzinfo=None)
+    except:
+        return parse(pub).replace(tzinfo=None)
+
+def get_date(post):
+    for k in ('published', 'created', 'updated'):
+        if k in post:
+            return post[k]
+
+def get_link(post):
+    return post.link
+
+def get_title(post):
+    if 'title' in post:
+        return post.title
+    else:
+        return ''
+
+def remove_html_tags(txt):
+    p = re.compile(r'<[^<]*?/?>')
+    return p.sub('', txt)
+
+def remove_extra_spaces(txt):
+    p = re.compile(r'\s+')
+    return p.sub(' ', txt)
+
+def create_extract(txt):
+    stxt = remove_extra_spaces(remove_html_tags(txt))
+    if len(stxt) < 250:
+        return stxt
+    if stxt.rfind('. ',200,250)>0:
+        return stxt[:stxt.rfind('. ',200,250)+1]+" [...]"
+    if stxt.rfind('! ',200,250)>0:
+        return stxt[:stxt.rfind('! ',200,250)+1]+" [...]"
+    if stxt.rfind('? ',200,250)>0:
+        return stxt[:stxt.rfind('? ',200,250)+1]+" [...]"
+    if stxt.rfind(', ',200,250)>0:
+        return stxt[:stxt.rfind(', ',200,250)+1]+" [...]"
+    if stxt.rfind(' ',200,250)>0:
+        return stxt[:stxt.rfind(' ',200,250)]+" [...]"
+    return stxt[:250]+"[...]"
+
+
+def parse_feeds(weeks, username, blog):
+    feedparser.USER_AGENT = "IronBloggerBot/0.2 +http://ironblogger.de/"
+    uri = blog[3]
+    feed = feedparser.parse(uri)
+
+    if not feed.entries:
+        print("WARN: no entries for ", uri, file=sys.stderr)
+    for post in feed.entries:
+        date = parse_published(get_date(post))
+
+        if date < START:
+            continue
+
+        key = date.strftime("%Y-%m-%d")
+
+        weeks.setdefault(key, [])
+
+        post = dict(date=date,
+                    title=get_title(post),
+                    url=get_link(post),
+                    username=username,
+                    blogname=blog[0],
+                    description=create_extract(post.description))
+        if post['url'] not in [p['url'] for p in weeks[key]]:
+            weeks[key].append(post)
+
+config=settings.load_settings()
+
+if os.path.exists('data'):
+    shutil.rmtree('data')
+
+gitrepo = Repo.clone_from('https://git.wazong.de/iron-blogger/content-society.git', 'data')
+
+with open('data/bloggers.yml') as f:
+    users = yaml.safe_load(f.read())
+
+if not os.path.exists('data/out'):
+    os.makedirs('data/out')
+try:
+    with open('data/out/report.yml') as f:
+        log = yaml.safe_load(f.read())
+except IOError:
+    log = {}
+
+START = datetime.datetime.strptime(config['start_date'],'%Y/%m/%d')
+
+if len(sys.argv) > 1:
+    for username in sys.argv[1:]:
+        blogs = log.setdefault(username, {})
+        for l in users[username]['links']:
+            parse_feeds(log, username, l)
+else:
+    for (username, u) in list(users.items()):
+        if 'end' in u:
+            enddate = datetime.datetime.strptime(u['end'],'%Y/%m/%d')
+            if enddate < datetime.datetime.now():
+                print("User inactive: ", username)
+                continue
+        for l in u['links']:
+            parse_feeds(log, username, l)
+
+with open('data/out/report.yml', 'w') as f:
+    yaml.safe_dump(log, f)

import-feeds.py

@@ -1,44 +1,49 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 from lxml import html
 import yaml
 import sys
-import urllib2
-import urlparse
+import urllib.request
+import urllib.parse
 
 with open('bloggers.yml') as f:
     users = yaml.safe_load(f.read())
 
 def fetch_links(url):
-    print >>sys.stderr, "Looking for feeds in %s" % (url,)
-    tree = html.fromstring(urllib2.urlopen(url).read())
-    links = tree.xpath(
-        '//link[@rel="alternate"][contains(@type, "rss") or ' +
-        'contains(@type, "atom") or contains(@type, "rdf")]')
-    candidates = [l for l in links if
+    print("Looking for feeds in %s" % (url,), file=sys.stderr)
+    try:
+        tree = html.document_fromstring(urllib.request.urlopen(url).read())
+        links = tree.xpath(
+             '//link[@rel="alternate"][contains(@type, "rss") or ' +
+             'contains(@type, "atom") or contains(@type, "rdf")]')
+        candidates = [l for l in links if
                   'atom' in l.attrib['type'] and
                   'comments' not in l.attrib['href'].lower() and
                   'comments' not in l.attrib.get('title','')]
+    except:
+        candidates = []
+        links = []
     if candidates:
         return candidates[0].attrib['href']
     elif links:
         return links[0].attrib['href']
     else:
-        print >>sys.stderr, "No link found for %s" % (url,)
+        print("No link found for %s" % (url,), file=sys.stderr)
         return None
 
-for (name, u) in users.items():
+for (name, u) in list(users.items()):
+    print("Processing user %s" % (name,), file=sys.stderr)
     for e in u['links']:
         (title, url) = e[1:3]
         try:
     	    e[1] = e[1].strip()
-    	except:
+        except:
     	    e[1] = e[1]
         if len(e) == 4:
             continue
         link = fetch_links(url)
         if link:
             if not link.startswith('http:'):
-                link = urlparse.urljoin(url, link)
+                link = urllib.parse.urljoin(url, link)
             e.append(link)
 
 with open('bloggers.yml', 'w') as f:

iron-blogger-server/ironblogger/.classpath

@@ -1,34 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
-  <classpathentry kind="src" path="src/test/java" output="target/test-classes" including="**/*.java"/>
-  <classpathentry kind="src" path="src/test/resources" output="target/test-classes" excluding="**/*.java"/>
-  <classpathentry kind="src" path="src/main/java" including="**/*.java"/>
-  <classpathentry kind="src" path="src/main/resources" excluding="**/*.java"/>
-  <classpathentry kind="output" path="target/classes"/>
-  <classpathentry kind="var" path="M2_REPO/javax/servlet/jstl/1.2/jstl-1.2.jar"/>
-  <classpathentry kind="var" path="M2_REPO/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar"/>
-  <classpathentry kind="var" path="M2_REPO/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar"/>
-  <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
-  <classpathentry kind="var" path="M2_REPO/com/sun/faces/jsf-api/2.1.7/jsf-api-2.1.7.jar"/>
-  <classpathentry kind="var" path="M2_REPO/com/sun/faces/jsf-impl/2.1.7/jsf-impl-2.1.7.jar"/>
-  <classpathentry kind="var" path="M2_REPO/org/richfaces/ui/richfaces-components-ui/4.1.0.Final/richfaces-components-ui-4.1.0.Final.jar"/>
-  <classpathentry kind="var" path="M2_REPO/org/richfaces/ui/richfaces-components-api/4.1.0.Final/richfaces-components-api-4.1.0.Final.jar"/>
-  <classpathentry kind="var" path="M2_REPO/org/richfaces/core/richfaces-core-api/4.1.0.Final/richfaces-core-api-4.1.0.Final.jar"/>
-  <classpathentry kind="var" path="M2_REPO/com/google/guava/guava/r08/guava-r08.jar"/>
-  <classpathentry kind="var" path="M2_REPO/org/richfaces/core/richfaces-core-impl/4.1.0.Final/richfaces-core-impl-4.1.0.Final.jar"/>
-  <classpathentry kind="var" path="M2_REPO/net/sourceforge/cssparser/cssparser/0.9.5/cssparser-0.9.5.jar"/>
-  <classpathentry kind="var" path="M2_REPO/org/w3c/css/sac/1.3/sac-1.3.jar"/>
-  <classpathentry kind="var" path="M2_REPO/junit/junit/4.8.1/junit-4.8.1.jar"/>
-  <classpathentry kind="var" path="M2_REPO/org/hibernate/ejb3-persistence/1.0.2.GA/ejb3-persistence-1.0.2.GA.jar"/>
-  <classpathentry kind="var" path="M2_REPO/org/hibernate/hibernate-entitymanager/3.4.0.GA/hibernate-entitymanager-3.4.0.GA.jar"/>
-  <classpathentry kind="var" path="M2_REPO/org/hibernate/hibernate-commons-annotations/3.1.0.GA/hibernate-commons-annotations-3.1.0.GA.jar"/>
-  <classpathentry kind="var" path="M2_REPO/org/slf4j/slf4j-api/1.4.2/slf4j-api-1.4.2.jar"/>
-  <classpathentry kind="var" path="M2_REPO/org/hibernate/hibernate-annotations/3.4.0.GA/hibernate-annotations-3.4.0.GA.jar"/>
-  <classpathentry kind="var" path="M2_REPO/org/hibernate/hibernate-core/3.3.1.GA/hibernate-core-3.3.1.GA.jar"/>
-  <classpathentry kind="var" path="M2_REPO/antlr/antlr/2.7.6/antlr-2.7.6.jar"/>
-  <classpathentry kind="var" path="M2_REPO/commons-collections/commons-collections/3.1/commons-collections-3.1.jar"/>
-  <classpathentry kind="var" path="M2_REPO/dom4j/dom4j/1.6.1/dom4j-1.6.1.jar"/>
-  <classpathentry kind="var" path="M2_REPO/xml-apis/xml-apis/1.0.b2/xml-apis-1.0.b2.jar"/>
-  <classpathentry kind="var" path="M2_REPO/javassist/javassist/3.4.GA/javassist-3.4.GA.jar"/>
-  <classpathentry kind="var" path="M2_REPO/log4j/log4j/1.2.16/log4j-1.2.16.jar"/>
-</classpath>

iron-blogger-server/ironblogger/.gitignore

@@ -1 +0,0 @@
-/target

iron-blogger-server/ironblogger/.project

@@ -1,27 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<projectDescription>
-  <name>ironblogger</name>
-  <comment>NO_M2ECLIPSE_SUPPORT: Project files created with the maven-eclipse-plugin are not supported in M2Eclipse.</comment>
-  <projects/>
-  <buildSpec>
-    <buildCommand>
-      <name>org.eclipse.jdt.core.javabuilder</name>
-    </buildCommand>
-    <buildCommand>
-      <name>org.eclipse.m2e.core.maven2Builder</name>
-    </buildCommand>
-    <buildCommand>
-      <name>org.eclipse.wst.common.project.facet.core.builder</name>
-    </buildCommand>
-    <buildCommand>
-      <name>org.eclipse.wst.validation.validationbuilder</name>
-    </buildCommand>
-  </buildSpec>
-  <natures>
-    <nature>org.eclipse.jdt.core.javanature</nature>
-    <nature>org.eclipse.m2e.core.maven2Nature</nature>
-    <nature>org.eclipse.wst.common.project.facet.core.nature</nature>
-    <nature>org.eclipse.wst.common.modulecore.ModuleCoreNature</nature>
-    <nature>org.eclipse.jem.workbench.JavaEMFNature</nature>
-  </natures>
-</projectDescription>

iron-blogger-server/ironblogger/.settings/org.eclipse.jdt.core.prefs

@@ -1,8 +0,0 @@
-eclipse.preferences.version=1
-org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled
-org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6
-org.eclipse.jdt.core.compiler.compliance=1.6
-org.eclipse.jdt.core.compiler.problem.assertIdentifier=error
-org.eclipse.jdt.core.compiler.problem.enumIdentifier=error
-org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning
-org.eclipse.jdt.core.compiler.source=1.6

iron-blogger-server/ironblogger/.settings/org.eclipse.m2e.core.prefs

@@ -1,4 +0,0 @@
-activeProfiles=
-eclipse.preferences.version=1
-resolveWorkspaceProjects=true
-version=1

iron-blogger-server/ironblogger/.settings/org.eclipse.wst.common.component

@@ -1,34 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project-modules id="moduleCoreId" project-version="1.5.0">
-  <wb-module deploy-name="ironblogger">
-    <property name="context-root" value="ironblogger"/>
-    <wb-resource deploy-path="/" source-path="src/main/webapp"/>
-    <property name="java-output-path" value="/target/classes"/>
-    <dependent-module archiveName="richfaces-components-ui-4.1.0.Final.jar" deploy-path="/WEB-INF/lib" handle="module:/classpath/var/M2_REPO/org/richfaces/ui/richfaces-components-ui/4.1.0.Final/richfaces-components-ui-4.1.0.Final.jar">
-      <dependency-type>uses</dependency-type>
-    </dependent-module>
-    <dependent-module archiveName="richfaces-components-api-4.1.0.Final.jar" deploy-path="/WEB-INF/lib" handle="module:/classpath/var/M2_REPO/org/richfaces/ui/richfaces-components-api/4.1.0.Final/richfaces-components-api-4.1.0.Final.jar">
-      <dependency-type>uses</dependency-type>
-    </dependent-module>
-    <dependent-module archiveName="richfaces-core-api-4.1.0.Final.jar" deploy-path="/WEB-INF/lib" handle="module:/classpath/var/M2_REPO/org/richfaces/core/richfaces-core-api/4.1.0.Final/richfaces-core-api-4.1.0.Final.jar">
-      <dependency-type>uses</dependency-type>
-    </dependent-module>
-    <dependent-module archiveName="guava-r08.jar" deploy-path="/WEB-INF/lib" handle="module:/classpath/var/M2_REPO/com/google/guava/guava/r08/guava-r08.jar">
-      <dependency-type>uses</dependency-type>
-    </dependent-module>
-    <dependent-module archiveName="richfaces-core-impl-4.1.0.Final.jar" deploy-path="/WEB-INF/lib" handle="module:/classpath/var/M2_REPO/org/richfaces/core/richfaces-core-impl/4.1.0.Final/richfaces-core-impl-4.1.0.Final.jar">
-      <dependency-type>uses</dependency-type>
-    </dependent-module>
-    <dependent-module archiveName="cssparser-0.9.5.jar" deploy-path="/WEB-INF/lib" handle="module:/classpath/var/M2_REPO/net/sourceforge/cssparser/cssparser/0.9.5/cssparser-0.9.5.jar">
-      <dependency-type>uses</dependency-type>
-    </dependent-module>
-    <dependent-module archiveName="sac-1.3.jar" deploy-path="/WEB-INF/lib" handle="module:/classpath/var/M2_REPO/org/w3c/css/sac/1.3/sac-1.3.jar">
-      <dependency-type>uses</dependency-type>
-    </dependent-module>
-    <wb-resource deploy-path="/WEB-INF/classes" source-path="src/main/resources"/>
-        <wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/resources"/>
-        <wb-resource deploy-path="/WEB-INF/classes" source-path="/src/main/java"/>
-        <wb-resource deploy-path="/WEB-INF/classes" source-path="/src/test/java"/>
-        <wb-resource deploy-path="/WEB-INF/classes" source-path="/src/test/resources"/>
-  </wb-module>
-</project-modules>

iron-blogger-server/ironblogger/.settings/org.eclipse.wst.common.project.facet.core.xml

@@ -1,7 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<faceted-project>
-  <fixed facet="jst.java"/>
-  <fixed facet="jst.web"/>
-  <installed facet="jst.web" version="2.4"/>
-  <installed facet="jst.java" version="1.6"/>
-</faceted-project>

iron-blogger-server/ironblogger/pom.xml

@@ -1,125 +0,0 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-	<modelVersion>4.0.0</modelVersion>
-	<groupId>de.wazong</groupId>
-	<artifactId>ironblogger</artifactId>
-	<packaging>war</packaging>
-	<version>0.0.1-SNAPSHOT</version>
-	<name>ironblogger Maven Webapp</name>
-	<url>http://maven.apache.org</url>
-	<dependencies>
-		<dependency>
-			<groupId>com.sun.faces</groupId>
-			<artifactId>jsf-api</artifactId>
-			<version>2.1.7</version>
-			<scope>runtime</scope>
-		</dependency>
-		<dependency>
-			<groupId>com.sun.faces</groupId>
-			<artifactId>jsf-impl</artifactId>
-			<version>2.1.7</version>
-			<scope>runtime</scope>
-		</dependency>
-
-		<dependency>
-			<groupId>javax.servlet</groupId>
-			<artifactId>jstl</artifactId>
-			<version>1.2</version>
-		</dependency>
- 
-		<dependency>
-			<groupId>javax.servlet</groupId>
-			<artifactId>servlet-api</artifactId>
-			<version>2.5</version>
-			<scope>provided</scope>
-		</dependency>
- 
-		<dependency>
-			<groupId>javax.servlet.jsp</groupId>
-			<artifactId>jsp-api</artifactId>
-			<version>2.1</version>
-			<scope>provided</scope>
-		</dependency>		
-		<dependency>
-			<groupId>org.richfaces.ui</groupId>
-			<artifactId>richfaces-components-ui</artifactId>
-			<scope>runtime</scope>
-		</dependency>
-		<dependency>
-			<groupId>org.richfaces.core</groupId>
-			<artifactId>richfaces-core-impl</artifactId>
-			<scope>runtime</scope>
-		</dependency>
-		<dependency>
-			<groupId>junit</groupId>
-			<artifactId>junit</artifactId>
-			<version>4.8.1</version>
-			<scope>test</scope>
-		</dependency>
-		                       <dependency>
-                                <groupId>org.hibernate</groupId>
-                                <artifactId>ejb3-persistence</artifactId>
-                                <version>1.0.2.GA</version>
-                        </dependency>
-                        <dependency>
-                                <groupId>org.hibernate</groupId>
-                                <artifactId>hibernate-entitymanager</artifactId>
-                                <version>3.4.0.GA</version>
-                        </dependency>
-                        <dependency>
-                                <groupId>org.hibernate</groupId>
-                                <artifactId>hibernate-commons-annotations</artifactId>
-                                <version>3.1.0.GA</version>
-                        </dependency>
-                        <dependency>
-                                <groupId>org.hibernate</groupId>
-                                <artifactId>hibernate-annotations</artifactId>
-                                <version>3.4.0.GA</version>
-                        </dependency>
-                        <dependency>
-                                <groupId>org.hibernate</groupId>
-                                <artifactId>hibernate-core</artifactId>
-                                <version>3.3.1.GA</version>
-                        </dependency>
-                           <dependency>
-                        <groupId>org.slf4j</groupId>
-                        <artifactId>slf4j-api</artifactId>
-                        <version>1.5.10</version>
-                        <scope>test</scope>
-                </dependency>
-                                 <dependency>
-                        <groupId>org.slf4j</groupId>
-                        <artifactId>slf4j-log4j12</artifactId>
-                        <version>1.5.10</version>
-                        <scope>test</scope>
-                </dependency>
-                        
-                                       <dependency>
-                        <groupId>log4j</groupId>
-                        <artifactId>log4j</artifactId>
-                        <version>1.2.16</version>
-                </dependency>
-                <dependency>
-	<groupId>mysql</groupId>
-	<artifactId>mysql-connector-java</artifactId>
-	<version>5.1.26</version>
-</dependency>
-	</dependencies>
-	<properties>
-		<org.richfaces.bom.version>4.1.0.Final</org.richfaces.bom.version>
-	</properties>
-	<dependencyManagement>
-		<dependencies>
-			<dependency>
-				<groupId>org.richfaces</groupId>
-				<artifactId>richfaces-bom</artifactId>
-				<version>${org.richfaces.bom.version}</version>
-				<scope>import</scope>
-				<type>pom</type>
-			</dependency>
-		</dependencies>
-	</dependencyManagement>
-	<build>
-		<finalName>ironblogger</finalName>
-	</build>
-</project>

iron-blogger-server/ironblogger/src/main/java/de/wazong/ironblogger/Test.java

@@ -1,5 +0,0 @@
-package de.wazong.ironblogger;
-
-public class Test {
-	
-}

iron-blogger-server/ironblogger/src/main/java/de/wazong/ironblogger/persistence/Feed.java

@@ -1,36 +0,0 @@
-package de.wazong.ironblogger.persistence;
-
-import javax.persistence.Entity;
-import javax.persistence.Id;
-import javax.persistence.Table;
-
-@Entity
-@Table(name="feed")
-public class Feed {
-	
-	@Id
-	private long id;
-	private String name;
-	private String feedURL;
-	
-	
-	public long getId() {
-		return id;
-	}
-	public void setId(long id) {
-		this.id = id;
-	}
-	public String getName() {
-		return name;
-	}
-	public void setName(String name) {
-		this.name = name;
-	}
-	public String getFeedURL() {
-		return feedURL;
-	}
-	public void setFeedURL(String feedURL) {
-		this.feedURL = feedURL;
-	}
-
-}

iron-blogger-server/ironblogger/src/main/webapp/WEB-INF/web.xml

@@ -1,47 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-	xmlns="http://java.sun.com/xml/ns/javaee" 
-        xmlns:web="http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd"
-	xsi:schemaLocation="http://java.sun.com/xml/ns/javaee 
-	http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd"
-	id="WebApp_ID" version="2.5">
- 
-	<display-name>JavaServerFaces</display-name>
- 
-	<!-- Change to "Production" when you are ready to deploy -->
-	<context-param>
-		<param-name>javax.faces.PROJECT_STAGE</param-name>
-		<param-value>Development</param-value>
-	</context-param>
- 
-	<!-- Welcome page -->
-	<welcome-file-list>
-		<welcome-file>faces/hello.xhtml</welcome-file>
-	</welcome-file-list>
- 
-	<!-- JSF mapping -->
-	<servlet>
-		<servlet-name>Faces Servlet</servlet-name>
-		<servlet-class>javax.faces.webapp.FacesServlet</servlet-class>
-		<load-on-startup>1</load-on-startup>
-	</servlet>
- 
-	<!-- Map these files with JSF -->
-	<servlet-mapping>
-		<servlet-name>Faces Servlet</servlet-name>
-		<url-pattern>/faces/*</url-pattern>
-	</servlet-mapping>
-	<servlet-mapping>
-		<servlet-name>Faces Servlet</servlet-name>
-		<url-pattern>*.jsf</url-pattern>
-	</servlet-mapping>
-	<servlet-mapping>
-		<servlet-name>Faces Servlet</servlet-name>
-		<url-pattern>*.faces</url-pattern>
-	</servlet-mapping>
-	<servlet-mapping>
-		<servlet-name>Faces Servlet</servlet-name>
-		<url-pattern>*.xhtml</url-pattern>
-	</servlet-mapping>
- 
-</web-app>

iron-blogger-server/ironblogger/src/main/webapp/faces/hello.xhtml

@@ -1,18 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" 
-"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml"
-      xmlns:f="http://java.sun.com/jsf/core"      
-      xmlns:h="http://java.sun.com/jsf/html">
- 
-    <h:head>
-        <title>JSF 2.0 Hello World</title>
-    </h:head>
-    <h:body>
-    	<h3>JSF 2.0 Hello World Example - hello.xhtml</h3>
-    	<h:form>
-    	   <h:inputText value="#{helloBean.name}"></h:inputText>
-    	   <h:commandButton value="Welcome Me" action="welcome"></h:commandButton>
-    	</h:form>
-    </h:body>
-</html>

iron-blogger-server/ironblogger/src/main/webapp/faces/welcome.xhtml

@@ -1,14 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" 
-"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml"    
-      xmlns:h="http://java.sun.com/jsf/html">
- 
-    <h:head>
-    	<title>JSF 2.0 Hello World</title>
-    </h:head>
-    <h:body bgcolor="white">
-    	<h3>JSF 2.0 Hello World Example - welcome.xhtml</h3>
-    	<h4>Welcome #{helloBean.name}</h4>
-    </h:body>
-</html>

iron-blogger-server/ironblogger/src/main/webapp/hello.xhtml

@@ -1,18 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" 
-"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml"
-      xmlns:f="http://java.sun.com/jsf/core"      
-      xmlns:h="http://java.sun.com/jsf/html">
- 
-    <h:head>
-        <title>JSF 2.0 Hello World</title>
-    </h:head>
-    <h:body>
-    	<h3>JSF 2.0 Hello World Example - hello.xhtml</h3>
-    	<h:form>
-    	   <h:inputText value="#{helloBean.name}"></h:inputText>
-    	   <h:commandButton value="Welcome Me" action="welcome"></h:commandButton>
-    	</h:form>
-    </h:body>
-</html>

iron-blogger-server/ironblogger/src/main/webapp/index.jsp

@@ -1,5 +0,0 @@
-<html>
-<body>
-<h2>Hello World!</h2>
-</body>
-</html>

iron-blogger-server/ironblogger/src/test/java/de/wazong/ironblogger/TestTest.java

@@ -1,57 +0,0 @@
-package de.wazong.ironblogger;
-
-import javax.persistence.EntityManager;
-import javax.persistence.EntityManagerFactory;
-import javax.persistence.Persistence;
-
-import org.apache.log4j.Logger;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-public class TestTest {
-
-	private static EntityManager em;
-	private static final Logger log = Logger.getLogger(TestTest.class);
-
-	@Test
-	public void testNothing() throws Exception {
-
-        if (em == null)setUp();
-
-        
-
-	}
-
-	/**
-	 * Creates an {@link EntityManager} for this test.
-	 * 
-	 * @throws Exception
-	 */
-	@BeforeClass
-	public static void setUp() throws Exception {
-		if (em == null) {
-			final EntityManagerFactory emf = Persistence
-					.createEntityManagerFactory("ibTest");
-			em = emf.createEntityManager();
-		}
-	}
-
-	/**
-	 * Cleans up after this test has run.
-	 */
-	@AfterClass
-	public static void cleanUp() {
-		em.clear();
-		em.getTransaction().begin();
-//		for (Record r : ids) {
-//			Object o = em.find(r.getClazz(), r.getId());
-//			if (o != null)
-//				em.remove(o);
-//		}
-		em.getTransaction().commit();
-		em.close();
-	}
-
-
-}

iron-blogger-server/ironblogger/src/test/resources/META-INF/persistence.xml

@@ -1,16 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<persistence version="2.0"
-        xmlns="http://java.sun.com/xml/ns/persistence" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
-        xsi:schemaLocation="http://java.sun.com/xml/ns/persistence persistence_2_0.xsd">
-        <persistence-unit name="ibTest" transaction-type="RESOURCE_LOCAL">
-                <class>de.wazong.ironblogger.persistence.Feed</class>
-                <properties>
-                        <property name="show_sql" value="true" />
-                        <property name="hibernate.connection.url" value="jdbc:mysql://localhost:3306/ironblogger" />
-                        <property name="hibernate.connection.username" value="ironblogger" />
-                        <property name="hibernate.connection.password" value="ironblogger" />
-                        <property name="hibernate.connection.driver_class" value="com.mysql.jdbc.Driver" />
-                        <property name="hibernate.dialect" value="org.hibernate.dialect.MySQLDialect" />
-                </properties>
-        </persistence-unit>
-</persistence>

iron-blogger-server/ironblogger/src/test/resources/log4j.xml

@@ -1,28 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" ?> <!DOCTYPE log4j:configuration SYSTEM "log4j.dtd"> 
-<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/"> 
-        <appender name="console" class="org.apache.log4j.ConsoleAppender"> 
-                <!-- param name="Threshold" value="info" / --> 
-                <param name="Target" value="System.out" /> 
-                <layout class="org.apache.log4j.PatternLayout"> 
-                        <param name="ConversionPattern" value="%d{ABSOLUTE} [%t] %-5p %c{1} - %m%n" /> 
-                </layout> 
-        </appender> 
-        <logger name="org.hibernate"> 
-                <!-- level value="TRACE" / --> 
-                <level value="ERROR" /> 
-                <appender-ref ref="console" /> 
-        </logger> 
-        <!-- logger name="org.hibernate.SQL"> 
-                <level value="debug" /> 
-                <appender-ref ref="console" /> 
-        </logger --> 
-        <!--  logger name="org.hibernate.type"> 
-                <level value="TRACE" /> 
-                <appender-ref ref="console" /> 
-        </logger --> 
-        <root> 
-                <priority value="INFO" /> 
-                <!-- appender-ref ref="console" / --> 
-                <!-- appender-ref ref="rolling-file" / --> 
-        </root> 
-</log4j:configuration>

render.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # This Python file uses the following encoding: utf-8
 import yaml
 from dateutil.parser import parse
@@ -29,6 +29,7 @@ def get_balance(acct):
 def get_debts():
     p = subprocess.Popen(['ledger', '-f', os.path.join(HERE, 'ledger'),
                           '-n', 'balance', 'Pool:Owed:'],
+                         universal_newlines=True,
                          stdout=subprocess.PIPE)
     (out, _) = p.communicate()
     debts = []
@@ -48,7 +49,7 @@ def parse_skip(rec):
     out = []
     for s in spec:
         if isinstance(s, list):
-            out.append(map(to_week_num, s))
+            out.append(list(map(to_week_num, s)))
         else:
             out.append(to_week_num(s))
     return out
@@ -72,7 +73,7 @@ def render_template(path, week=None, **kwargs):
     else:
         week = START
 
-    week = (week - START).days / 7
+    week = int( (week - START).days / 7)
     week_start = START + (week * datetime.timedelta(7))
     week_end   = START + ((week + 1) * datetime.timedelta(7))
 
@@ -84,7 +85,7 @@ def render_template(path, week=None, **kwargs):
     class User(object):
         pass
 
-    for (un, rec) in users.items():
+    for (un, rec) in list(users.items()):
         u = User()
         u.username = un
         u.name  = rec['name']
@@ -97,8 +98,8 @@ def render_template(path, week=None, **kwargs):
         u.stop  = rec.get('stop')
         u.skip  = parse_skip(rec)
         u.posts = report.get(un, {})
-	u.goodblogs = []
-	u.lameblogs = []
+        u.goodblogs = []
+        u.lameblogs = []
 
         userlist.append(u)
 
@@ -121,13 +122,13 @@ def render_template(path, week=None, **kwargs):
             continue
         if should_skip(u.skip, week):
             skipped_users.append(u)
-	    continue
+            continue
         elif user_start > week_start:
             skip.append(u)
-	    continue
-	for blog in u.links:
-	    b=blog[0]
-	    weeks=u.posts[b]
+            continue
+        for blog in u.links:
+            b=blog[0]
+            weeks=u.posts[b]
             if len(weeks) <= week or not weeks[week]:
                 u.lameblogs.append(b)
             else:
@@ -144,10 +145,10 @@ def render_template(path, week=None, **kwargs):
 
 if __name__ == '__main__':
     if len(sys.argv) < 2:
-        print >>sys.stderr, "Usage: %s TEMPLATE [WEEK]"
+        print("Usage: %s TEMPLATE [WEEK]", file=sys.stderr)
         sys.exit(1)
 
     template = sys.argv[1]
     week = None
     if len(sys.argv) > 2: week = sys.argv[2]
-    print render_template(template, week)
+    print(render_template(template, week))

requirements.txt

@@ -0,0 +1,4 @@
+pyyaml
+feedparser
+python-dateutil
+GitPython

scan-feeds.py

@@ -1,80 +0,0 @@
-#!/usr/bin/python
-import yaml
-import feedparser
-import datetime
-import sys
-import os
-from dateutil.parser import parse
-import dateutil.tz as tz
-import settings
-
-config=settings.load_settings()
-
-with open('bloggers.yml') as f:
-    users = yaml.safe_load(f.read())
-
-if not os.path.exists('out'):
-    os.makedirs('out')
-try:
-    with open('out/report.yml') as f:
-        log = yaml.safe_load(f.read())
-except IOError:
-    log = {}
-
-START = datetime.datetime.strptime(config['start_date'],'%Y/%m/%d')
-
-def parse_published(pub):
-    try:
-        return parse(pub).astimezone(tz.tzlocal()).replace(tzinfo=None)
-    except:
-        return parse(pub).replace(tzinfo=None)
-def get_date(post):
-    for k in ('published', 'created', 'updated'):
-        if k in post:
-            return post[k]
-
-def get_link(post):
-    return post.link
-
-def parse_feeds(weeks, uri):
-    feedparser.USER_AGENT = "IronBloggerBot/0.1 +http://ironblogger.de/"
-    feed = feedparser.parse(uri)
-
-    if not feed.entries:
-        print >>sys.stderr, "WARN: no entries for ", uri
-    for post in feed.entries:
-        date = parse_published(get_date(post))
-
-        if date < START:
-            continue
-        wn = (date - START).days / 7
-
-        while len(weeks) <= wn:
-            weeks.append([])
-
-        if post.has_key('title'):
-            post = dict(date=date,
-                        title=post.title,
-                        url=get_link(post))
-        if not post.has_key('title'):
-            post = dict(date=date,
-                        title="",
-                        url=get_link(post))
-        if post['url'] not in [p['url'] for p in weeks[wn]]:
-            weeks[wn].append(post)
-
-if len(sys.argv) > 1:
-    for username in sys.argv[1:]:
-        blogs = log.setdefault(username, {})
-        for l in users[username]['links']:
-	    weeks = blogs.setdefault(l[0], [])
-            parse_feeds(weeks, l[3])
-else:
-    for (username, u) in users.items():
-        blogs = log.setdefault(username, {})
-        for l in u['links']:
-	    weeks = blogs.setdefault(l[0], [])
-            parse_feeds(weeks, l[3])
-
-with open('out/report.yml', 'w') as f:
-    yaml.safe_dump(log, f)

settings.py

@@ -1,8 +1,8 @@
 #!/usr/bin/python
-import ConfigParser, os
+import configparser, os
 
 def load_settings():
-    configfile = ConfigParser.ConfigParser()
+    configfile = configparser.ConfigParser()
     configfile.read('settings.cfg')
     config=dict()
     config['mail']=configfile.get("general","mail")

templates/email.txt

@@ -53,7 +53,7 @@ Paid:      € ${paid}
 Events:    € ${event}
 Individual debts:
 % for (u, v) in sorted(debts, key=lambda p:p[1], reverse=True):
-${u"%20s %d \u20AC" % (u, v)}
+${"%20s %d \u20AC" % (u, v)}
 % endfor
 
 PREVIOUSLY PUNTED (pay € 30 balance to return):

update-participants.py

@@ -1,18 +1,18 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 
 import render
 import os
 import sys
-import xmlrpclib
+import xmlrpc.client
 import subprocess
 import settings
 
 config=settings.load_settings()
 
-x = xmlrpclib.ServerProxy(config['xmlrpc_endpoint'])
+x = xmlrpc.client.ServerProxy(config['xmlrpc_endpoint'])
 page = x.wp.getPage(config['blog_id'], config['participants_page_id'], config['username'], config['password'])
 
-text = render.render_template('templates/users.tmpl')
+text = render.render_template('templates/users.tmpl').decode("utf-8")
 page['description'] = text
 
 x.wp.editPage(config['blog_id'], config['participants_page_id'], config['username'], config['password'],page,True)

weekly-update.py

@@ -1,9 +1,9 @@
-#!/usr/bin/python
+#!/usr/bin/python3
 # This Python file uses the following encoding: utf-8
 import render
 import os
 import sys
-import xmlrpclib
+import xmlrpc.client
 import subprocess
 import datetime
 import yaml
@@ -22,7 +22,7 @@ if len(args)>0:
     if args[0] == '-q':
         dry_run = True
         quick_view = True
-	send_mail = False
+        send_mail = False
         args = args[1:]
 
     if args[0] == '-r':
@@ -32,14 +32,15 @@ if len(args)>0:
 
     if args[0] == '-n':
         dry_run = True
-	send_mail = False
+        send_mail = False
         args = args[1:]
 
 date = args[0]
 
 with open('ledger', 'a') as f:
     f.write("\n")
-    f.write(render.render_template('templates/ledger', date))
+    # print(render.render_template('templates/ledger', date).decode("utf-8"))
+    f.write(render.render_template('templates/ledger', date).decode("utf-8"))
 
 if not dry_run:
     subprocess.check_call(["git", "commit", "ledger",
@@ -61,7 +62,7 @@ with open('ledger', 'a') as f:
 
 
 if not dry_run:
-    text = render.render_template('templates/week.tmpl', date, punt=punt)
+    text = render.render_template('templates/week.tmpl', date, punt=punt).decode("utf-8")
 
     lines = text.split("\n")
     title = lines[0]
@@ -69,16 +70,16 @@ if not dry_run:
 
     page = dict(title = title, description = body)
 
-    x = xmlrpclib.ServerProxy(config['xmlrpc_endpoint'])
+    x = xmlrpc.client.ServerProxy(config['xmlrpc_endpoint'])
     x.metaWeblog.newPost(config['blog_id'], config['username'], config['password'], page, True)
 if not reminder:
     email = render.render_template('templates/email.txt', date, punt=punt,mail=config['mail'])
 else:
     email = render.render_template('templates/reminder.txt', date, punt=punt,mail=config['mail'])
 if quick_view:
-    print(render.render_template('templates/quick_view.tmpl',date,punt=punt))
+    print((render.render_template('templates/quick_view.tmpl',date,punt=punt)))
 if dry_run and not quick_view:
-    print email
+    print(email)
 if send_mail:
     # p = subprocess.Popen(['mutt', '-H', '/dev/stdin'],
     p = subprocess.Popen(['/usr/sbin/sendmail', '-oi', '-t'],