@@ -0,0 +1,133 @@ | |||
# Created by https://www.gitignore.io/api/python | |||
### Python ### | |||
# Byte-compiled / optimized / DLL files | |||
__pycache__/ | |||
*.py[cod] | |||
*$py.class | |||
# C extensions | |||
*.so | |||
# Distribution / packaging | |||
.Python | |||
build/ | |||
develop-eggs/ | |||
dist/ | |||
downloads/ | |||
eggs/ | |||
.eggs/ | |||
lib/ | |||
lib64/ | |||
parts/ | |||
sdist/ | |||
var/ | |||
wheels/ | |||
*.egg-info/ | |||
.installed.cfg | |||
*.egg | |||
MANIFEST | |||
# PyInstaller | |||
# Usually these files are written by a python script from a template | |||
# before PyInstaller builds the exe, so as to inject date/other infos into it. | |||
*.manifest | |||
*.spec | |||
# Installer logs | |||
pip-log.txt | |||
pip-delete-this-directory.txt | |||
# Unit test / coverage reports | |||
htmlcov/ | |||
.tox/ | |||
.nox/ | |||
.coverage | |||
.coverage.* | |||
.cache | |||
nosetests.xml | |||
coverage.xml | |||
*.cover | |||
.hypothesis/ | |||
.pytest_cache/ | |||
# Translations | |||
*.mo | |||
*.pot | |||
# Django stuff: | |||
*.log | |||
local_settings.py | |||
db.sqlite3 | |||
# Flask stuff: | |||
instance/ | |||
.webassets-cache | |||
# Scrapy stuff: | |||
.scrapy | |||
# Sphinx documentation | |||
docs/_build/ | |||
# PyBuilder | |||
target/ | |||
# Jupyter Notebook | |||
.ipynb_checkpoints | |||
# IPython | |||
profile_default/ | |||
ipython_config.py | |||
# pyenv | |||
.python-version | |||
# celery beat schedule file | |||
celerybeat-schedule | |||
# SageMath parsed files | |||
*.sage.py | |||
# Environments | |||
.env | |||
.venv | |||
env/ | |||
venv/ | |||
ENV/ | |||
env.bak/ | |||
venv.bak/ | |||
# Spyder project settings | |||
.spyderproject | |||
.spyproject | |||
# Rope project settings | |||
.ropeproject | |||
# mkdocs documentation | |||
/site | |||
# mypy | |||
.mypy_cache/ | |||
.dmypy.json | |||
dmypy.json | |||
### Python Patch ### | |||
.venv/ | |||
### Python.VirtualEnv Stack ### | |||
# Virtualenv | |||
# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/ | |||
[Bb]in | |||
[Ii]nclude | |||
[Ll]ib | |||
[Ll]ib64 | |||
[Ll]ocal | |||
[Ss]cripts | |||
pyvenv.cfg | |||
pip-selfcheck.json | |||
# End of https://www.gitignore.io/api/python |
@@ -21,6 +21,10 @@ python euroexchange.py | |||
* 0.1.1 | |||
- Bugfix: only update image if update period is reached | |||
* 0.2 | |||
- Checks gnuplot exitcode and only upload chart if successfull created | |||
- Update gnuplot script on commons file description page | |||
## Bugs | |||
[jogobot-euroexchange Issues](https://git.golderweb.de/wiki/jogobot-euroexchange/issues) | |||
@@ -0,0 +1,47 @@ | |||
#!/usr/bin/env python3 | |||
# -*- coding: utf-8 -*- | |||
# | |||
# config.py | |||
# | |||
# Copyright 2018 Jonathan Golder <jonathan@golderweb.de> | |||
# | |||
# This program is free software; you can redistribute it and/or modify | |||
# it under the terms of the GNU General Public License as published by | |||
# the Free Software Foundation; either version 2 of the License, or | |||
# (at your option) any later version. | |||
# | |||
# This program is distributed in the hope that it will be useful, | |||
# but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||
# GNU General Public License for more details. | |||
# | |||
# You should have received a copy of the GNU General Public License | |||
# along with this program; if not, write to the Free Software | |||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, | |||
# MA 02110-1301, USA. | |||
# | |||
# | |||
""" | |||
Loads configuration of EuroExchangeBot | |||
""" | |||
import os | |||
import jogobot | |||
class Config(): | |||
""" | |||
Configuration container for EuroExchangeBot | |||
""" | |||
base_dir = os.path.expanduser(jogobot.config["euroexchange"]["base_dir"]) | |||
working_dir = os.path.join( base_dir, "working_dir" ) | |||
gnuplot_script_dir = os.path.join(base_dir, "gnuplot_scripts") | |||
gnuplot = jogobot.config["euroexchange"]["gnuplot_bin"] | |||
data_source = jogobot.config["euroexchange"]["data_source"] | |||
zip_file = jogobot.config["euroexchange"]["data_zip_filename"] | |||
csv_file = jogobot.config["euroexchange"]["data_csv_filename"] | |||
upload_comment = jogobot.config["euroexchange"]["upload_comment"] | |||
gnuplot_script_comment = jogobot.config["euroexchange"]["gnuplot_script_comment"] | |||
gnuplot_script_help = jogobot.config["euroexchange"]["gnuplot_script_help"] |
@@ -0,0 +1,129 @@ | |||
#!/usr/bin/env python3 | |||
# -*- coding: utf-8 -*- | |||
# | |||
# descpage.py | |||
# | |||
# Copyright 2018 Jonathan Golder <jonathan@golderweb.de> | |||
# | |||
# This program is free software; you can redistribute it and/or modify | |||
# it under the terms of the GNU General Public License as published by | |||
# the Free Software Foundation; either version 2 of the License, or | |||
# (at your option) any later version. | |||
# | |||
# This program is distributed in the hope that it will be useful, | |||
# but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||
# GNU General Public License for more details. | |||
# | |||
# You should have received a copy of the GNU General Public License | |||
# along with this program; if not, write to the Free Software | |||
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, | |||
# MA 02110-1301, USA. | |||
# | |||
# | |||
import os | |||
import re | |||
import pywikibot | |||
import mwparserfromhell as mwparser | |||
from config import Config | |||
class DescPageBot(pywikibot.bot.Bot): | |||
""" | |||
Updates file description page with EuroExchangeBot related content | |||
""" | |||
def treat_job( self, job ): | |||
""" | |||
Handles work on file description page for given job | |||
@param job Job to work on | |||
@type EuroExchangeBotJob | |||
""" | |||
# Store job | |||
self.job = job | |||
# Store file page object | |||
self.current_page = job.filepage | |||
# Parse filepage | |||
self.parse_page() | |||
# Update gnuplot script | |||
self.update_gnuplot_script() | |||
# Update wiki page | |||
self.update_page() | |||
def load_gnuplot_script(self): | |||
""" | |||
Load the gnuplot script for current job | |||
@return Gnuplot script content | |||
@rtype str | |||
""" | |||
with open( os.path.join( | |||
Config.gnuplot_script_dir, self.job.script + ".plt" ), "r") as fd: | |||
return fd.read() | |||
def prepare_gnuplot_script(self): | |||
""" | |||
Prepare gnuplot script code for publishing on image description page | |||
""" | |||
# Load gnuplot script | |||
gnuplot_script = self.load_gnuplot_script() | |||
# Strip leadig and trailing whitespace | |||
gnuplot_script = gnuplot_script.strip(" \n") | |||
# Replace | |||
gnuplot_script = gnuplot_script.\ | |||
replace( "system(\"echo $INFILE\")", | |||
"'{}'".format( os.path.basename( Config.csv_file ) ) ).\ | |||
replace( "system(\"echo $OUTFILE\")", | |||
"'{}'".format (os.path.basename( self.job.image ) ) ) | |||
# Locate first empty line | |||
m = re.search(r"^\s*$", gnuplot_script, re.MULTILINE) | |||
if m: | |||
# Insert help lines | |||
gnuplot_script = gnuplot_script[:m.end()] +\ | |||
Config.gnuplot_script_help + gnuplot_script[m.end():] | |||
return gnuplot_script | |||
def parse_page(self): | |||
""" | |||
Load current page content and parse with mwparser | |||
""" | |||
self.current_page.wikicode = mwparser.parse(self.current_page.text) | |||
def update_gnuplot_script(self): | |||
""" | |||
Update the gnuplot script embedded in page | |||
""" | |||
# Get source tag with gnuplot script | |||
gnuplot_script = next( | |||
self.current_page.wikicode.ifilter_tags( | |||
matches="<source lang=\"gnuplot\">" ) ) | |||
# Replace script | |||
gnuplot_script.contents = "\n" + self.prepare_gnuplot_script() + "\n" | |||
def update_page(self): | |||
""" | |||
Put updated content to wiki | |||
""" | |||
# Convert wikicode back to str | |||
new_text = str(self.current_page.wikicode) | |||
# Save new text | |||
self.userPut( self.current_page, | |||
self.current_page.text, | |||
new_text, | |||
summary=Config.gnuplot_script_comment ) |
@@ -33,6 +33,8 @@ import pywikibot.specialbots | |||
import jogobot | |||
from config import Config | |||
from descpage import DescPageBot | |||
class EuroExchangeBotJob(): | |||
""" | |||
@@ -53,20 +55,14 @@ class EuroExchangeBotJob(): | |||
class EuroExchangeBot( pywikibot.bot.BaseBot ): | |||
base_dir = os.path.expanduser(jogobot.config["euroexchange"]["base_dir"]) | |||
working_dir = os.path.join( base_dir, "working_dir" ) | |||
gnuplot_script_dir = os.path.join(base_dir, "gnuplot_scripts") | |||
gnuplot = jogobot.config["euroexchange"]["gnuplot_bin"] | |||
data_source = jogobot.config["euroexchange"]["data_source"] | |||
zip_file = jogobot.config["euroexchange"]["data_zip_filename"] | |||
csv_file = jogobot.config["euroexchange"]["data_csv_filename"] | |||
upload_comment = jogobot.config["euroexchange"]["upload_comment"] | |||
def __init__( self, genFactory, **kwargs ): | |||
# Init working directory | |||
self.init_wdir() | |||
# Prepare DescPage editing bot | |||
self.descpagebot = DescPageBot( **kwargs ) | |||
super().__init__(**kwargs) | |||
def run(self): | |||
@@ -85,7 +81,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ): | |||
""" | |||
#Normalize working dir | |||
self.wdir = os.path.realpath(type(self).working_dir) | |||
self.wdir = os.path.realpath(Config.working_dir) | |||
if os.path.exists(self.wdir): | |||
@@ -108,7 +104,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ): | |||
""" | |||
# Check if zip file exists | |||
if os.path.exists( os.path.join(self.wdir, type(self).zip_file) ): | |||
if os.path.exists( os.path.join(self.wdir, Config.zip_file) ): | |||
# If file is outdated, remove data input files | |||
if not self.is_zip_uptodate(): | |||
@@ -132,7 +128,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ): | |||
@rtype bool | |||
""" | |||
# Get file stat | |||
stat = os.stat( os.path.join(self.wdir, type(self).zip_file) ) | |||
stat = os.stat( os.path.join(self.wdir, Config.zip_file) ) | |||
# Get file modification datetime | |||
mdt = datetime.datetime.fromtimestamp( stat.st_mtime ) | |||
@@ -158,8 +154,8 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ): | |||
Deletes data input files | |||
""" | |||
input_files = ( os.path.join(self.wdir, type(self).zip_file), | |||
os.path.join(self.wdir, type(self).csv_file) ) | |||
input_files = ( os.path.join(self.wdir, Config.zip_file), | |||
os.path.join(self.wdir, Config.csv_file) ) | |||
for f in input_files: | |||
os.remove( f ) | |||
@@ -170,9 +166,9 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ): | |||
Download the zipfile from EZB | |||
""" | |||
# Download the file and save it locally | |||
with urllib.request.urlopen(type(self).data_source) as response,\ | |||
with urllib.request.urlopen(Config.data_source) as response,\ | |||
open( os.path.join(self.wdir, | |||
type(self).zip_file), 'wb') as out_file: | |||
Config.zip_file), 'wb') as out_file: | |||
shutil.copyfileobj(response, out_file) | |||
@@ -182,7 +178,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ): | |||
response.info()["Last-Modified"]) | |||
# Set ctime to value from http header | |||
os.utime( os.path.join(self.wdir, type(self).zip_file), | |||
os.utime( os.path.join(self.wdir, Config.zip_file), | |||
(datetime.datetime.now().timestamp(), mdate.timestamp()) ) | |||
# Log | |||
@@ -192,14 +188,14 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ): | |||
""" | |||
Extract csv file from zip archive | |||
""" | |||
if not os.path.exists( os.path.join(self.wdir, type(self).csv_file) ): | |||
if not os.path.exists( os.path.join(self.wdir, Config.csv_file) ): | |||
with zipfile.ZipFile( | |||
os.path.join(self.wdir, type(self).zip_file)) as zipobj: | |||
os.path.join(self.wdir, Config.zip_file)) as zipobj: | |||
zipobj.extract( | |||
os.path.basename( | |||
os.path.join(self.wdir, type(self).csv_file)), | |||
os.path.join(self.wdir, Config.csv_file)), | |||
path=self.wdir ) | |||
def load_jobs( self ): | |||
@@ -211,7 +207,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ): | |||
""" | |||
# Load json jobs file | |||
with open( os.path.join(self.base_dir, "jobs.json"), "r") as fd: | |||
with open( os.path.join(Config.base_dir, "jobs.json"), "r") as fd: | |||
jobs_js = json.load( fd ) | |||
# yield each job | |||
@@ -247,19 +243,27 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ): | |||
# Check if update is necessary | |||
if self.image_update_needed(): | |||
self.call_gnuplot( job ) | |||
try: | |||
self.call_gnuplot( job ) | |||
if self.file_changed(): | |||
self.upload_file( job ) | |||
else: | |||
jogobot.output( "No upload needed for Job {}.".format( | |||
self.current_job.image) ) | |||
if self.file_changed(): | |||
self.upload_file( job ) | |||
else: | |||
jogobot.output( "No upload needed for Job {}.".format( | |||
self.current_job.image) ) | |||
except subprocess.CalledProcessError as e: | |||
jogobot.output( "Subprocess terminated with exit code {}!". | |||
format( e.returncode), "ERROR" ) | |||
# Nothing to do | |||
else: | |||
jogobot.output( "No update needed for Job {}".format( | |||
self.current_job.image) ) | |||
# Update file description page | |||
self.descpagebot.treat_job( self.current_job ) | |||
def image_update_needed( self ): | |||
""" | |||
Checks weather image update intervall is reached. | |||
@@ -289,15 +293,15 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ): | |||
@type job: EuroExchangeBotJob | |||
""" | |||
cmd = shlex.split ( type(self).gnuplot + " " + os.path.realpath( | |||
os.path.join( type(self).gnuplot_script_dir, | |||
cmd = shlex.split ( Config.gnuplot + " " + os.path.realpath( | |||
os.path.join( Config.gnuplot_script_dir, | |||
job.script + ".plt" ) ) ) | |||
plt_env = os.environ.copy() | |||
plt_env["INFILE"] = type(self).csv_file | |||
plt_env["INFILE"] = Config.csv_file | |||
plt_env["OUTFILE"] = job.image | |||
subprocess.call( cmd, cwd=self.wdir, env=plt_env ) | |||
subprocess.check_call( cmd, cwd=self.wdir, env=plt_env ) | |||
def file_changed( self ): | |||
""" | |||
@@ -326,7 +330,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ): | |||
@type job: EuroExchangeBotJob | |||
""" | |||
comment = type(self).upload_comment | |||
comment = Config.upload_comment | |||
filename = job.image | |||
filepath = [ os.path.join(self.wdir, job.image) ] | |||