Kaynağa Gözat

Merge branch 'release-0.2'

master v0.2
Jonathan Golder 6 yıl önce
ebeveyn
işleme
d37f4dbbd1
  1. 133
      .gitignore
  2. 4
      README.md
  3. 47
      euroexchange/config.py
  4. 129
      euroexchange/descpage.py
  5. 68
      euroexchange/euroexchange.py

133
.gitignore

@ -0,0 +1,133 @@
# Created by https://www.gitignore.io/api/python
### Python ###
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
.hypothesis/
.pytest_cache/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
.python-version
# celery beat schedule file
celerybeat-schedule
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
### Python Patch ###
.venv/
### Python.VirtualEnv Stack ###
# Virtualenv
# http://iamzed.com/2009/05/07/a-primer-on-virtualenv/
[Bb]in
[Ii]nclude
[Ll]ib
[Ll]ib64
[Ll]ocal
[Ss]cripts
pyvenv.cfg
pip-selfcheck.json
# End of https://www.gitignore.io/api/python

4
README.md

@ -21,6 +21,10 @@ python euroexchange.py
* 0.1.1
- Bugfix: only update image if update period is reached
* 0.2
- Checks gnuplot exitcode and only upload chart if successfull created
- Update gnuplot script on commons file description page
## Bugs
[jogobot-euroexchange Issues](https://git.golderweb.de/wiki/jogobot-euroexchange/issues)

47
euroexchange/config.py

@ -0,0 +1,47 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# config.py
#
# Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
#
#
"""
Loads configuration of EuroExchangeBot
"""
import os
import jogobot
class Config():
"""
Configuration container for EuroExchangeBot
"""
base_dir = os.path.expanduser(jogobot.config["euroexchange"]["base_dir"])
working_dir = os.path.join( base_dir, "working_dir" )
gnuplot_script_dir = os.path.join(base_dir, "gnuplot_scripts")
gnuplot = jogobot.config["euroexchange"]["gnuplot_bin"]
data_source = jogobot.config["euroexchange"]["data_source"]
zip_file = jogobot.config["euroexchange"]["data_zip_filename"]
csv_file = jogobot.config["euroexchange"]["data_csv_filename"]
upload_comment = jogobot.config["euroexchange"]["upload_comment"]
gnuplot_script_comment = jogobot.config["euroexchange"]["gnuplot_script_comment"]
gnuplot_script_help = jogobot.config["euroexchange"]["gnuplot_script_help"]

129
euroexchange/descpage.py

@ -0,0 +1,129 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# descpage.py
#
# Copyright 2018 Jonathan Golder <jonathan@golderweb.de>
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA 02110-1301, USA.
#
#
import os
import re
import pywikibot
import mwparserfromhell as mwparser
from config import Config
class DescPageBot(pywikibot.bot.Bot):
"""
Updates file description page with EuroExchangeBot related content
"""
def treat_job( self, job ):
"""
Handles work on file description page for given job
@param job Job to work on
@type EuroExchangeBotJob
"""
# Store job
self.job = job
# Store file page object
self.current_page = job.filepage
# Parse filepage
self.parse_page()
# Update gnuplot script
self.update_gnuplot_script()
# Update wiki page
self.update_page()
def load_gnuplot_script(self):
"""
Load the gnuplot script for current job
@return Gnuplot script content
@rtype str
"""
with open( os.path.join(
Config.gnuplot_script_dir, self.job.script + ".plt" ), "r") as fd:
return fd.read()
def prepare_gnuplot_script(self):
"""
Prepare gnuplot script code for publishing on image description page
"""
# Load gnuplot script
gnuplot_script = self.load_gnuplot_script()
# Strip leadig and trailing whitespace
gnuplot_script = gnuplot_script.strip(" \n")
# Replace
gnuplot_script = gnuplot_script.\
replace( "system(\"echo $INFILE\")",
"'{}'".format( os.path.basename( Config.csv_file ) ) ).\
replace( "system(\"echo $OUTFILE\")",
"'{}'".format (os.path.basename( self.job.image ) ) )
# Locate first empty line
m = re.search(r"^\s*$", gnuplot_script, re.MULTILINE)
if m:
# Insert help lines
gnuplot_script = gnuplot_script[:m.end()] +\
Config.gnuplot_script_help + gnuplot_script[m.end():]
return gnuplot_script
def parse_page(self):
"""
Load current page content and parse with mwparser
"""
self.current_page.wikicode = mwparser.parse(self.current_page.text)
def update_gnuplot_script(self):
"""
Update the gnuplot script embedded in page
"""
# Get source tag with gnuplot script
gnuplot_script = next(
self.current_page.wikicode.ifilter_tags(
matches="<source lang=\"gnuplot\">" ) )
# Replace script
gnuplot_script.contents = "\n" + self.prepare_gnuplot_script() + "\n"
def update_page(self):
"""
Put updated content to wiki
"""
# Convert wikicode back to str
new_text = str(self.current_page.wikicode)
# Save new text
self.userPut( self.current_page,
self.current_page.text,
new_text,
summary=Config.gnuplot_script_comment )

68
euroexchange/euroexchange.py

@ -33,6 +33,8 @@ import pywikibot.specialbots
import jogobot
from config import Config
from descpage import DescPageBot
class EuroExchangeBotJob():
"""
@ -53,20 +55,14 @@ class EuroExchangeBotJob():
class EuroExchangeBot( pywikibot.bot.BaseBot ):
base_dir = os.path.expanduser(jogobot.config["euroexchange"]["base_dir"])
working_dir = os.path.join( base_dir, "working_dir" )
gnuplot_script_dir = os.path.join(base_dir, "gnuplot_scripts")
gnuplot = jogobot.config["euroexchange"]["gnuplot_bin"]
data_source = jogobot.config["euroexchange"]["data_source"]
zip_file = jogobot.config["euroexchange"]["data_zip_filename"]
csv_file = jogobot.config["euroexchange"]["data_csv_filename"]
upload_comment = jogobot.config["euroexchange"]["upload_comment"]
def __init__( self, genFactory, **kwargs ):
# Init working directory
self.init_wdir()
# Prepare DescPage editing bot
self.descpagebot = DescPageBot( **kwargs )
super().__init__(**kwargs)
def run(self):
@ -85,7 +81,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
"""
#Normalize working dir
self.wdir = os.path.realpath(type(self).working_dir)
self.wdir = os.path.realpath(Config.working_dir)
if os.path.exists(self.wdir):
@ -108,7 +104,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
"""
# Check if zip file exists
if os.path.exists( os.path.join(self.wdir, type(self).zip_file) ):
if os.path.exists( os.path.join(self.wdir, Config.zip_file) ):
# If file is outdated, remove data input files
if not self.is_zip_uptodate():
@ -132,7 +128,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
@rtype bool
"""
# Get file stat
stat = os.stat( os.path.join(self.wdir, type(self).zip_file) )
stat = os.stat( os.path.join(self.wdir, Config.zip_file) )
# Get file modification datetime
mdt = datetime.datetime.fromtimestamp( stat.st_mtime )
@ -158,8 +154,8 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
Deletes data input files
"""
input_files = ( os.path.join(self.wdir, type(self).zip_file),
os.path.join(self.wdir, type(self).csv_file) )
input_files = ( os.path.join(self.wdir, Config.zip_file),
os.path.join(self.wdir, Config.csv_file) )
for f in input_files:
os.remove( f )
@ -170,9 +166,9 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
Download the zipfile from EZB
"""
# Download the file and save it locally
with urllib.request.urlopen(type(self).data_source) as response,\
with urllib.request.urlopen(Config.data_source) as response,\
open( os.path.join(self.wdir,
type(self).zip_file), 'wb') as out_file:
Config.zip_file), 'wb') as out_file:
shutil.copyfileobj(response, out_file)
@ -182,7 +178,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
response.info()["Last-Modified"])
# Set ctime to value from http header
os.utime( os.path.join(self.wdir, type(self).zip_file),
os.utime( os.path.join(self.wdir, Config.zip_file),
(datetime.datetime.now().timestamp(), mdate.timestamp()) )
# Log
@ -192,14 +188,14 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
"""
Extract csv file from zip archive
"""
if not os.path.exists( os.path.join(self.wdir, type(self).csv_file) ):
if not os.path.exists( os.path.join(self.wdir, Config.csv_file) ):
with zipfile.ZipFile(
os.path.join(self.wdir, type(self).zip_file)) as zipobj:
os.path.join(self.wdir, Config.zip_file)) as zipobj:
zipobj.extract(
os.path.basename(
os.path.join(self.wdir, type(self).csv_file)),
os.path.join(self.wdir, Config.csv_file)),
path=self.wdir )
def load_jobs( self ):
@ -211,7 +207,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
"""
# Load json jobs file
with open( os.path.join(self.base_dir, "jobs.json"), "r") as fd:
with open( os.path.join(Config.base_dir, "jobs.json"), "r") as fd:
jobs_js = json.load( fd )
# yield each job
@ -247,19 +243,27 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
# Check if update is necessary
if self.image_update_needed():
self.call_gnuplot( job )
try:
self.call_gnuplot( job )
if self.file_changed():
self.upload_file( job )
else:
jogobot.output( "No upload needed for Job {}.".format(
self.current_job.image) )
if self.file_changed():
self.upload_file( job )
else:
jogobot.output( "No upload needed for Job {}.".format(
self.current_job.image) )
except subprocess.CalledProcessError as e:
jogobot.output( "Subprocess terminated with exit code {}!".
format( e.returncode), "ERROR" )
# Nothing to do
else:
jogobot.output( "No update needed for Job {}".format(
self.current_job.image) )
# Update file description page
self.descpagebot.treat_job( self.current_job )
def image_update_needed( self ):
"""
Checks weather image update intervall is reached.
@ -289,15 +293,15 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
@type job: EuroExchangeBotJob
"""
cmd = shlex.split ( type(self).gnuplot + " " + os.path.realpath(
os.path.join( type(self).gnuplot_script_dir,
cmd = shlex.split ( Config.gnuplot + " " + os.path.realpath(
os.path.join( Config.gnuplot_script_dir,
job.script + ".plt" ) ) )
plt_env = os.environ.copy()
plt_env["INFILE"] = type(self).csv_file
plt_env["INFILE"] = Config.csv_file
plt_env["OUTFILE"] = job.image
subprocess.call( cmd, cwd=self.wdir, env=plt_env )
subprocess.check_call( cmd, cwd=self.wdir, env=plt_env )
def file_changed( self ):
"""
@ -326,7 +330,7 @@ class EuroExchangeBot( pywikibot.bot.BaseBot ):
@type job: EuroExchangeBotJob
"""
comment = type(self).upload_comment
comment = Config.upload_comment
filename = job.image
filepath = [ os.path.join(self.wdir, job.image) ]

Yükleniyor…
İptal
Kaydet