Browse Source

Refactoring of app.py to use the newer Flask CLI, also adapted for unittests

pull/37/head
Dashie der otter 11 months ago
parent
commit
a668ce3339
Signed by: Dashie <dashie@sigpipe.me> GPG Key ID: C2D57B325840B755
5 changed files with 293 additions and 202 deletions
  1. 18
    10
      README.md
  2. 0
    142
      ahrl.py
  3. 271
    47
      app.py
  4. 3
    3
      tests/config_tests.py
  5. 1
    0
      version.py

+ 18
- 10
README.md View File

@@ -7,7 +7,8 @@ Another Ham Radio Log


# Versions requirement
- Python >= 3.3 (3.0, 3.1, 3.2 not supported)
- Python >= 3.6 (all under 3.6 are not supported) (say bye-bye to debian stable, sorry)
- try https://github.com/chriskuehl/python3.6-debian-stretch if you use debian stable

# Installation
Install a BDD (mysql is supported, SQLite maybe, PostgreSQL should be)
@@ -16,19 +17,26 @@ Another Ham Radio Log
cd ahrl
git submodule init
git submodule update
pip3 install git+https://github.com/ggramaize/libqth.git
pip3 install git+http://dev.sigpipe.me/DashieHam/pyHamQth.git
pip3 install --requirement requirements.txt # if present
pip3 install --requirement requirements.txt
cp config.py.sample config.py
$EDITOR config.py
python3 ahrl.py db upgrade
python3 ahrl.py db seed
python3 ahrl.py cron update_dxcc_from_cty
python3 ahrl.py runserver # or whatever gunicorn whatever stuff
export FLASK_ENV=<development or production>
$ create your postgresql database like "ahrl"
flask db upgrade
flask seed
flask cron update_dxcc_from_cty
flask run
Don't forget to update default Config by getting to "Your user" (top right) then "Config"

# Gunicorn
gunicorn -w 2 -b 127.0.0.1:8000 --error-logfile=errors.log --access-logfile=access.log --chdir=$PWD ahrl:app
# Creating an user

If you have enabled registration in config, the first user registered will be ADMIN !

Or if you have disabled registration, use the ``` flask createuser ``` command to create an user.

# Production running

TODO: venv, systemd services

# Default config
- LOTW Download URL: https://p1k.arrl.org/lotwuser/lotwreport.adi

+ 0
- 142
ahrl.py View File

@@ -1,142 +0,0 @@
# encoding: utf-8
from pprint import pprint as pp
import datetime
import os
import texttable
from flask_debugtoolbar import DebugToolbarExtension
from flask_migrate import MigrateCommand
from flask_script import Manager
from crons import (
update_qsos_without_countries,
update_dxcc_from_cty_xml,
populate_logs_gridsquare_cache,
cron_sync_eqsl,
update_qsos_from_hamqth,
cron_sync_from_eqsl,
)
from dbseed import make_db_seed
from models import db

try:
from raven.contrib.flask import Sentry
import raven

print(" * Sentry support loaded")
HAS_SENTRY = True
except ImportError:
print(" * No Sentry support")
HAS_SENTRY = False

from app import create_app

app = create_app()

if HAS_SENTRY:
app.config["SENTRY_RELEASE"] = raven.fetch_git_sha(os.path.dirname(__file__))
sentry = Sentry(app, dsn=app.config["SENTRY_DSN"])
print(" * Sentry support activated")
print(" * Sentry DSN: %s" % app.config["SENTRY_DSN"])

toolbar = DebugToolbarExtension(app)
manager = Manager(app)


# Other commands
@manager.command
def routes():
"""Dump all routes of defined app"""
table = texttable.Texttable()
table.set_deco(texttable.Texttable().HEADER)
table.set_cols_dtype(["t", "t", "t"])
table.set_cols_align(["l", "l", "l"])
table.set_cols_width([60, 30, 90])

table.add_rows([["Prefix", "Verb", "URI Pattern"]])

for rule in sorted(app.url_map.iter_rules(), key=lambda x: str(x)):
methods = ",".join(rule.methods)
table.add_row([rule.endpoint, methods, rule])

print(table.draw())


@manager.command
def config():
"""Dump config"""
pp(app.config)


@MigrateCommand.command
def seed():
"""Seed database with default content"""
make_db_seed(db)


CacheCommand = Manager(usage="Perform cache actions")
CronCommand = Manager(usage="Perform crons actions")


@CronCommand.command
def update_dxcc_from_cty():
"""Update DXCC tables from cty.xml"""
print("-- STARTED on {0}".format(datetime.datetime.now()))
update_dxcc_from_cty_xml()
print("-- FINISHED on {0}".format(datetime.datetime.now()))


@CronCommand.command
def update_qsos_countries():
"""Update QSOs with empty country"""
print("-- STARTED on {0}".format(datetime.datetime.now()))
update_qsos_without_countries()
print("-- FINISHED on {0}".format(datetime.datetime.now()))


@CronCommand.command
@CronCommand.option(
"--dryrun", dest="dry_run", action="store_true", default=False, help="Dry run, doesn't commit anything"
)
def sync_to_eqsl(dry_run=False):
"""Push to eQSL logs with requested eQSL sync"""
print("-- STARTED on {0}".format(datetime.datetime.now()))
cron_sync_eqsl(dry_run)
print("-- FINISHED on {0}".format(datetime.datetime.now()))


@CronCommand.command
@CronCommand.option(
"--dryrun", dest="dry_run", action="store_true", default=False, help="Dry run, doesn't commit anything"
)
def sync_from_eqsl(dry_run=False):
"""Fetch from eQSL logs """
print("-- STARTED on {0}".format(datetime.datetime.now()))
cron_sync_from_eqsl(dry_run)
print("-- FINISHED on {0}".format(datetime.datetime.now()))


@CronCommand.command
def update_qsos_hamqth():
"""Update QSOs with datas from HamQTH"""
print("-- STARTED on {0}".format(datetime.datetime.now()))
update_qsos_from_hamqth()
print("-- FINISHED on {0}".format(datetime.datetime.now()))


@CacheCommand.command
def populate_logs_gridsquare():
"""Update QSOs with empty gridsquare cache"""
print("-- STARTED on {0}".format(datetime.datetime.now()))
populate_logs_gridsquare_cache()
print("-- FINISHED on {0}".format(datetime.datetime.now()))


manager.add_command("db", MigrateCommand)
manager.add_command("cache", CacheCommand)
manager.add_command("cron", CronCommand)

if __name__ == "__main__":
try:
manager.run()
except KeyboardInterrupt as e:
print("Got KeyboardInterrupt, halting...")
print(e)

+ 271
- 47
app.py View File

@@ -1,47 +1,78 @@
# encoding: utf-8
import logging
import os
import subprocess
from logging.handlers import RotatingFileHandler

from flask import Flask, render_template, g, send_from_directory, jsonify
from flask import Flask, render_template, g, send_from_directory, jsonify, request, safe_join, Response
from flask_bootstrap import Bootstrap
from flask_mail import Mail
from flask_migrate import Migrate
from flask_security import Security
from flask_uploads import configure_uploads, UploadSet, IMAGES

from controllers.admin import bp_admin
from controllers.contacts import bp_contacts
from controllers.logbooks import bp_logbooks
from controllers.main import bp_main
from controllers.notes import bp_notes
from controllers.qsos import bp_qsos
from controllers.tools import bp_tools
from controllers.users import bp_users
from controllers.extapi import bp_extapi
from flask_security import signals as FlaskSecuritySignals
from flask_security import confirmable as FSConfirmable
from flask_security.utils import encrypt_password
from flask_uploads import configure_uploads, UploadSet, IMAGES, patch_request_class
from flask_babelex import gettext, Babel

from forms import ExtendedRegisterForm
from models import db, user_datastore
from models import user_datastore, Role
from utils import dt_utc_to_user_tz, InvalidUsage, show_date_no_offset, is_admin
from pprint import pprint as pp
import datetime
import os
import texttable
from flask_debugtoolbar import DebugToolbarExtension
from crons import (
update_qsos_without_countries,
update_dxcc_from_cty_xml,
populate_logs_gridsquare_cache,
cron_sync_eqsl,
update_qsos_from_hamqth,
cron_sync_from_eqsl,
)
from dbseed import make_db_seed
from models import db
import click

from version import VERSION

__VERSION__ = VERSION

try:
from raven.contrib.flask import Sentry
import raven

print(" * Sentry support loaded")
HAS_SENTRY = True
except ImportError:
print(" * No sentry support")
HAS_SENTRY = False

__VERSION__ = "0.0.1"
mail = Mail()


def create_app(cfg=None):
# App Configuration
if cfg is None:
cfg = {}
app = Flask(__name__)
app.config.from_pyfile("config.py")
app.config.update(cfg)
def create_app(config_filename="config.py", app_name=None, register_blueprints=True):
# App configuration
app = Flask(app_name or __name__)
app.config.from_pyfile(config_filename)

Bootstrap(app)

app.jinja_env.add_extension("jinja2.ext.with_")
app.jinja_env.add_extension("jinja2.ext.do")
app.jinja_env.globals.update(is_admin=is_admin)
app.jinja_env.filters["localize"] = dt_utc_to_user_tz
app.jinja_env.filters["show_date_no_offset"] = show_date_no_offset
app.jinja_env.globals.update(is_admin=is_admin)

if HAS_SENTRY:
app.config["SENTRY_RELEASE"] = raven.fetch_git_sha(os.path.dirname(__file__))
sentry = Sentry(app, dsn=app.config["SENTRY_DSN"]) # noqa: F841
print(" * Sentry support activated")
print(" * Sentry DSN: %s" % app.config["SENTRY_DSN"])

if app.config["DEBUG"] is True:
app.jinja_env.auto_reload = True
app.logger.setLevel(logging.DEBUG)

# Logging
if not app.debug:
@@ -51,13 +82,30 @@ def create_app(cfg=None):
file_handler.setFormatter(formatter)
app.logger.addHandler(file_handler)

mail = Mail(app) # noqa: F841
mail.init_app(app)
migrate = Migrate(app, db) # noqa: F841
babel = Babel(app) # noqa: F841
toolbar = DebugToolbarExtension(app) # noqa: F841

db.init_app(app)

# Setup Flask-Security
security = Security(app, user_datastore, register_form=ExtendedRegisterForm) # noqa: F841
security = Security( # noqa: F841
app, user_datastore, register_form=ExtendedRegisterForm, confirm_register_form=ExtendedRegisterForm
)

@FlaskSecuritySignals.password_reset.connect_via(app)
@FlaskSecuritySignals.password_changed.connect_via(app)
def log_password_reset(sender, user):
if not user:
return
# add_user_log(user.id, user.id, "user", "info", "Your password has been changed !")

@FlaskSecuritySignals.reset_password_instructions_sent.connect_via(app)
def log_reset_password_instr(sender, user, token):
if not user:
return
# add_user_log(user.id, user.id, "user", "info", "Password reset instructions sent.")

git_version = ""
gitpath = os.path.join(os.getcwd(), ".git")
@@ -66,13 +114,31 @@ def create_app(cfg=None):
if git_version:
git_version = git_version.strip().decode("UTF-8")

@babel.localeselector
def get_locale():
# if a user is logged in, use the locale from the user settings
identity = getattr(g, "identity", None)
if identity is not None and identity.id:
return identity.user.locale
# otherwise try to guess the language from the user accept
# header the browser transmits. We support fr/en in this
# example. The best match wins.
return request.accept_languages.best_match(["fr", "en"])

@babel.timezoneselector
def get_timezone():
identity = getattr(g, "identity", None)
if identity is not None and identity.id:
return identity.user.timezone

@app.before_request
def before_request():
g.cfg = {
"AHRL_VERSION_VER": __VERSION__,
cfg = {
"AHRL_VERSION_VER": VERSION,
"AHRL_VERSION_GIT": git_version,
"AHRL_VERSION": "{0} ({1})".format(__VERSION__, git_version),
"AHRL_VERSION": "{0} ({1})".format(VERSION, git_version),
}
g.cfg = cfg

@app.errorhandler(InvalidUsage)
def handle_invalid_usage(error):
@@ -82,43 +148,201 @@ def create_app(cfg=None):

pictures = UploadSet("pictures", IMAGES)
configure_uploads(app, pictures)
patch_request_class(app, 5 * 1024 * 1024) # 5m limit

if register_blueprints:
from controllers.admin import bp_admin

app.register_blueprint(bp_admin)

from controllers.contacts import bp_contacts

app.register_blueprint(bp_contacts)

from controllers.logbooks import bp_logbooks

app.register_blueprint(bp_logbooks)

from controllers.main import bp_main

app.register_blueprint(bp_main)

from controllers.notes import bp_notes

app.register_blueprint(bp_notes)

from controllers.qsos import bp_qsos

app.register_blueprint(bp_qsos)

from controllers.tools import bp_tools

app.register_blueprint(bp_main)
app.register_blueprint(bp_users)
app.register_blueprint(bp_notes)
app.register_blueprint(bp_qsos)
app.register_blueprint(bp_tools)
app.register_blueprint(bp_contacts)
app.register_blueprint(bp_logbooks)
app.register_blueprint(bp_admin)
app.register_blueprint(bp_extapi)

# Used in development
@app.route("/uploads/<path:stuff>", methods=["GET"])
def get_uploads_stuff(stuff):
print("Get {0} from {1}".format(stuff, app.config["UPLOADS_DEFAULT_DEST"]))
return send_from_directory(app.config["UPLOADS_DEFAULT_DEST"], stuff, as_attachment=False)
app.register_blueprint(bp_tools)

from controllers.users import bp_users

app.register_blueprint(bp_users)

from controllers.extapi import bp_extapi

app.register_blueprint(bp_extapi)

@app.route("/uploads/<string:thing>/<path:stuff>", methods=["GET"])
def get_uploads_stuff(thing, stuff):
if app.debug:
directory = safe_join(app.config["UPLOADS_DEFAULT_DEST"], thing)
app.logger.debug(f"serving {stuff} from {directory}")
return send_from_directory(directory, stuff, as_attachment=True)
else:
app.logger.debug(f"X-Accel-Redirect serving {stuff}")
resp = Response("")
resp.headers["Content-Disposition"] = f"attachment; filename={stuff}"
resp.headers["X-Accel-Redirect"] = f"/_protected/media/tracks/{thing}/{stuff}"
return resp

@app.errorhandler(404)
def page_not_found(msg):
pcfg = {"title": "Whoops, something failed.", "error": 404, "message": "Page not found", "e": msg}
pcfg = {
"title": gettext("Whoops, something failed."),
"error": 404,
"message": gettext("Page not found"),
"e": msg,
}
return render_template("error_page.jinja2", pcfg=pcfg), 404

@app.errorhandler(403)
def err_forbidden(msg):
pcfg = {"title": "Whoops, something failed.", "error": 403, "message": "Access forbidden", "e": msg}
pcfg = {
"title": gettext("Whoops, something failed."),
"error": 403,
"message": gettext("Access forbidden"),
"e": msg,
}
return render_template("error_page.jinja2", pcfg=pcfg), 403

@app.errorhandler(410)
def err_gone(msg):
pcfg = {"title": "Whoops, something failed.", "error": 410, "message": "Gone", "e": msg}
pcfg = {"title": gettext("Whoops, something failed."), "error": 410, "message": gettext("Gone"), "e": msg}
return render_template("error_page.jinja2", pcfg=pcfg), 410

if not app.debug:

@app.errorhandler(500)
def err_failed(msg):
pcfg = {"title": "Whoops, something failed.", "error": 500, "message": "Something is broken", "e": msg}
pcfg = {
"title": gettext("Whoops, something failed."),
"error": 500,
"message": gettext("Something is broken"),
"e": msg,
}
return render_template("error_page.jinja2", pcfg=pcfg), 500

@app.after_request
def set_x_powered_by(response):
response.headers["X-Powered-By"] = "ahrl"
return response

# Commands from Flask CLI

@app.cli.command()
def routes():
"""Dump all routes of defined app"""
table = texttable.Texttable()
table.set_deco(texttable.Texttable().HEADER)
table.set_cols_dtype(["t", "t", "t"])
table.set_cols_align(["l", "l", "l"])
table.set_cols_width([50, 30, 80])

table.add_rows([["Prefix", "Verb", "URI Pattern"]])

for rule in sorted(app.url_map.iter_rules(), key=lambda x: str(x)):
methods = ",".join(rule.methods)
table.add_row([rule.endpoint, methods, rule])

print(table.draw())

@app.cli.command()
def config():
"""Dump config"""
pp(app.config)

@app.cli.command()
def seed():
"""Seed database with default content"""
make_db_seed(db)

@app.cli.command()
def createuser():
"""Create an user"""
username = click.prompt("Username", type=str)
email = click.prompt("Email", type=str)
password = click.prompt("Password", type=str, hide_input=True, confirmation_prompt=True)
while True:
role = click.prompt("Role [admin/user]", type=str)
if role == "admin" or role == "user":
break

if click.confirm("Do you want to continue ?"):
role = Role.query.filter(Role.name == role).first()
if not role:
raise click.UsageError("Roles not present in database")
u = user_datastore.create_user(
name=username, email=email, password=encrypt_password(password), roles=[role]
)

db.session.commit()

if FSConfirmable.requires_confirmation(u):
FSConfirmable.send_confirmation_instructions(u)
print("Look at your emails for validation instructions.")

@app.cli.group()
def cron():
"""Commands to be run regullary"""
pass

@cron.command()
def update_dxcc_from_cty():
"""Update DXCC tables from cty.xml"""
print("-- STARTED on {0}".format(datetime.datetime.now()))
update_dxcc_from_cty_xml()
print("-- FINISHED on {0}".format(datetime.datetime.now()))

@cron.command()
def update_qsos_countries():
"""Update QSOs with empty country"""
print("-- STARTED on {0}".format(datetime.datetime.now()))
update_qsos_without_countries()
print("-- FINISHED on {0}".format(datetime.datetime.now()))

@cron.command()
@click.option("--dryrun", default=False, help="Dry run, doesn't commit anything")
def sync_to_eqsl(dryrun=False):
"""Push to eQSL logs with requested eQSL sync"""
print("-- STARTED on {0}".format(datetime.datetime.now()))
cron_sync_eqsl(dryrun)
print("-- FINISHED on {0}".format(datetime.datetime.now()))

@cron.command()
@click.option("--dryrun", default=False, help="Dry run, doesn't commit anything")
def sync_from_eqsl(dryrun=False):
"""Fetch from eQSL logs """
print("-- STARTED on {0}".format(datetime.datetime.now()))
cron_sync_from_eqsl(dryrun)
print("-- FINISHED on {0}".format(datetime.datetime.now()))

@cron.command()
def update_qsos_hamqth():
"""Update QSOs with datas from HamQTH"""
print("-- STARTED on {0}".format(datetime.datetime.now()))
update_qsos_from_hamqth()
print("-- FINISHED on {0}".format(datetime.datetime.now()))

@cron.command()
def populate_logs_gridsquare():
"""Update QSOs with empty gridsquare cache"""
print("-- STARTED on {0}".format(datetime.datetime.now()))
populate_logs_gridsquare_cache()
print("-- FINISHED on {0}".format(datetime.datetime.now()))

return app

+ 3
- 3
tests/config_tests.py View File

@@ -17,7 +17,7 @@ UPLOADS_DEFAULT_DEST = "/home/dashie/dev/ahrl/uploads"
TEMP_DOWNLOAD_FOLDER = "/home/dashie/dev/ahrl/tmp"

# If using sentry, set a DSN
SENTRY_USER_ATTRS = ['name', 'email']
SENTRY_USER_ATTRS = ["name", "email"]
SENTRY_DSN = ""

# Domain serving this app
@@ -39,7 +39,7 @@ SECURITY_REGISTERABLE = True
SECURITY_RECOVERABLE = True
# Salt used for password hashing
# Do not change after users have registered
SECURITY_PASSWORD_SALT = 'awooo'
SECURITY_PASSWORD_SALT = "awooo"
# Do not change after users have registered
SECRET_KEY = "ahahahahahahahahahaquack"

@@ -73,7 +73,7 @@ SQLALCHEMY_RECORD_QUERIES = True
# Do not disable, will breaks things
SECURITY_CHANGEABLE = True
# Password hash algorithm
SECURITY_PASSWORD_HASH = 'bcrypt'
SECURITY_PASSWORD_HASH = "bcrypt"
BABEL_DEFAULT_TIMEZONE = "UTC"



+ 1
- 0
version.py View File

@@ -0,0 +1 @@
VERSION = "0.0.1"

Loading…
Cancel
Save