Browse Source

Drop pycurl

pull/58/head
Dashie der otter 11 months ago
parent
commit
7ac23ef340
Signed by: dashie <rhaamo@leloop.org> GPG Key ID: C2D57B325840B755
5 changed files with 26 additions and 33 deletions
  1. +0
    -7
      docs/install.md
  2. +0
    -1
      requirements.txt
  3. +0
    -1
      setup.py
  4. +20
    -0
      update_db.py
  5. +6
    -24
      utils.py

+ 0
- 7
docs/install.md View File

@@ -1,12 +1,5 @@
# Installation

# OSX Specificities
Uses:

PYCURL_SSL_LIBRARY=openssl LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" pip install --no-cache-dir pycurl
To install pycurl before the requirements.

# Prerequirements

sudo apt install build-essential libcurl4-openssl-dev libssl-dev libncursesw5-dev libsqlite3-dev libbz2-dev zlib1g-dev


+ 0
- 1
requirements.txt View File

@@ -14,7 +14,6 @@ sqlalchemy_searchable==1.0.3
flask_debugtoolbar==0.10.1
redis==3.2.1
Bootstrap-Flask==1.0.10
pycurl
pyais==1.0.0
Flask-SocketIO==4.2.1
eventlet==0.25.1

+ 0
- 1
setup.py View File

@@ -29,7 +29,6 @@ setup(
"python-magic",
"redis",
"flask-socketio",
"pycurl",
"eventlet",
],
setup_requires=["pytest-runner"],


+ 20
- 0
update_db.py View File

@@ -29,6 +29,8 @@ def update_aircrafts():
print(f"Error: file at path {file_path} doesn't exist !")
return False

ingested = 0

print("Cleaning old datas...")
# Remove old datas
db.session.query(Aircrafts).delete()
@@ -55,9 +57,11 @@ def update_aircrafts():
ac.wake_category = row[8]
ac.mfr = row[9]
db.session.add(ac)
ingested += 1
db.session.commit()

print("Done.")
print(f"Ingested {ingested} items.")


def update_registrations():
@@ -68,6 +72,8 @@ def update_registrations():
print(f"Error: file at path {file_path} doesn't exist !")
return False

ingested = 0

print("Cleaning old datas...")
# Remove old datas
db.session.query(AircraftRegistration).delete()
@@ -85,12 +91,18 @@ def update_registrations():
acr.country = row[0]
acr.prefix = row[1]
db.session.add(acr)
ingested += 1
db.session.commit()

print("Done.")
print(f"Ingested {ingested} items.")


def update_translation():
"""
FIXME: finish this function
:return:
"""
print("Downloading ACARS translations...")
dl_infos = URLS["translation"]
downloaded = download_and_extract_zip(dl_infos["url"], dl_infos["filename"], "translation.csv")
@@ -123,6 +135,8 @@ def update_mode_s():
print("An error occured while downloading ModeS datas")
return False

ingested = 0

print("Cleaning old datas...")

# Remove old datas
@@ -150,6 +164,7 @@ def update_mode_s():
acm.source = fname
acm.type_flight = "military" if row["UserString4"] == "M" else None
db.session.add(acm)
ingested += 1

# Add Aircraft Owner only if not empty, and not Private individual
if (
@@ -174,6 +189,7 @@ def update_mode_s():
)

print("Done.")
print(f"Ingested {ingested} items.")


def update_mode_s_ogn():
@@ -185,6 +201,8 @@ def update_mode_s_ogn():
print("An error occured while downloading ModeS OGN datas")
return False

ingested = 0

print("Cleaning old datas...")
# Remove old datas
db.session.query(AircraftModes).filter(
@@ -223,6 +241,7 @@ def update_mode_s_ogn():
if acm.registration != "" and acm.registration != "0000" and acm.icao_type_code != "":
acm.last_modified = datetime.datetime.utcnow()
db.session.add(acm)
ingested += 1

db.session.commit()

@@ -232,6 +251,7 @@ def update_mode_s_ogn():
db.session.query(AircraftModes).filter(and_(AircraftModes.source == fname, AircraftModes.mode_s.in_(q))).delete(
synchronize_session="fetch"
)
print(f"Ingested {ingested} items.")


def update_all():


+ 6
- 24
utils.py View File

@@ -4,8 +4,9 @@ import os
from os.path import splitext
import csv
import zipfile
import pycurl
import urllib.request
import gzip
import shutil

from flask import current_app

@@ -139,12 +140,8 @@ def utils_download_and_ingest_faa():
except OSError:
None

with open(file_target, "wb") as zip_file:
crl = pycurl.Curl()
crl.setopt(crl.URL, current_app.config["FAA_ARCHIVE_URL"])
crl.setopt(crl.WRITEDATA, zip_file)
crl.perform()
crl.close()
with urllib.request.urlopen(current_app.config["FAA_ARCHIVE_URL"]) as response, open(file_target, "wb") as zip_file:
shutil.copyfileobj(response, zip_file)

print("Extracting files...")

@@ -177,23 +174,8 @@ def download_file(url: str, filename: str):
except OSError:
None

try:
with open(file_target, "wb") as zip_file:
crl = pycurl.Curl()
crl.setopt(crl.URL, url)
crl.setopt(crl.WRITEDATA, zip_file)
crl.setopt(crl.FOLLOWLOCATION, True)
crl.perform()
code = crl.getinfo(pycurl.HTTP_CODE)
if code != 200:
print(f"Error, received code !200: {code}")
crl.close()
return False
except pycurl.error as e:
print(f"Error while downloading file: {e}")
return False
finally:
crl.close()
with urllib.request.urlopen(url) as response, open(file_target, "wb") as zip_file:
shutil.copyfileobj(response, zip_file)

try:
if os.path.getsize(file_target) > 0:


Loading…
Cancel
Save