Drop pycurl

docs/install.md

@@ -1,12 +1,5 @@
 # Installation
 
-# OSX Specificities
-Uses:
-
-    PYCURL_SSL_LIBRARY=openssl LDFLAGS="-L/usr/local/opt/openssl/lib" CPPFLAGS="-I/usr/local/opt/openssl/include" pip install --no-cache-dir pycurl
-    
-To install pycurl before the requirements.
-
 # Prerequirements
 
     sudo apt install build-essential libcurl4-openssl-dev libssl-dev libncursesw5-dev libsqlite3-dev libbz2-dev zlib1g-dev

requirements.txt

@@ -14,7 +14,6 @@ sqlalchemy_searchable==1.0.3
 flask_debugtoolbar==0.10.1
 redis==3.2.1
 Bootstrap-Flask==1.0.10
-pycurl
 pyais==1.0.0
 Flask-SocketIO==4.2.1
 eventlet==0.25.1

setup.py

@@ -29,7 +29,6 @@ setup(
         "python-magic",
         "redis",
         "flask-socketio",
-        "pycurl",
         "eventlet",
     ],
     setup_requires=["pytest-runner"],

update_db.py

@@ -29,6 +29,8 @@ def update_aircrafts():
         print(f"Error: file at path {file_path} doesn't exist !")
         return False
 
+    ingested = 0
+
     print("Cleaning old datas...")
     # Remove old datas
     db.session.query(Aircrafts).delete()
@@ -55,9 +57,11 @@ def update_aircrafts():
             ac.wake_category = row[8]
             ac.mfr = row[9]
             db.session.add(ac)
+            ingested += 1
         db.session.commit()
 
     print("Done.")
+    print(f"Ingested {ingested} items.")
 
 
 def update_registrations():
@@ -68,6 +72,8 @@ def update_registrations():
         print(f"Error: file at path {file_path} doesn't exist !")
         return False
 
+    ingested = 0
+
     print("Cleaning old datas...")
     # Remove old datas
     db.session.query(AircraftRegistration).delete()
@@ -85,12 +91,18 @@ def update_registrations():
             acr.country = row[0]
             acr.prefix = row[1]
             db.session.add(acr)
+            ingested += 1
         db.session.commit()
 
     print("Done.")
+    print(f"Ingested {ingested} items.")
 
 
 def update_translation():
+    """
+    FIXME: finish this function
+    :return:
+    """
     print("Downloading ACARS translations...")
     dl_infos = URLS["translation"]
     downloaded = download_and_extract_zip(dl_infos["url"], dl_infos["filename"], "translation.csv")
@@ -123,6 +135,8 @@ def update_mode_s():
         print("An error occured while downloading ModeS datas")
         return False
 
+    ingested = 0
+
     print("Cleaning old datas...")
 
     # Remove old datas
@@ -150,6 +164,7 @@ def update_mode_s():
             acm.source = fname
             acm.type_flight = "military" if row["UserString4"] == "M" else None
             db.session.add(acm)
+            ingested += 1
 
             # Add Aircraft Owner only if not empty, and not Private individual
             if (
@@ -174,6 +189,7 @@ def update_mode_s():
     )
 
     print("Done.")
+    print(f"Ingested {ingested} items.")
 
 
 def update_mode_s_ogn():
@@ -185,6 +201,8 @@ def update_mode_s_ogn():
         print("An error occured while downloading ModeS OGN datas")
         return False
 
+    ingested = 0
+
     print("Cleaning old datas...")
     # Remove old datas
     db.session.query(AircraftModes).filter(
@@ -223,6 +241,7 @@ def update_mode_s_ogn():
             if acm.registration != "" and acm.registration != "0000" and acm.icao_type_code != "":
                 acm.last_modified = datetime.datetime.utcnow()
                 db.session.add(acm)
+                ingested += 1
 
     db.session.commit()
 
@@ -232,6 +251,7 @@ def update_mode_s_ogn():
     db.session.query(AircraftModes).filter(and_(AircraftModes.source == fname, AircraftModes.mode_s.in_(q))).delete(
         synchronize_session="fetch"
     )
+    print(f"Ingested {ingested} items.")
 
 
 def update_all():

utils.py

@@ -4,8 +4,9 @@ import os
 from os.path import splitext
 import csv
 import zipfile
-import pycurl
+import urllib.request
 import gzip
+import shutil
 
 from flask import current_app
 
@@ -139,12 +140,8 @@ def utils_download_and_ingest_faa():
     except OSError:
         None
 
-    with open(file_target, "wb") as zip_file:
-        crl = pycurl.Curl()
-        crl.setopt(crl.URL, current_app.config["FAA_ARCHIVE_URL"])
-        crl.setopt(crl.WRITEDATA, zip_file)
-        crl.perform()
-        crl.close()
+    with urllib.request.urlopen(current_app.config["FAA_ARCHIVE_URL"]) as response, open(file_target, "wb") as zip_file:
+        shutil.copyfileobj(response, zip_file)
 
     print("Extracting files...")
 
@@ -177,23 +174,8 @@ def download_file(url: str, filename: str):
     except OSError:
         None
 
-    try:
-        with open(file_target, "wb") as zip_file:
-            crl = pycurl.Curl()
-            crl.setopt(crl.URL, url)
-            crl.setopt(crl.WRITEDATA, zip_file)
-            crl.setopt(crl.FOLLOWLOCATION, True)
-            crl.perform()
-            code = crl.getinfo(pycurl.HTTP_CODE)
-            if code != 200:
-                print(f"Error, received code !200: {code}")
-                crl.close()
-                return False
-    except pycurl.error as e:
-        print(f"Error while downloading file: {e}")
-        return False
-    finally:
-        crl.close()
+    with urllib.request.urlopen(url) as response, open(file_target, "wb") as zip_file:
+        shutil.copyfileobj(response, zip_file)
 
     try:
         if os.path.getsize(file_target) > 0: