pyAirwaves is an ADS-B and AIS processing, storage, and display application with Leaflets integration.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

278 lines
7.9 KiB

  1. import datetime
  2. import hashlib
  3. import os
  4. from os.path import splitext
  5. import csv
  6. import zipfile
  7. import urllib.request
  8. import gzip
  9. import shutil
  10. from flask import current_app
  11. class InvalidUsage(Exception):
  12. status_code = 400
  13. def __init__(self, message, status_code=None, payload=None):
  14. Exception.__init__(self)
  15. self.message = message
  16. if status_code is not None:
  17. self.status_code = status_code
  18. self.payload = payload
  19. def to_dict(self):
  20. rv = dict(self.payload or ())
  21. rv["message"] = self.message
  22. rv["status"] = "error"
  23. rv["code"] = self.status_code
  24. return rv
  25. def duration_elapsed_human(seconds):
  26. print(seconds)
  27. seconds = round(seconds)
  28. minutes, seconds = divmod(seconds, 60)
  29. hours, minutes = divmod(minutes, 60)
  30. days, hours = divmod(hours, 24)
  31. years, days = divmod(days, 365.242199)
  32. minutes = int(minutes)
  33. hours = int(hours)
  34. days = int(days)
  35. years = int(years)
  36. if years > 0:
  37. return "%d y" % years
  38. elif days > 0:
  39. return "%d d" % days
  40. elif hours > 0:
  41. return "%d h" % hours + "s" * (hours != 1)
  42. elif minutes > 0:
  43. return "%d mn" % minutes + "s" * (minutes != 1)
  44. else:
  45. return "right now"
  46. def duration_song_human(seconds):
  47. if seconds is None:
  48. return "error"
  49. seconds = float(seconds)
  50. seconds = seconds
  51. minutes, seconds = divmod(seconds, 60)
  52. hours, minutes = divmod(minutes, 60)
  53. days, hours = divmod(hours, 24)
  54. years, days = divmod(days, 365.242199)
  55. minutes = int(minutes)
  56. hours = int(hours)
  57. days = int(days)
  58. years = int(years)
  59. if years > 0:
  60. return "%d year" % years + "s" * (years != 1)
  61. elif days > 0:
  62. return "%d day" % days + "s" * (days != 1)
  63. elif hours > 0:
  64. return "%d hour" % hours + "s" * (hours != 1)
  65. elif minutes > 0:
  66. return "%d mn" % minutes + "s" * (minutes != 1)
  67. else:
  68. return "%.2f sec" % seconds + "s" * (seconds != 1)
  69. def get_hashed_filename(filename):
  70. f_n, f_e = splitext(filename)
  71. fs_fname = hashlib.sha256()
  72. hashed_format = "%s-%s" % (f_n, datetime.datetime.now())
  73. fs_fname.update(hashed_format.encode("utf-8"))
  74. fs_fname = fs_fname.hexdigest()
  75. return fs_fname + f_e
  76. def ingest_faa_aircrafts(file: str):
  77. # clean previous entries
  78. # rmed = FaaMfr.query.delete()
  79. # print(f"Removed {rmed} entries")
  80. data_row = False
  81. rows = {}
  82. with open(file, "rt") as csvFile:
  83. for row in csv.reader(csvFile):
  84. # blank the row
  85. this_row = {}
  86. if data_row:
  87. # Type-correct our CSV data.
  88. this_row.update({"mfgName": row[1].strip()})
  89. this_row.update({"modelName": row[2].strip()})
  90. this_row.update({"acType": int(row[3].strip())})
  91. this_row.update({"engType": int(row[4].strip())})
  92. this_row.update({"acCat": int(row[5].strip())})
  93. this_row.update({"buldCert": int(row[6].strip())})
  94. this_row.update({"engCt": int(row[7].strip())})
  95. this_row.update({"seatCt": int(row[8].strip())})
  96. this_row.update({"weight": int(row[9].replace("CLASS ", "").strip())})
  97. this_row.update({"cruiseSpd": int(row[10].strip())})
  98. print(this_row)
  99. rows.update({row[0].strip(): this_row})
  100. else:
  101. data_row = True
  102. # Then eat the datas
  103. # print(rows)
  104. def utils_download_and_ingest_faa():
  105. print("Downloading and ingesting FAA infos")
  106. file_target = os.path.join(current_app.config["FAA_DL_TEMP_PATH"], "faa.zip")
  107. print("Downloading file...")
  108. try:
  109. os.makedirs(current_app.config["FAA_DL_TEMP_PATH"])
  110. except OSError:
  111. None
  112. with urllib.request.urlopen(current_app.config["FAA_ARCHIVE_URL"]) as response, open(file_target, "wb") as zip_file:
  113. shutil.copyfileobj(response, zip_file)
  114. print("Extracting files...")
  115. try:
  116. # Open and extract MASTER, AIRCRAFT and ENGINE
  117. zip = zipfile.ZipFile(file_target, "r")
  118. zip.extractall(current_app.config["FAA_DL_TEMP_PATH"], ["MASTER.txt", "ACFTREF.txt", "ENGINE.txt"])
  119. finally:
  120. zip.close()
  121. aircrafts_file = os.path.join(current_app.config["FAA_DL_TEMP_PATH"], "ACFTREF.txt")
  122. # engines_file = os.path.join(current_app.config["FAA_DL_TEMP_PATH"], "ENGINE.txt")
  123. # master_file = os.path.join(current_app.config["FAA_DL_TEMP_PATH"], "MASTER.txt")
  124. ingest_faa_aircrafts(aircrafts_file)
  125. def download_file(url: str, filename: str):
  126. """
  127. Download a file
  128. :param url: File URL to download
  129. :param filename: Filename to download to
  130. :return: Bool, download succeeded or not
  131. """
  132. print(f"Downloading {url}...")
  133. file_target = os.path.join(current_app.config["UPDATE_DB_TEMP_PATH"], filename)
  134. try:
  135. os.makedirs(current_app.config["UPDATE_DB_TEMP_PATH"])
  136. except OSError:
  137. None
  138. with urllib.request.urlopen(url) as response, open(file_target, "wb") as zip_file:
  139. shutil.copyfileobj(response, zip_file)
  140. try:
  141. if os.path.getsize(file_target) > 0:
  142. print(f"Downloaded file to {file_target}")
  143. return True
  144. else:
  145. return False
  146. except OSError:
  147. return False
  148. def gunzip(source_filepath, dest_filepath, block_size=65536):
  149. with gzip.open(source_filepath, "rb") as s_file, open(dest_filepath, "wb") as d_file:
  150. while True:
  151. block = s_file.read(block_size)
  152. if not block:
  153. break
  154. else:
  155. d_file.write(block)
  156. d_file.write(block)
  157. return True
  158. def download_and_gunzip(url: str, f_source: str, f_target: str):
  159. """
  160. Download and gunzip file
  161. :param url: File URL to download
  162. :param f_source: Filename to download to
  163. :param f_target: Filename after decompression
  164. :return: Bool, Download and/or decompression succeded
  165. """
  166. # Download file
  167. downloaded = download_file(url, f_source)
  168. if not downloaded:
  169. return False
  170. print(f"gunzip-ing file {f_source}...")
  171. file_source = os.path.join(current_app.config["UPDATE_DB_TEMP_PATH"], f_source)
  172. file_target = os.path.join(current_app.config["UPDATE_DB_TEMP_PATH"], f_target)
  173. ret = gunzip(file_source, file_target)
  174. if not ret:
  175. return False
  176. if os.path.exists(file_source):
  177. os.remove(file_source)
  178. try:
  179. if os.path.getsize(file_target) > 0:
  180. print(f"File extracted to {file_target}")
  181. return True
  182. else:
  183. return False
  184. except OSError:
  185. return False
  186. def download_and_extract_zip(url: str, archive_name: str, filename: str):
  187. """
  188. Download and extract ZIP
  189. :param url: File URL to download
  190. :param archive_name: Filename to download to
  191. :param filename: Filename to extract
  192. :return: Bool, download and/or extraction succeeded or not
  193. """
  194. # Download the file first
  195. downloaded = download_file(url, archive_name)
  196. if not downloaded:
  197. return False
  198. file_target = os.path.join(current_app.config["UPDATE_DB_TEMP_PATH"], filename)
  199. print(f"Extracting {filename}...")
  200. try:
  201. zip_file = zipfile.ZipFile(file_target, "r")
  202. zip_file.extract(filename, current_app.config["UPDATE_DB_TEMP_PATH"])
  203. except zipfile.BadZipFile as e:
  204. print(f"Error opening zip archive: {e}")
  205. return False
  206. except KeyError as e:
  207. print(f"Error extracting file: {e}")
  208. return False
  209. finally:
  210. zip_file.close()
  211. if os.path.exists(file_target):
  212. os.remove(file_target)
  213. filename_final = os.path.join(current_app.config["UPDATE_DB_TEMP_PATH"], filename)
  214. try:
  215. if os.path.getsize(filename_final) > 0:
  216. print(f"File extracted to {filename_final}")
  217. return True
  218. else:
  219. return False
  220. except OSError:
  221. return False