268 lines
9.5 KiB
Python
268 lines
9.5 KiB
Python
from django.shortcuts import render
|
||
from .models import *
|
||
import hashlib, json
|
||
from datetime import datetime, timedelta
|
||
from django.db.models import Q
|
||
from timezonefinder import TimezoneFinder
|
||
|
||
tzf = TimezoneFinder()
|
||
|
||
def search_cities_in_db(search_str):
|
||
|
||
|
||
|
||
Q_obj = Q(name_en__icontains=search_str) | Q(name_ru__icontains=search_str) | \
|
||
Q(country__name_en__icontains=search_str) | Q(country__name_ru__icontains=search_str)
|
||
res_data = City.objects.filter(Q_obj).values('id', 'name', 'country__name')
|
||
return list(res_data)
|
||
|
||
def search_airports_in_db(search_str):
|
||
Q_obj = Q(iata_code__icontains=search_str) | \
|
||
Q(name_en__icontains=search_str) | Q(name_ru__icontains=search_str) | \
|
||
Q(city__name_en__icontains=search_str) | Q(city__name_ru__icontains=search_str) | \
|
||
Q(city__country__name_en__icontains=search_str) | \
|
||
Q(city__country__name_ru__icontains=search_str)
|
||
res_data = Airport.objects.filter(Q_obj).values('id', 'name', 'iata_code', 'city__name', 'city__country__name')
|
||
return list(res_data)
|
||
|
||
|
||
def get_country_area_id_by_countryName(class_obj, name):
|
||
try:
|
||
obj = class_obj.objects.get(name=name)
|
||
except class_obj.DoesNotExist:
|
||
return None
|
||
|
||
return obj.area_id
|
||
|
||
def get_countries_key_data():
|
||
data = Country.objects.all().values('short_code', 'area_id')
|
||
data = {item['short_code']: item['area_id'] for item in data}
|
||
return data
|
||
|
||
|
||
def get_cities_by_country_name_en(country_name_en):
|
||
data = City.objects.filter(country__name_en=country_name_en).values('name_en', 'area_id')
|
||
data = {item['name_en']: item['area_id'] for item in data}
|
||
return data
|
||
|
||
|
||
def create_airports_by_airportsList(airportsList, city=None):
|
||
|
||
airports_create_objs = None
|
||
|
||
airports_objs = []
|
||
for airport_Dict in airportsList:
|
||
|
||
airport = None
|
||
try:
|
||
kwargs = {}
|
||
if airport_Dict['iata']:
|
||
kwargs.update({'iata_code': airport_Dict['iata']})
|
||
airport = Airport.objects.get(**kwargs)
|
||
if airport.geo_lat and airport.geo_lon and not airport.timezone:
|
||
airport.timezone = tzf.timezone_at(
|
||
lng=float(airport.geo_lon), lat=float(airport.geo_lat))
|
||
airport.save(update_fields=['timezone'])
|
||
print(f'airport {airport.international_name} - {airport.timezone}')
|
||
except Airport.DoesNotExist:
|
||
print(f' - - {airport_Dict["iata"]} не найден в БД > добавляем')
|
||
except Exception as e:
|
||
print(f'error = {str(e)}')
|
||
|
||
if not airport:
|
||
geo_lat = float(airport_Dict['@lat'])
|
||
geo_lon = float(airport_Dict['@lon'])
|
||
tz = tzf.timezone_at(lng=geo_lon, lat=geo_lat)
|
||
print(f'airport {airport_Dict["int_name"]} - {tz}')
|
||
|
||
airport_kwargs = {
|
||
'city': city,
|
||
|
||
# 'name_ru': airport_Dict['name:ru'],
|
||
# 'name_en': airport_Dict['name:en'],
|
||
'timezone': tz,
|
||
|
||
'geo_lat': str(geo_lat),
|
||
'geo_lon': str(geo_lon),
|
||
|
||
'international_name': airport_Dict['int_name'],
|
||
|
||
'iata_code': airport_Dict['iata'],
|
||
'icao_code': airport_Dict['icao'],
|
||
}
|
||
|
||
if airport_Dict['name:ru']:
|
||
airport_kwargs.update({'name_ru': airport_Dict['name:ru']})
|
||
|
||
if airport_Dict['name:en']:
|
||
airport_kwargs.update({'name_en': airport_Dict['name:en']})
|
||
|
||
if 'area_id' in airport_Dict:
|
||
airport_kwargs.update({'area_id': airport_Dict['area_id']})
|
||
|
||
airports_objs.append(Airport(**airport_kwargs))
|
||
|
||
if airports_objs:
|
||
airports_create_objs = Airport.objects.bulk_create(airports_objs)
|
||
|
||
return airports_create_objs
|
||
|
||
|
||
|
||
def parse_data():
|
||
# Airport.objects.all().delete()
|
||
# Country.objects.all().delete()
|
||
|
||
from BaseModels.OpenStreetMap.osm_api import osm_get_countries, osm_get_country_w_cities_n_airports
|
||
# data = osm_get_countries_n_cities_n_airports()
|
||
|
||
db_countries = get_countries_key_data()
|
||
|
||
countries_list = osm_get_countries()
|
||
|
||
for country_item in countries_list:
|
||
|
||
country = None
|
||
|
||
# получаем страну из БД
|
||
try:
|
||
kwargs = {}
|
||
if country_item['ISO3166-1']:
|
||
kwargs.update({'short_code': country_item['ISO3166-1']})
|
||
|
||
country = Country.objects.get(**kwargs)
|
||
|
||
if (country.parsing_finished_DT
|
||
and (datetime.now() - country.parsing_finished_DT).days < 30
|
||
and country.timezone
|
||
):
|
||
print(f' + {country.name} - существует в БД, не требует парсинга')
|
||
continue
|
||
|
||
except Country.DoesNotExist:
|
||
print(f'{country_item["ISO3166-1"]} не найдена в БД > добавляем')
|
||
except Exception as e:
|
||
print(f'error = {str(e)}')
|
||
|
||
|
||
area_id = None
|
||
if country_item['ISO3166-1'] in db_countries.keys():
|
||
area_id = db_countries[country_item['ISO3166-1']]
|
||
|
||
# country_Dict - полная версия с городами и аэропортами
|
||
country_Dict, airports_wo_city = osm_get_country_w_cities_n_airports(country_item, area_id)
|
||
|
||
if country and not country.area_id and 'area_id' in country_Dict and country_Dict['area_id']:
|
||
country.area_id = country_item['area_id']
|
||
country.save(update_fields=['area_id'])
|
||
|
||
|
||
|
||
if not country:
|
||
country_kwargs = {
|
||
# 'name_ru': country_Dict['name:ru'],
|
||
# 'name_en': country_Dict['name:en'],
|
||
'international_name': country_Dict['int_name'],
|
||
'official_name': country_Dict['official_name'],
|
||
|
||
'short_code': country_Dict['ISO3166-1'],
|
||
'code': country_Dict['ISO3166-1:alpha3'],
|
||
'num_code': country_Dict['ISO3166-1:numeric'],
|
||
|
||
'flag_img_url': country_Dict['flag'],
|
||
|
||
'geo_lat': str(country_Dict['@lat']),
|
||
'geo_lon': str(country_Dict['@lon']),
|
||
}
|
||
|
||
if country_Dict['name:ru']:
|
||
country_kwargs.update({'name_ru': country_Dict['name:ru']})
|
||
|
||
if country_Dict['name:en']:
|
||
country_kwargs.update({'name_en': country_Dict['name:en']})
|
||
|
||
if 'area_id' in country_Dict:
|
||
country_kwargs.update({'area_id': country_Dict['area_id']})
|
||
|
||
country = Country.objects.create(**country_kwargs)
|
||
|
||
if 'cities' in country_Dict:
|
||
for city_Dict in country_Dict['cities']:
|
||
|
||
city = None
|
||
|
||
kwargs = {}
|
||
if city_Dict['name:en']:
|
||
kwargs.update({'name_en': city_Dict['name:en']})
|
||
|
||
if country:
|
||
kwargs.update({'country': country})
|
||
|
||
try:
|
||
city = City.objects.get(**kwargs)
|
||
except City.DoesNotExist:
|
||
print(f' - {city_Dict["name:en"]} не найдена в БД > добавляем')
|
||
except Exception as e:
|
||
cities = City.objects.filter(**kwargs)
|
||
if cities:
|
||
city = cities[0]
|
||
cities.exclude(id=city.id).delete()
|
||
else:
|
||
print(f'error = {str(e)}')
|
||
|
||
geo_lat = float(city_Dict['@lat'])
|
||
geo_lon = float(city_Dict['@lon'])
|
||
tz = tzf.timezone_at(lng=geo_lon, lat=geo_lat)
|
||
if not city or not city.timezone:
|
||
print(f'city {city_Dict["name:en"]} - {tz}')
|
||
|
||
# собираем данные
|
||
city_kwargs = {
|
||
'country': country,
|
||
|
||
# 'name_ru': city_Dict['name:ru'],
|
||
# 'name_en': city_Dict['name:en'],
|
||
|
||
'timezone': tz,
|
||
|
||
'geo_lat': str(geo_lat),
|
||
'geo_lon': str(geo_lon),
|
||
|
||
}
|
||
|
||
if city_Dict['name:ru']:
|
||
kwargs.update({'name_ru': city_Dict['name:ru']})
|
||
|
||
city_kwargs.update(kwargs)
|
||
|
||
if 'area_id' in city_Dict:
|
||
city_kwargs.update({'area_id': city_Dict['area_id']})
|
||
|
||
if not city:
|
||
city = City.objects.create(**city_kwargs)
|
||
else:
|
||
City.objects.filter(id=city.id).update(**city_kwargs)
|
||
|
||
|
||
if 'airports' in city_Dict:
|
||
create_airports_by_airportsList(city_Dict['airports'], city)
|
||
|
||
if 'parsing_status' in city_Dict and city_Dict['parsing_status'] == 'finished':
|
||
city.parsing_finished_DT = datetime.now()
|
||
city.save(update_fields=['parsing_finished_DT'])
|
||
|
||
if airports_wo_city:
|
||
create_airports_by_airportsList(airports_wo_city)
|
||
|
||
hash_data = hashlib.md5(json.dumps(country_Dict, sort_keys=True, ensure_ascii=True).encode('utf-8')).hexdigest()
|
||
country.add_node_to_json_data({'hash': hash_data})
|
||
|
||
if not country.timezone:
|
||
country.timezone = tzf.timezone_at(lng=float(country.geo_lon), lat=float(country.geo_lat))
|
||
print(f'country {country.name} - {country.timezone}')
|
||
|
||
if 'parsing_status' in country_Dict and country_Dict['parsing_status'] == 'finished':
|
||
country.parsing_finished_DT = datetime.now()
|
||
country.save()
|
||
|
||
return True |