from django.shortcuts import render from .models import * import hashlib, json from datetime import datetime, timedelta from django.db.models import Q def search_cities_in_db(search_str): Q_obj = Q(name_en__icontains=search_str) | Q(name_ru__icontains=search_str) | \ Q(country__name_en__icontains=search_str) | Q(country__name_ru__icontains=search_str) res_data = City.objects.filter(Q_obj).values('id', 'name', 'country__name') return list(res_data) def search_airports_in_db(search_str): Q_obj = Q(iata_code__icontains=search_str) | \ Q(name_en__icontains=search_str) | Q(name_ru__icontains=search_str) | \ Q(city__name_en__icontains=search_str) | Q(city__name_ru__icontains=search_str) | \ Q(city__country__name_en__icontains=search_str) | \ Q(city__country__name_ru__icontains=search_str) res_data = Airport.objects.filter(Q_obj).values('id', 'name', 'iata_code', 'city__name', 'city__country__name') return list(res_data) def get_country_area_id_by_countryName(class_obj, name): try: obj = class_obj.objects.get(name=name) except class_obj.DoesNotExist: return None return obj.area_id def get_countries_key_data(): data = Country.objects.all().values('short_code', 'area_id') data = {item['short_code']: item['area_id'] for item in data} return data def get_cities_by_country_name_en(country_name_en): data = City.objects.filter(country__name_en=country_name_en).values('name_en', 'area_id') data = {item['name_en']: item['area_id'] for item in data} return data def create_airports_by_airportsList(airportsList, city=None): airports_create_objs = None airports_objs = [] for airport_Dict in airportsList: airport = None try: kwargs = {} if airport_Dict['iata']: kwargs.update({'iata_code': airport_Dict['iata']}) airport = Airport.objects.get(**kwargs) except Airport.DoesNotExist: print(f' - - {airport_Dict["iata"]} не найден в БД > добавляем') except Exception as e: print(f'error = {str(e)}') if not airport: airport_kwargs = { 'city': city, # 'name_ru': airport_Dict['name:ru'], # 'name_en': airport_Dict['name:en'], 'geo_lat': str(airport_Dict['@lat']), 'geo_lon': str(airport_Dict['@lon']), 'international_name': airport_Dict['int_name'], 'iata_code': airport_Dict['iata'], 'icao_code': airport_Dict['icao'], } if airport_Dict['name:ru']: airport_kwargs.update({'name_ru': airport_Dict['name:ru']}) if airport_Dict['name:en']: airport_kwargs.update({'name_en': airport_Dict['name:en']}) if 'area_id' in airport_Dict: airport_kwargs.update({'area_id': airport_Dict['area_id']}) airports_objs.append(Airport(**airport_kwargs)) if airports_objs: airports_create_objs = Airport.objects.bulk_create(airports_objs) return airports_create_objs def parse_data(): # Airport.objects.all().delete() # Country.objects.all().delete() from BaseModels.OpenStreetMap.osm_api import osm_get_countries, osm_get_country_w_cities_n_airports # data = osm_get_countries_n_cities_n_airports() db_countries = get_countries_key_data() countries_list = osm_get_countries() for country_item in countries_list: country = None # получаем страну из БД try: kwargs = {} if country_item['ISO3166-1']: kwargs.update({'short_code': country_item['ISO3166-1']}) country = Country.objects.get(**kwargs) if country.parsing_finished_DT and (datetime.now() - country.parsing_finished_DT).days < 30: print(f' + {country.name} - существует в БД, не требует парсинга') continue except Country.DoesNotExist: print(f'{country_item["ISO3166-1"]} не найдена в БД > добавляем') except Exception as e: print(f'error = {str(e)}') area_id = None if country_item['ISO3166-1'] in db_countries.keys(): area_id = db_countries[country_item['ISO3166-1']] # country_Dict - полная версия с городами и аэропортами country_Dict, airports_wo_city = osm_get_country_w_cities_n_airports(country_item, area_id) if country and not country.area_id and 'area_id' in country_Dict and country_Dict['area_id']: country.area_id = country_item['area_id'] country.save(update_fields=['area_id']) if not country: country_kwargs = { # 'name_ru': country_Dict['name:ru'], # 'name_en': country_Dict['name:en'], 'international_name': country_Dict['int_name'], 'official_name': country_Dict['official_name'], 'short_code': country_Dict['ISO3166-1'], 'code': country_Dict['ISO3166-1:alpha3'], 'num_code': country_Dict['ISO3166-1:numeric'], 'flag_img_url': country_Dict['flag'], 'geo_lat': str(country_Dict['@lat']), 'geo_lon': str(country_Dict['@lon']), } if country_Dict['name:ru']: country_kwargs.update({'name_ru': country_Dict['name:ru']}) if country_Dict['name:en']: country_kwargs.update({'name_en': country_Dict['name:en']}) if 'area_id' in country_Dict: country_kwargs.update({'area_id': country_Dict['area_id']}) country = Country.objects.create(**country_kwargs) if 'cities' in country_Dict: for city_Dict in country_Dict['cities']: city = None kwargs = {} if city_Dict['name:en']: kwargs.update({'name_en': city_Dict['name:en']}) if country: kwargs.update({'country': country}) try: city = City.objects.get(**kwargs) except City.DoesNotExist: print(f' - {city_Dict["name:en"]} не найдена в БД > добавляем') except Exception as e: cities = City.objects.filter(**kwargs) if cities: city = cities[0] cities.exclude(id=city.id).delete() else: print(f'error = {str(e)}') # собираем данные city_kwargs = { 'country': country, # 'name_ru': city_Dict['name:ru'], # 'name_en': city_Dict['name:en'], 'geo_lat': str(city_Dict['@lat']), 'geo_lon': str(city_Dict['@lon']), } if city_Dict['name:ru']: kwargs.update({'name_ru': city_Dict['name:ru']}) city_kwargs.update(kwargs) if 'area_id' in city_Dict: city_kwargs.update({'area_id': city_Dict['area_id']}) if not city: city = City.objects.create(**city_kwargs) else: City.objects.filter(id=city.id).update(**city_kwargs) if 'airports' in city_Dict: create_airports_by_airportsList(city_Dict['airports'], city) if 'parsing_status' in city_Dict and city_Dict['parsing_status'] == 'finished': city.parsing_finished_DT = datetime.now() city.save(update_fields=['parsing_finished_DT']) if airports_wo_city: create_airports_by_airportsList(airports_wo_city) hash_data = hashlib.md5(json.dumps(country_Dict, sort_keys=True, ensure_ascii=True).encode('utf-8')).hexdigest() country.add_node_to_json_data({'hash': hash_data}) if 'parsing_status' in country_Dict and country_Dict['parsing_status'] == 'finished': country.parsing_finished_DT = datetime.now() country.save(update_fields=['parsing_finished_DT']) return True