from django.shortcuts import render from .models import * import hashlib, json from datetime import datetime, timedelta from django.db.models import Q def search_cities_in_db(search_str): Q_obj = Q(name_en__icontains=search_str) | Q(name_ru__icontains=search_str) | \ Q(country__name_en__icontains=search_str) | Q(country__name_ru__icontains=search_str) res_data = City.objects.filter(Q_obj).values('name', 'country__name') return res_data def search_airports_in_db(search_str): Q_obj = Q(iata_code__icontains=search_str) | \ Q(name_en__icontains=search_str) | Q(name_ru__icontains=search_str) | \ Q(city__name_en__icontains=search_str) | Q(city__name_ru__icontains=search_str) | \ Q(city__country__name_en__icontains=search_str) | \ Q(city__country__name_ru__icontains=search_str) res_data = Airport.objects.filter(Q_obj).values('name', 'iata_code', 'city__name', 'city__country__name') return res_data def get_country_area_id_by_countryName(class_obj, name): try: obj = class_obj.objects.get(name=name) except class_obj.DoesNotExist: return None return obj.area_id def get_countries_key_data(): data = Country.objects.all().values('short_code', 'area_id') data = {item['short_code']: item['area_id'] for item in data} return data def get_cities_by_country_name_en(country_name_en): data = City.objects.filter(country__name_en=country_name_en).values('name_en', 'area_id') data = {item['name_en']: item['area_id'] for item in data} return data def create_airports_by_airportsList(airportsList, city=None): airports_create_objs = None airports_objs = [] for airport_Dict in airportsList: airport = None try: kwargs = {} if airport_Dict['iata']: kwargs.update({'iata_code': airport_Dict['iata']}) airport = Airport.objects.get(**kwargs) except Exception as e: print(f'error = {str(e)}') if not airport: airport_kwargs = { 'city': city, 'name_ru': airport_Dict['name:ru'], 'name_en': airport_Dict['name:en'], 'geo_lat': str(airport_Dict['@lat']), 'geo_lon': str(airport_Dict['@lon']), 'international_name': airport_Dict['int_name'], 'iata_code': airport_Dict['iata'], 'icao_code': airport_Dict['icao'], } if 'area_id' in airport_Dict: airport_kwargs.update({'area_id': airport_Dict['area_id']}) airports_objs.append(Airport(**airport_kwargs)) if airports_objs: airports_create_objs = Airport.objects.bulk_create(airports_objs) return airports_create_objs def parse_data(): # Country.objects.all().delete() from BaseModels.OpenStreetMap.osm_api import osm_get_countries, osm_get_country_w_cities_n_airports # data = osm_get_countries_n_cities_n_airports() db_countries = get_countries_key_data() countries_list = osm_get_countries() for country_item in countries_list: country = None # получаем страну из БД try: kwargs = {} if country_item['ISO3166-1']: kwargs.update({'short_code': country_item['ISO3166-1']}) country = Country.objects.get(**kwargs) if country.parsing_finished_DT and (datetime.now() - country.parsing_finished_DT).days < 30: print(f' + {country.name} - существует в БД, не требует парсинга') continue except Country.DoesNotExist: pass except Exception as e: print(f'error = {str(e)}') area_id = None if country_item['ISO3166-1'] in db_countries.keys(): area_id = db_countries[country_item['ISO3166-1']] # country_Dict - полная версия с городами и аэропортами country_Dict, airports_wo_city = osm_get_country_w_cities_n_airports(country_item, area_id) if country and not country.area_id and 'area_id' in country_Dict and country_Dict['area_id']: country.area_id = country_item['area_id'] country.save(update_fields=['area_id']) if not country: country_kwargs = { 'name_ru': country_Dict['name:ru'], 'name_en': country_Dict['name:en'], 'international_name': country_Dict['int_name'], 'official_name': country_Dict['official_name'], 'short_code': country_Dict['ISO3166-1'], 'code': country_Dict['ISO3166-1:alpha3'], 'num_code': country_Dict['ISO3166-1:numeric'], 'flag_img_url': country_Dict['flag'], 'geo_lat': str(country_Dict['@lat']), 'geo_lon': str(country_Dict['@lon']), } if 'area_id' in country_Dict: country_kwargs.update({'area_id': country_Dict['area_id']}) country = Country.objects.create(**country_kwargs) if 'cities' in country_Dict: for city_Dict in country_Dict['cities']: city = None try: kwargs = {} if country_Dict['name:en']: kwargs.update({'name_en': country_Dict['name:en']}) elif country_Dict['name:ru']: kwargs.update({'name_ru': country_Dict['name:ru']}) city = City.objects.get(**kwargs) except Exception as e: print(f'error = {str(e)}') if not city: city_kwargs = { 'country': country, 'name_ru': city_Dict['name:ru'], 'name_en': city_Dict['name:en'], 'geo_lat': str(city_Dict['@lat']), 'geo_lon': str(city_Dict['@lon']), } if 'area_id' in city_Dict: city_kwargs.update({'area_id': city_Dict['area_id']}) city = City.objects.create(**city_kwargs) if 'airports' in city_Dict: create_airports_by_airportsList(city_Dict['airports'], city) if 'parsing_status' in city_Dict and city_Dict['parsing_status'] == 'finished': city.parsing_finished_DT = datetime.now() city.save(update_fields=['parsing_finished_DT']) if airports_wo_city: create_airports_by_airportsList(airports_wo_city) hash_data = hashlib.md5(json.dumps(country_Dict, sort_keys=True, ensure_ascii=True).encode('utf-8')).hexdigest() country.add_node_to_json_data({'hash': hash_data}) if 'parsing_status' in country_Dict and country_Dict['parsing_status'] == 'finished': country.parsing_finished_DT = datetime.now() country.save(update_fields=['parsing_finished_DT']) return True