From b768988a5ddf07941afa84debf241ea2cb0b5787 Mon Sep 17 00:00:00 2001 From: SDE Date: Thu, 20 Jul 2023 14:04:32 +0300 Subject: [PATCH] 0.0.27 osm parsed --- BaseModels/OpenStreetMap/osm_api.py | 288 ++++++++++++++++++ GeneralApp/views.py | 3 + ReferenceDataApp/__init__.py | 0 ReferenceDataApp/admin.py | 16 + ReferenceDataApp/apps.py | 6 + ReferenceDataApp/funcs.py | 186 +++++++++++ ReferenceDataApp/migrations/0001_initial.py | 95 ++++++ ...ort_area_id_alter_city_area_id_and_more.py | 28 ++ ...finished_city_parsing_finished_and_more.py | 28 ++ .../migrations/0004_alter_airport_city.py | 19 ++ ...emove_airport_parsing_finished_and_more.py | 40 +++ ReferenceDataApp/migrations/__init__.py | 0 ReferenceDataApp/models.py | 98 ++++++ ReferenceDataApp/tests.py | 3 + ReferenceDataApp/translation.py | 22 ++ ReferenceDataApp/views.py | 0 TWB/settings.py | 1 + requirements.pip | 2 + sets/admin.py | 40 +-- 19 files changed, 855 insertions(+), 20 deletions(-) create mode 100644 BaseModels/OpenStreetMap/osm_api.py create mode 100644 ReferenceDataApp/__init__.py create mode 100644 ReferenceDataApp/admin.py create mode 100644 ReferenceDataApp/apps.py create mode 100644 ReferenceDataApp/funcs.py create mode 100644 ReferenceDataApp/migrations/0001_initial.py create mode 100644 ReferenceDataApp/migrations/0002_alter_airport_area_id_alter_city_area_id_and_more.py create mode 100644 ReferenceDataApp/migrations/0003_airport_parsing_finished_city_parsing_finished_and_more.py create mode 100644 ReferenceDataApp/migrations/0004_alter_airport_city.py create mode 100644 ReferenceDataApp/migrations/0005_remove_airport_parsing_finished_and_more.py create mode 100644 ReferenceDataApp/migrations/__init__.py create mode 100644 ReferenceDataApp/models.py create mode 100644 ReferenceDataApp/tests.py create mode 100644 ReferenceDataApp/translation.py create mode 100644 ReferenceDataApp/views.py diff --git a/BaseModels/OpenStreetMap/osm_api.py b/BaseModels/OpenStreetMap/osm_api.py new file mode 100644 index 0000000..1ec1819 --- /dev/null +++ b/BaseModels/OpenStreetMap/osm_api.py @@ -0,0 +1,288 @@ +import time + +import overpass +import copy +from geopy.geocoders import Nominatim + +Nominatim_last_request = None + +def osm_api_request(data, res_type): + + time.sleep(1) + + api = overpass.API(timeout=100) + + res = api.get(data, responseformat=res_type) + + pres_names = [] + res_list = [] + if res_type[:3].lower() == 'csv': + data_type_list = copy.deepcopy(res[0]) + for line in res[1:]: + if line[0] in pres_names: + continue + + i = 0 + line_Dict = {} + while i < len(data_type_list): + line_Dict.update({data_type_list[i]: line[i]}) + i += 1 + if 'name:en' in line_Dict and 'name' in line_Dict: + if not line_Dict["name:en"]: + if line_Dict["name"]: + line_Dict["name:en"] = line_Dict["name"] + + if not line_Dict["name:en"]: + continue + + res_list.append(line_Dict) + pres_names.append(line[0]) + else: + res_list = copy.deepcopy(res) + + return res_list + + +def osm_get_area_id_by_params_dict(paramsDict): + + area_id = None + + try: + + # Geocoding request via Nominatim + time.sleep(1) + geolocator = Nominatim(user_agent='TWB') + geo_results = geolocator.geocode(paramsDict, exactly_one=False, limit=3) + + if not geo_results: + return area_id + + # Searching for relation in result set + country_obj = None + for r in geo_results: + print(r.address, r.raw.get("osm_type")) + if r.raw.get("osm_type") == "relation": + country_obj = r + break + + if not country_obj: + return area_id + + # Calculating area id + area_id = int(country_obj.raw.get("osm_id")) + 3600000000 + + except Exception as e: + print(f'osm_get_area_id_by_params_dict Error = {e}') + + return area_id + + +def osm_get_countries(): + res_type = 'csv("name", "name:en", "name:ru", "ISO3166-1", "flag", "int_name", "official_name", "ISO3166-1:alpha3", ' \ + '"ISO3166-1:numeric", ::lon, ::lat)' + data = 'relation["admin_level"="2"]["ISO3166-1:alpha3"~"^...$"]; ' \ + 'out center;' + + res = osm_api_request(data, res_type) + from operator import itemgetter + res = sorted(res, key=itemgetter('name')) + + return res + + +def osm_get_cities_by_country(country_Dict): + + res = [] + + try: + + if 'area_id' in country_Dict and country_Dict['area_id']: + area_id = country_Dict['area_id'] + else: + return [] + + res_type = 'csv("name", "name:ru", "name:en", ::lon, ::lat)' + data = f'area({area_id})->.searchArea;' \ + '(node[place~"city|town"](area.searchArea););' \ + 'out center;' + + res = osm_api_request(data, res_type) + + except Exception as e: + print(f'osm_get_cities_by_country Error = {e}') + + return res + + + +def osm_get_airports(area_id, city_find_str=None): + + airports_cities_Dict = {} + + try: + + if not area_id: + return airports_cities_Dict + + res_type = 'csv("name", "name:ru", "name:en", "iata", "icao", "place", "int_name", "addr:country", "city_served", ::lon, ::lat)' + data = f'area({area_id})->.searchArea;' \ + f'nwr["aeroway"="aerodrome"]["iata"~"^...$"]["icao"~"^....$"]["abandoned"!~".*"]["landuse"!="military"]["was:landuse"!="military"]["aerodrome:type"!~"airfield|military|private"]["amenity"!="flight_school"]["closed"!="yes"](area.searchArea)->.airports;' \ + + if city_find_str: + data = f'{data}' \ + f'foreach.airports->.elem(nwr(around.elem:20000)[{city_find_str}]->.city; .elem out center; .city out center;);' + else: + data = f'{data} .airports out center;' + + res = osm_api_request(data, res_type) + + if not city_find_str: + return res + + present_IATA_list = [] + + i = 0 + while i < len(res): + + if not res[i]['iata'] or not res[i]['icao']: + del res[i] + continue + + if res[i]['iata'] and res[i]['iata'] in present_IATA_list: + del res[i] + continue + present_IATA_list.append(res[i]['iata']) + + # если аэропорт + if res[i]['iata']: + if i + 1 < len(res) and res[i+1]['place']: + + # ищем город на следующих строках + i2 = i + 1 + linked = None + while i2 < len(res) and res[i2]['place']: + if not linked: + if (res[i2]['name'] and res[i]['name'] and res[i2]['name'] in res[i]['name']) or \ + (res[i2]['name:en'] and res[i]['name:en'] and res[i2]['name:en'] in res[i]['name:en']) or \ + (res[i2]['name:ru'] and res[i]['name:ru'] and res[i2]['name:ru'] in res[i]['name:ru']): + linked = i2 + + i2 += 1 + + if not linked: + linked = i + 1 + + res[i]['city'] = copy.deepcopy(res[linked]) + if not res[i]['city']['name'] in airports_cities_Dict: + airports_cities_Dict.update({res[i]['city']['name:en']: [res[i]]}) + else: + airports_cities_Dict[res[i]['city']['name:en']].append(res[i]) + + while i < i2: + del res[i] + i2 -= 1 + + continue + + # если не найдена связка с городом + elif res[i]['city_served']: + + res[i]['city'] = {'name:en': res[i]['city_served']} + if not res[i]['city']['name:en'] in airports_cities_Dict: + airports_cities_Dict.update({res[i]['city']['name:en']: [res[i]]}) + else: + airports_cities_Dict[res[i]['city']['name:en']].append(res[i]) + del res[i] + continue + + # текущий элемент - не аэропорт - удаляем + else: + del res[i] + continue + + i += 1 + + + if res: + airports_cities_Dict[None] = res + + except Exception as e: + print(f'osm_get_airports Error = {e}') + if e.args[0] == 25: + return {'error': 'timeout'} + + + return airports_cities_Dict + + + +def osm_get_country_w_cities_n_airports(country_Dict, area_id): + print(f'{country_Dict["name:en"]}') + + if area_id: + country_Dict['area_id'] = area_id + else: + country_Dict['area_id'] = osm_get_area_id_by_params_dict({'country': country_Dict['name']}) + + airports_Dict = osm_get_airports(country_Dict['area_id'], 'place~"city|town"') + if airports_Dict and 'error' in airports_Dict and airports_Dict['error'] == 'timeout': + airports_Dict = osm_get_airports(country_Dict['area_id'], 'place="city"') + + from ReferenceDataApp.funcs import get_countries_key_data, get_cities_by_country_name_en + db_cities = get_cities_by_country_name_en(country_Dict["name:en"]) + + cities = osm_get_cities_by_country(country_Dict) + for city in cities: + print(f' > {city["name:en"]}') + + if airports_Dict and 'error' in airports_Dict and airports_Dict['error'] == 'timeout': + + if city['name:en'] in db_cities.keys(): + city['area_id'] = db_cities[city['name:en']] + else: + city['area_id'] = osm_get_area_id_by_params_dict( + {'country': country_Dict['name:en'], 'city': city['name:en']}) + + airports_list = osm_get_airports(city['area_id']) + if airports_list: + city['airports'] = copy.deepcopy(airports_list) + else: + if city['name:en'] in airports_Dict.keys(): + city['airports'] = copy.deepcopy(airports_Dict[city['name:en']]) + del airports_Dict[city['name:en']] + + if not 'airports' in city: + city['airports'] = [] + + print(f' > > airports count={str(len(city["airports"]))}') + + city['parsing_status'] = 'finished' + + country_Dict['cities'] = cities + country_Dict['parsing_status'] = 'finished' + + airports_wo_city = [] + if airports_Dict and None in airports_Dict: + airports_wo_city = airports_Dict[None] + + return country_Dict, airports_wo_city + + +def osm_get_countries_n_cities_n_airports(): + + airports_wo_city = [] + + from ReferenceDataApp.funcs import get_countries_key_data, get_cities_by_country_name_en + db_countries = get_countries_key_data() + + countries = osm_get_countries() + i = 0 + while i < len(countries): + area_id = None + if countries[i]['name:en'] in db_countries.keys(): + area_id = db_countries[countries[i]['name:en']] + countries[i], airports_wo_city_for_country = osm_get_country_w_cities_n_airports(countries[i], area_id) + if airports_wo_city_for_country: + airports_wo_city.extend(airports_wo_city_for_country) + + return countries \ No newline at end of file diff --git a/GeneralApp/views.py b/GeneralApp/views.py index e34966b..3f16e3b 100644 --- a/GeneralApp/views.py +++ b/GeneralApp/views.py @@ -11,6 +11,9 @@ from django.conf import settings def MainPage(request): + from ReferenceDataApp.funcs import parse_data + res = parse_data() + page = StaticPage.objects.get(url='main') Dict = { diff --git a/ReferenceDataApp/__init__.py b/ReferenceDataApp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ReferenceDataApp/admin.py b/ReferenceDataApp/admin.py new file mode 100644 index 0000000..b052688 --- /dev/null +++ b/ReferenceDataApp/admin.py @@ -0,0 +1,16 @@ +from django.contrib import admin +from sets.admin import Admin_Trans_BaseModel +from .models import * +from modeltranslation.admin import TranslationAdmin + +class Admin_Country(Admin_Trans_BaseModel): + pass +admin.site.register(Country, Admin_Country) + +class Admin_City(Admin_Trans_BaseModel): + pass +admin.site.register(City, Admin_City) + +class Admin_Airport(Admin_Trans_BaseModel): + pass +admin.site.register(Airport, Admin_Airport) \ No newline at end of file diff --git a/ReferenceDataApp/apps.py b/ReferenceDataApp/apps.py new file mode 100644 index 0000000..f57b607 --- /dev/null +++ b/ReferenceDataApp/apps.py @@ -0,0 +1,6 @@ +from django.apps import AppConfig + + +class ReferencedataappConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'ReferenceDataApp' diff --git a/ReferenceDataApp/funcs.py b/ReferenceDataApp/funcs.py new file mode 100644 index 0000000..1e91c18 --- /dev/null +++ b/ReferenceDataApp/funcs.py @@ -0,0 +1,186 @@ +from django.shortcuts import render +from .models import * +import hashlib, json +from datetime import datetime, timedelta + +def get_country_area_id_by_countryName(class_obj, name): + try: + obj = class_obj.objects.get(name=name) + except class_obj.DoesNotExist: + return None + + return obj.area_id + +def get_countries_key_data(): + data = Country.objects.all().values('short_code', 'area_id') + data = {item['short_code']: item['area_id'] for item in data} + return data + + +def get_cities_by_country_name_en(country_name_en): + data = City.objects.filter(country__name_en=country_name_en).values('name_en', 'area_id') + data = {item['name_en']: item['area_id'] for item in data} + return data + + +def create_airports_by_airportsList(airportsList, city=None): + + airports_create_objs = None + + airports_objs = [] + for airport_Dict in airportsList: + + airport = None + try: + kwargs = {} + if airport_Dict['iata']: + kwargs.update({'iata_code': airport_Dict['iata']}) + airport = Airport.objects.get(**kwargs) + except Exception as e: + print(f'error = {str(e)}') + + if not airport: + airport_kwargs = { + 'city': city, + + 'name_ru': airport_Dict['name:ru'], + 'name_en': airport_Dict['name:en'], + + 'geo_lat': str(airport_Dict['@lat']), + 'geo_lon': str(airport_Dict['@lon']), + + 'international_name': airport_Dict['int_name'], + + 'iata_code': airport_Dict['iata'], + 'icao_code': airport_Dict['icao'], + } + + if 'area_id' in airport_Dict: + airport_kwargs.update({'area_id': airport_Dict['area_id']}) + + airports_objs.append(Airport(**airport_kwargs)) + + if airports_objs: + airports_create_objs = Airport.objects.bulk_create(airports_objs) + + return airports_create_objs + + + +def parse_data(): + # Country.objects.all().delete() + + from BaseModels.OpenStreetMap.osm_api import osm_get_countries, osm_get_country_w_cities_n_airports + # data = osm_get_countries_n_cities_n_airports() + + db_countries = get_countries_key_data() + + countries_list = osm_get_countries() + + for country_item in countries_list: + + country = None + + # получаем страну из БД + try: + kwargs = {} + if country_item['ISO3166-1']: + kwargs.update({'short_code': country_item['ISO3166-1']}) + + country = Country.objects.get(**kwargs) + + if country.parsing_finished_DT and (datetime.now() - country.parsing_finished_DT).days < 30: + print(f' + {country.name} - существует в БД, не требует парсинга') + continue + + except Country.DoesNotExist: + pass + except Exception as e: + print(f'error = {str(e)}') + + + area_id = None + if country_item['ISO3166-1'] in db_countries.keys(): + area_id = db_countries[country_item['ISO3166-1']] + + # country_Dict - полная версия с городами и аэропортами + country_Dict, airports_wo_city = osm_get_country_w_cities_n_airports(country_item, area_id) + + if country and not country.area_id and 'area_id' in country_Dict and country_Dict['area_id']: + country.area_id = country_item['area_id'] + country.save(update_fields=['area_id']) + + + + if not country: + country_kwargs = { + 'name_ru': country_Dict['name:ru'], + 'name_en': country_Dict['name:en'], + 'international_name': country_Dict['int_name'], + 'official_name': country_Dict['official_name'], + + 'short_code': country_Dict['ISO3166-1'], + 'code': country_Dict['ISO3166-1:alpha3'], + 'num_code': country_Dict['ISO3166-1:numeric'], + + 'flag_img_url': country_Dict['flag'], + + 'geo_lat': str(country_Dict['@lat']), + 'geo_lon': str(country_Dict['@lon']), + } + if 'area_id' in country_Dict: + country_kwargs.update({'area_id': country_Dict['area_id']}) + + country = Country.objects.create(**country_kwargs) + + if 'cities' in country_Dict: + for city_Dict in country_Dict['cities']: + + city = None + try: + kwargs = {} + if country_Dict['name:en']: + kwargs.update({'name_en': country_Dict['name:en']}) + elif country_Dict['name:ru']: + kwargs.update({'name_ru': country_Dict['name:ru']}) + city = City.objects.get(**kwargs) + except Exception as e: + print(f'error = {str(e)}') + + if not city: + + city_kwargs = { + 'country': country, + + 'name_ru': city_Dict['name:ru'], + 'name_en': city_Dict['name:en'], + + 'geo_lat': str(city_Dict['@lat']), + 'geo_lon': str(city_Dict['@lon']), + } + + if 'area_id' in city_Dict: + city_kwargs.update({'area_id': city_Dict['area_id']}) + + city = City.objects.create(**city_kwargs) + + + + if 'airports' in city_Dict: + create_airports_by_airportsList(city_Dict['airports'], city) + + if 'parsing_status' in city_Dict and city_Dict['parsing_status'] == 'finished': + city.parsing_finished_DT = datetime.now() + city.save(update_fields=['parsing_finished_DT']) + + if airports_wo_city: + create_airports_by_airportsList(airports_wo_city) + + hash_data = hashlib.md5(json.dumps(country_Dict, sort_keys=True, ensure_ascii=True).encode('utf-8')).hexdigest() + country.add_node_to_json_data({'hash': hash_data}) + + if 'parsing_status' in country_Dict and country_Dict['parsing_status'] == 'finished': + country.parsing_finished_DT = datetime.now() + country.save(update_fields=['parsing_finished_DT']) + + return True \ No newline at end of file diff --git a/ReferenceDataApp/migrations/0001_initial.py b/ReferenceDataApp/migrations/0001_initial.py new file mode 100644 index 0000000..159210d --- /dev/null +++ b/ReferenceDataApp/migrations/0001_initial.py @@ -0,0 +1,95 @@ +# Generated by Django 4.2.2 on 2023-07-19 16:37 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='Country', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')), + ('name_ru', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')), + ('name_en', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')), + ('name_plural', models.TextField(blank=True, null=True, verbose_name='Название (множественное число)')), + ('order', models.IntegerField(blank=True, null=True, verbose_name='Очередность отображения')), + ('createDT', models.DateTimeField(auto_now_add=True, verbose_name='Дата и время создания')), + ('modifiedDT', models.DateTimeField(blank=True, null=True, verbose_name='Дата и время последнего изменения')), + ('enable', models.BooleanField(db_index=True, default=True, verbose_name='Включено')), + ('json_data', models.JSONField(blank=True, default=dict, verbose_name='Дополнительные данные')), + ('international_name', models.CharField(blank=True, max_length=250, null=True, verbose_name='Международное название')), + ('official_name', models.CharField(blank=True, max_length=250, null=True, verbose_name='Официальное название')), + ('short_code', models.CharField(max_length=2, verbose_name='Код страны по ISO3166-1:alpha2')), + ('code', models.CharField(max_length=3, verbose_name='Код страны по ISO3166-1:alpha3')), + ('num_code', models.CharField(blank=True, max_length=3, null=True, verbose_name='Код страны по ISO3166-1:numeric')), + ('flag_img_url', models.URLField(blank=True, null=True, verbose_name='Ссылка на изображение флага')), + ('geo_lat', models.CharField(blank=True, max_length=20, null=True, verbose_name='GPS широта')), + ('geo_lon', models.CharField(blank=True, max_length=20, null=True, verbose_name='GPS долгота')), + ('area_id', models.IntegerField(blank=True, null=True)), + ], + options={ + 'verbose_name': 'Страна', + 'verbose_name_plural': 'Страны', + 'ordering': ('name', 'code'), + }, + ), + migrations.CreateModel( + name='City', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')), + ('name_ru', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')), + ('name_en', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')), + ('name_plural', models.TextField(blank=True, null=True, verbose_name='Название (множественное число)')), + ('order', models.IntegerField(blank=True, null=True, verbose_name='Очередность отображения')), + ('createDT', models.DateTimeField(auto_now_add=True, verbose_name='Дата и время создания')), + ('modifiedDT', models.DateTimeField(blank=True, null=True, verbose_name='Дата и время последнего изменения')), + ('enable', models.BooleanField(db_index=True, default=True, verbose_name='Включено')), + ('json_data', models.JSONField(blank=True, default=dict, verbose_name='Дополнительные данные')), + ('geo_lat', models.CharField(blank=True, max_length=20, null=True, verbose_name='GPS широта')), + ('geo_lon', models.CharField(blank=True, max_length=20, null=True, verbose_name='GPS долгота')), + ('area_id', models.IntegerField(blank=True, null=True)), + ('country', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rel_cities_for_country', to='ReferenceDataApp.country', verbose_name='Страна')), + ], + options={ + 'verbose_name': 'Город', + 'verbose_name_plural': 'Города', + 'ordering': ('name',), + }, + ), + migrations.CreateModel( + name='Airport', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')), + ('name_ru', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')), + ('name_en', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')), + ('name_plural', models.TextField(blank=True, null=True, verbose_name='Название (множественное число)')), + ('order', models.IntegerField(blank=True, null=True, verbose_name='Очередность отображения')), + ('createDT', models.DateTimeField(auto_now_add=True, verbose_name='Дата и время создания')), + ('modifiedDT', models.DateTimeField(blank=True, null=True, verbose_name='Дата и время последнего изменения')), + ('enable', models.BooleanField(db_index=True, default=True, verbose_name='Включено')), + ('json_data', models.JSONField(blank=True, default=dict, verbose_name='Дополнительные данные')), + ('international_name', models.CharField(blank=True, max_length=250, null=True, verbose_name='Международное название')), + ('iata_code', models.CharField(max_length=3, verbose_name='IATA')), + ('icao_code', models.CharField(max_length=4, verbose_name='ICAO')), + ('geo_lat', models.CharField(blank=True, max_length=20, null=True, verbose_name='GPS широта')), + ('geo_lon', models.CharField(blank=True, max_length=20, null=True, verbose_name='GPS долгота')), + ('area_id', models.IntegerField(blank=True, null=True)), + ('city', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rel_airports_for_city', to='ReferenceDataApp.city', verbose_name='Город')), + ], + options={ + 'verbose_name': 'Аэропорт', + 'verbose_name_plural': 'Аэропорты', + 'ordering': ('name', 'iata_code'), + }, + ), + ] diff --git a/ReferenceDataApp/migrations/0002_alter_airport_area_id_alter_city_area_id_and_more.py b/ReferenceDataApp/migrations/0002_alter_airport_area_id_alter_city_area_id_and_more.py new file mode 100644 index 0000000..f8514b7 --- /dev/null +++ b/ReferenceDataApp/migrations/0002_alter_airport_area_id_alter_city_area_id_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2.2 on 2023-07-19 17:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ReferenceDataApp', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='airport', + name='area_id', + field=models.BigIntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='city', + name='area_id', + field=models.BigIntegerField(blank=True, null=True), + ), + migrations.AlterField( + model_name='country', + name='area_id', + field=models.BigIntegerField(blank=True, null=True), + ), + ] diff --git a/ReferenceDataApp/migrations/0003_airport_parsing_finished_city_parsing_finished_and_more.py b/ReferenceDataApp/migrations/0003_airport_parsing_finished_city_parsing_finished_and_more.py new file mode 100644 index 0000000..9551946 --- /dev/null +++ b/ReferenceDataApp/migrations/0003_airport_parsing_finished_city_parsing_finished_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2.2 on 2023-07-19 19:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ReferenceDataApp', '0002_alter_airport_area_id_alter_city_area_id_and_more'), + ] + + operations = [ + migrations.AddField( + model_name='airport', + name='parsing_finished', + field=models.BooleanField(default=False, verbose_name='Парсинг завершен'), + ), + migrations.AddField( + model_name='city', + name='parsing_finished', + field=models.BooleanField(default=False, verbose_name='Парсинг завершен'), + ), + migrations.AddField( + model_name='country', + name='parsing_finished', + field=models.BooleanField(default=False, verbose_name='Парсинг завершен'), + ), + ] diff --git a/ReferenceDataApp/migrations/0004_alter_airport_city.py b/ReferenceDataApp/migrations/0004_alter_airport_city.py new file mode 100644 index 0000000..79a7e2d --- /dev/null +++ b/ReferenceDataApp/migrations/0004_alter_airport_city.py @@ -0,0 +1,19 @@ +# Generated by Django 4.2.2 on 2023-07-19 19:52 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('ReferenceDataApp', '0003_airport_parsing_finished_city_parsing_finished_and_more'), + ] + + operations = [ + migrations.AlterField( + model_name='airport', + name='city', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='rel_airports_for_city', to='ReferenceDataApp.city', verbose_name='Город'), + ), + ] diff --git a/ReferenceDataApp/migrations/0005_remove_airport_parsing_finished_and_more.py b/ReferenceDataApp/migrations/0005_remove_airport_parsing_finished_and_more.py new file mode 100644 index 0000000..1cd0114 --- /dev/null +++ b/ReferenceDataApp/migrations/0005_remove_airport_parsing_finished_and_more.py @@ -0,0 +1,40 @@ +# Generated by Django 4.2.2 on 2023-07-20 02:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('ReferenceDataApp', '0004_alter_airport_city'), + ] + + operations = [ + migrations.RemoveField( + model_name='airport', + name='parsing_finished', + ), + migrations.RemoveField( + model_name='city', + name='parsing_finished', + ), + migrations.RemoveField( + model_name='country', + name='parsing_finished', + ), + migrations.AddField( + model_name='airport', + name='parsing_finished_DT', + field=models.DateTimeField(blank=True, null=True, verbose_name='Дата и время завершения парсинга'), + ), + migrations.AddField( + model_name='city', + name='parsing_finished_DT', + field=models.DateTimeField(blank=True, null=True, verbose_name='Дата и время завершения парсинга'), + ), + migrations.AddField( + model_name='country', + name='parsing_finished_DT', + field=models.DateTimeField(blank=True, null=True, verbose_name='Дата и время завершения парсинга'), + ), + ] diff --git a/ReferenceDataApp/migrations/__init__.py b/ReferenceDataApp/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ReferenceDataApp/models.py b/ReferenceDataApp/models.py new file mode 100644 index 0000000..f8b027b --- /dev/null +++ b/ReferenceDataApp/models.py @@ -0,0 +1,98 @@ +from django.db import models +from BaseModels.base_models import BaseModel +from django.utils.translation import gettext_lazy as _ + +class Country(BaseModel): + international_name = models.CharField(max_length=250, verbose_name='Международное название', blank=True, null=True) + official_name = models.CharField(max_length=250, verbose_name='Официальное название', blank=True, null=True) + + short_code = models.CharField(max_length=2, verbose_name='Код страны по ISO3166-1:alpha2') + code = models.CharField(max_length=3, verbose_name='Код страны по ISO3166-1:alpha3') + num_code = models.CharField(max_length=3, verbose_name='Код страны по ISO3166-1:numeric', blank=True, null=True) + + flag_img_url = models.URLField(verbose_name='Ссылка на изображение флага', blank=True, null=True) + + geo_lat = models.CharField(max_length=20, verbose_name='GPS широта', blank=True, null=True) + geo_lon = models.CharField(max_length=20, verbose_name='GPS долгота', blank=True, null=True) + + area_id = models.BigIntegerField(blank=True, null=True) + + parsing_finished_DT = models.DateTimeField(verbose_name='Дата и время завершения парсинга', blank=True, null=True) + + def __str__(self): + if self.name: + return f'{self.name}' + elif self.international_name: + return f'{self.international_name}' + elif self.official_name: + return f'{self.official_name}' + elif self.code: + return f'{self.code}' + + + class Meta: + verbose_name = _('Страна') + verbose_name_plural = _('Страны') + ordering = ('name', 'code') + + +class City(BaseModel): + + country = models.ForeignKey( + Country, verbose_name='Страна', related_name='rel_cities_for_country', on_delete=models.CASCADE) + + geo_lat = models.CharField(max_length=20, verbose_name='GPS широта', blank=True, null=True) + geo_lon = models.CharField(max_length=20, verbose_name='GPS долгота', blank=True, null=True) + + area_id = models.BigIntegerField(blank=True, null=True) + + parsing_finished_DT = models.DateTimeField(verbose_name='Дата и время завершения парсинга', blank=True, null=True) + + def __str__(self): + if self.name: + return f'{self.name}' + else: + return f'{self.id}' + + + class Meta: + verbose_name = _('Город') + verbose_name_plural = _('Города') + ordering = ('name',) + + +class Airport(BaseModel): + + city = models.ForeignKey( + City, verbose_name='Город', related_name='rel_airports_for_city', on_delete=models.CASCADE, + blank=True, null=True) + + international_name = models.CharField(max_length=250, verbose_name='Международное название', blank=True, null=True) + + iata_code = models.CharField(max_length=3, verbose_name='IATA') + icao_code = models.CharField(max_length=4, verbose_name='ICAO') + + geo_lat = models.CharField(max_length=20, verbose_name='GPS широта', blank=True, null=True) + geo_lon = models.CharField(max_length=20, verbose_name='GPS долгота', blank=True, null=True) + + area_id = models.BigIntegerField(blank=True, null=True) + + parsing_finished_DT = models.DateTimeField(verbose_name='Дата и время завершения парсинга', blank=True, null=True) + + def __str__(self): + if self.name: + return f'{self.name}' + elif self.international_name: + return f'{self.international_name}' + elif self.iata_code: + return f'{self.iata_code}' + elif self.icao_code: + return f'{self.icao_code}' + else: + return f'{self.id}' + + + class Meta: + verbose_name = _('Аэропорт') + verbose_name_plural = _('Аэропорты') + ordering = ('name', 'iata_code') \ No newline at end of file diff --git a/ReferenceDataApp/tests.py b/ReferenceDataApp/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/ReferenceDataApp/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/ReferenceDataApp/translation.py b/ReferenceDataApp/translation.py new file mode 100644 index 0000000..76de0db --- /dev/null +++ b/ReferenceDataApp/translation.py @@ -0,0 +1,22 @@ +from modeltranslation.translator import translator, TranslationOptions +from .models import * + + +class Country_TranslationOptions(TranslationOptions): + fields = ( + 'name', + ) +translator.register(Country, Country_TranslationOptions) + +class City_TranslationOptions(TranslationOptions): + fields = ( + 'name', + ) +translator.register(City, City_TranslationOptions) + + +class Airport_TranslationOptions(TranslationOptions): + fields = ( + 'name', + ) +translator.register(Airport, Airport_TranslationOptions) diff --git a/ReferenceDataApp/views.py b/ReferenceDataApp/views.py new file mode 100644 index 0000000..e69de29 diff --git a/TWB/settings.py b/TWB/settings.py index 314992e..24a9513 100644 --- a/TWB/settings.py +++ b/TWB/settings.py @@ -47,6 +47,7 @@ INSTALLED_APPS = [ 'GeneralApp', 'AuthApp', 'RoutesApp', + 'ReferenceDataApp', ] MIDDLEWARE = [ diff --git a/requirements.pip b/requirements.pip index c93faea..8330467 100644 --- a/requirements.pip +++ b/requirements.pip @@ -4,3 +4,5 @@ psycopg2-binary==2.9.6 requests Pillow django-modeltranslation=0.18.10 +overpass +geopy \ No newline at end of file diff --git a/sets/admin.py b/sets/admin.py index f0521c5..5e9ef51 100644 --- a/sets/admin.py +++ b/sets/admin.py @@ -8,26 +8,26 @@ class AdminStacked_FAQitem(Admin_GenericBaseIconStackedInline): fields = ['order', 'question', 'answer'] class Admin_BaseModel(Admin_BaseIconModel): - def get_fieldsets(self, request, obj=None): - fieldsets = [ - [None, { - 'classes': ['wide'], - 'fields': [ - 'name', 'enable', - ] - }] - ] - - if request.user.is_superuser: - add_block = [ - 'Служебная инфа', { - 'classes': ['wide'], - 'fields': [ - 'json_data' - ] - }] - fieldsets.append(add_block) - return fieldsets + # def get_fieldsets(self, request, obj=None): + # fieldsets = [ + # [None, { + # 'classes': ['wide'], + # 'fields': [ + # 'name', 'enable', + # ] + # }] + # ] + # + # if request.user.is_superuser: + # add_block = [ + # 'Служебная инфа', { + # 'classes': ['wide'], + # 'fields': [ + # 'json_data' + # ] + # }] + # fieldsets.append(add_block) + # return fieldsets save_on_top = True list_display = ['id', 'name', 'enable', 'order',