0.0.27

osm parsed

BaseModels/OpenStreetMap/osm_api.py

@@ -0,0 +1,288 @@
+import time
+
+import overpass
+import copy
+from geopy.geocoders import Nominatim
+
+Nominatim_last_request = None
+
+def osm_api_request(data, res_type):
+
+    time.sleep(1)
+
+    api = overpass.API(timeout=100)
+
+    res = api.get(data, responseformat=res_type)
+
+    pres_names = []
+    res_list = []
+    if res_type[:3].lower() == 'csv':
+        data_type_list = copy.deepcopy(res[0])
+        for line in res[1:]:
+            if line[0] in pres_names:
+                continue
+
+            i = 0
+            line_Dict = {}
+            while i < len(data_type_list):
+                line_Dict.update({data_type_list[i]: line[i]})
+                i += 1
+            if 'name:en' in line_Dict and 'name' in line_Dict:
+                if not line_Dict["name:en"]:
+                    if line_Dict["name"]:
+                        line_Dict["name:en"] = line_Dict["name"]
+
+                if not line_Dict["name:en"]:
+                    continue
+
+            res_list.append(line_Dict)
+            pres_names.append(line[0])
+    else:
+        res_list = copy.deepcopy(res)
+
+    return res_list
+
+
+def osm_get_area_id_by_params_dict(paramsDict):
+
+    area_id = None
+
+    try:
+
+        # Geocoding request via Nominatim
+        time.sleep(1)
+        geolocator = Nominatim(user_agent='TWB')
+        geo_results = geolocator.geocode(paramsDict, exactly_one=False, limit=3)
+
+        if not geo_results:
+            return area_id
+
+        # Searching for relation in result set
+        country_obj = None
+        for r in geo_results:
+            print(r.address, r.raw.get("osm_type"))
+            if r.raw.get("osm_type") == "relation":
+                country_obj = r
+                break
+
+        if not country_obj:
+            return area_id
+
+        # Calculating area id
+        area_id = int(country_obj.raw.get("osm_id")) + 3600000000
+
+    except Exception as e:
+        print(f'osm_get_area_id_by_params_dict Error = {e}')
+
+    return area_id
+
+
+def osm_get_countries():
+    res_type = 'csv("name", "name:en", "name:ru", "ISO3166-1", "flag", "int_name", "official_name", "ISO3166-1:alpha3", ' \
+               '"ISO3166-1:numeric", ::lon, ::lat)'
+    data = 'relation["admin_level"="2"]["ISO3166-1:alpha3"~"^...$"]; ' \
+           'out center;'
+
+    res = osm_api_request(data, res_type)
+    from operator import itemgetter
+    res = sorted(res, key=itemgetter('name'))
+
+    return res
+
+
+def osm_get_cities_by_country(country_Dict):
+
+    res = []
+
+    try:
+
+        if 'area_id' in country_Dict and country_Dict['area_id']:
+            area_id = country_Dict['area_id']
+        else:
+            return []
+
+        res_type = 'csv("name", "name:ru", "name:en", ::lon, ::lat)'
+        data = f'area({area_id})->.searchArea;' \
+               '(node[place~"city|town"](area.searchArea););' \
+               'out center;'
+
+        res = osm_api_request(data, res_type)
+
+    except Exception as e:
+        print(f'osm_get_cities_by_country Error = {e}')
+
+    return res
+
+
+
+def osm_get_airports(area_id, city_find_str=None):
+
+    airports_cities_Dict = {}
+
+    try:
+
+        if not area_id:
+            return airports_cities_Dict
+
+        res_type = 'csv("name", "name:ru", "name:en", "iata", "icao", "place", "int_name", "addr:country", "city_served", ::lon, ::lat)'
+        data = f'area({area_id})->.searchArea;' \
+               f'nwr["aeroway"="aerodrome"]["iata"~"^...$"]["icao"~"^....$"]["abandoned"!~".*"]["landuse"!="military"]["was:landuse"!="military"]["aerodrome:type"!~"airfield|military|private"]["amenity"!="flight_school"]["closed"!="yes"](area.searchArea)->.airports;' \
+
+        if city_find_str:
+            data = f'{data}' \
+                   f'foreach.airports->.elem(nwr(around.elem:20000)[{city_find_str}]->.city; .elem out center; .city out center;);'
+        else:
+            data = f'{data} .airports out center;'
+
+        res = osm_api_request(data, res_type)
+
+        if not city_find_str:
+            return res
+
+        present_IATA_list = []
+
+        i = 0
+        while i < len(res):
+
+            if not res[i]['iata'] or not res[i]['icao']:
+                del res[i]
+                continue
+
+            if res[i]['iata'] and res[i]['iata'] in present_IATA_list:
+                del res[i]
+                continue
+            present_IATA_list.append(res[i]['iata'])
+
+            # если аэропорт
+            if res[i]['iata']:
+                if i + 1 < len(res) and res[i+1]['place']:
+
+                    # ищем город на следующих строках
+                    i2 = i + 1
+                    linked = None
+                    while i2 < len(res) and res[i2]['place']:
+                        if not linked:
+                            if (res[i2]['name'] and res[i]['name'] and res[i2]['name'] in res[i]['name']) or \
+                                  (res[i2]['name:en'] and res[i]['name:en'] and res[i2]['name:en'] in res[i]['name:en']) or \
+                                  (res[i2]['name:ru'] and res[i]['name:ru'] and res[i2]['name:ru'] in res[i]['name:ru']):
+                                linked = i2
+
+                        i2 += 1
+
+                    if not linked:
+                        linked = i + 1
+
+                    res[i]['city'] = copy.deepcopy(res[linked])
+                    if not res[i]['city']['name'] in airports_cities_Dict:
+                        airports_cities_Dict.update({res[i]['city']['name:en']: [res[i]]})
+                    else:
+                        airports_cities_Dict[res[i]['city']['name:en']].append(res[i])
+
+                    while i < i2:
+                        del res[i]
+                        i2 -= 1
+
+                    continue
+
+                # если не найдена связка с городом
+                elif res[i]['city_served']:
+
+                    res[i]['city'] = {'name:en': res[i]['city_served']}
+                    if not res[i]['city']['name:en'] in airports_cities_Dict:
+                        airports_cities_Dict.update({res[i]['city']['name:en']: [res[i]]})
+                    else:
+                        airports_cities_Dict[res[i]['city']['name:en']].append(res[i])
+                    del res[i]
+                    continue
+
+            # текущий элемент - не аэропорт - удаляем
+            else:
+                del res[i]
+                continue
+
+            i += 1
+
+
+        if res:
+            airports_cities_Dict[None] = res
+
+    except Exception as e:
+        print(f'osm_get_airports Error = {e}')
+        if e.args[0] == 25:
+            return {'error': 'timeout'}
+
+
+    return airports_cities_Dict
+
+
+
+def osm_get_country_w_cities_n_airports(country_Dict, area_id):
+    print(f'{country_Dict["name:en"]}')
+
+    if area_id:
+        country_Dict['area_id'] = area_id
+    else:
+        country_Dict['area_id'] = osm_get_area_id_by_params_dict({'country': country_Dict['name']})
+
+    airports_Dict = osm_get_airports(country_Dict['area_id'], 'place~"city|town"')
+    if airports_Dict and 'error' in airports_Dict and airports_Dict['error'] == 'timeout':
+        airports_Dict = osm_get_airports(country_Dict['area_id'], 'place="city"')
+
+    from ReferenceDataApp.funcs import get_countries_key_data, get_cities_by_country_name_en
+    db_cities = get_cities_by_country_name_en(country_Dict["name:en"])
+
+    cities = osm_get_cities_by_country(country_Dict)
+    for city in cities:
+        print(f' > {city["name:en"]}')
+
+        if airports_Dict and 'error' in airports_Dict and airports_Dict['error'] == 'timeout':
+
+            if city['name:en'] in db_cities.keys():
+                city['area_id'] = db_cities[city['name:en']]
+            else:
+                city['area_id'] = osm_get_area_id_by_params_dict(
+                    {'country': country_Dict['name:en'], 'city': city['name:en']})
+
+            airports_list = osm_get_airports(city['area_id'])
+            if airports_list:
+                city['airports'] = copy.deepcopy(airports_list)
+        else:
+            if city['name:en'] in airports_Dict.keys():
+                city['airports'] = copy.deepcopy(airports_Dict[city['name:en']])
+                del airports_Dict[city['name:en']]
+
+        if not 'airports' in city:
+            city['airports'] = []
+
+        print(f' > > airports count={str(len(city["airports"]))}')
+
+        city['parsing_status'] = 'finished'
+
+    country_Dict['cities'] = cities
+    country_Dict['parsing_status'] = 'finished'
+
+    airports_wo_city = []
+    if airports_Dict and None in airports_Dict:
+        airports_wo_city = airports_Dict[None]
+
+    return country_Dict, airports_wo_city
+
+
+def osm_get_countries_n_cities_n_airports():
+
+    airports_wo_city = []
+
+    from ReferenceDataApp.funcs import get_countries_key_data, get_cities_by_country_name_en
+    db_countries = get_countries_key_data()
+
+    countries = osm_get_countries()
+    i = 0
+    while i < len(countries):
+        area_id = None
+        if countries[i]['name:en'] in db_countries.keys():
+            area_id = db_countries[countries[i]['name:en']]
+        countries[i], airports_wo_city_for_country  = osm_get_country_w_cities_n_airports(countries[i], area_id)
+        if airports_wo_city_for_country:
+            airports_wo_city.extend(airports_wo_city_for_country)
+
+    return countries

GeneralApp/views.py

@@ -11,6 +11,9 @@ from django.conf import settings
 
 def MainPage(request):
 
+    from ReferenceDataApp.funcs import parse_data
+    res = parse_data()
+
     page = StaticPage.objects.get(url='main')
 
     Dict = {

ReferenceDataApp/admin.py

@@ -0,0 +1,16 @@
+from django.contrib import admin
+from sets.admin import Admin_Trans_BaseModel
+from .models import *
+from modeltranslation.admin import TranslationAdmin
+
+class Admin_Country(Admin_Trans_BaseModel):
+    pass
+admin.site.register(Country, Admin_Country)
+
+class Admin_City(Admin_Trans_BaseModel):
+    pass
+admin.site.register(City, Admin_City)
+
+class Admin_Airport(Admin_Trans_BaseModel):
+    pass
+admin.site.register(Airport, Admin_Airport)

ReferenceDataApp/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class ReferencedataappConfig(AppConfig):
+    default_auto_field = 'django.db.models.BigAutoField'
+    name = 'ReferenceDataApp'

ReferenceDataApp/funcs.py

@@ -0,0 +1,186 @@
+from django.shortcuts import render
+from .models import *
+import hashlib, json
+from datetime import datetime, timedelta
+
+def get_country_area_id_by_countryName(class_obj, name):
+    try:
+        obj = class_obj.objects.get(name=name)
+    except class_obj.DoesNotExist:
+        return None
+
+    return obj.area_id
+
+def get_countries_key_data():
+    data = Country.objects.all().values('short_code', 'area_id')
+    data = {item['short_code']: item['area_id'] for item in data}
+    return data
+
+
+def get_cities_by_country_name_en(country_name_en):
+    data = City.objects.filter(country__name_en=country_name_en).values('name_en', 'area_id')
+    data = {item['name_en']: item['area_id'] for item in data}
+    return data
+
+
+def create_airports_by_airportsList(airportsList, city=None):
+
+    airports_create_objs = None
+
+    airports_objs = []
+    for airport_Dict in airportsList:
+
+        airport = None
+        try:
+            kwargs = {}
+            if airport_Dict['iata']:
+                kwargs.update({'iata_code': airport_Dict['iata']})
+            airport = Airport.objects.get(**kwargs)
+        except Exception as e:
+            print(f'error = {str(e)}')
+
+        if not airport:
+            airport_kwargs = {
+                'city': city,
+
+                'name_ru': airport_Dict['name:ru'],
+                'name_en': airport_Dict['name:en'],
+
+                'geo_lat': str(airport_Dict['@lat']),
+                'geo_lon': str(airport_Dict['@lon']),
+
+                'international_name': airport_Dict['int_name'],
+
+                'iata_code': airport_Dict['iata'],
+                'icao_code': airport_Dict['icao'],
+            }
+
+            if 'area_id' in airport_Dict:
+                airport_kwargs.update({'area_id': airport_Dict['area_id']})
+
+            airports_objs.append(Airport(**airport_kwargs))
+
+    if airports_objs:
+        airports_create_objs = Airport.objects.bulk_create(airports_objs)
+
+    return airports_create_objs
+
+
+
+def parse_data():
+    # Country.objects.all().delete()
+
+    from BaseModels.OpenStreetMap.osm_api import osm_get_countries, osm_get_country_w_cities_n_airports
+    # data = osm_get_countries_n_cities_n_airports()
+
+    db_countries = get_countries_key_data()
+
+    countries_list = osm_get_countries()
+
+    for country_item in countries_list:
+
+        country = None
+
+        # получаем страну из БД
+        try:
+            kwargs = {}
+            if country_item['ISO3166-1']:
+                kwargs.update({'short_code': country_item['ISO3166-1']})
+
+            country = Country.objects.get(**kwargs)
+
+            if country.parsing_finished_DT and (datetime.now() - country.parsing_finished_DT).days < 30:
+                print(f' + {country.name} - существует в БД, не требует парсинга')
+                continue
+
+        except Country.DoesNotExist:
+            pass
+        except Exception as e:
+            print(f'error = {str(e)}')
+
+
+        area_id = None
+        if country_item['ISO3166-1'] in db_countries.keys():
+            area_id = db_countries[country_item['ISO3166-1']]
+
+        # country_Dict - полная версия с городами и аэропортами
+        country_Dict, airports_wo_city = osm_get_country_w_cities_n_airports(country_item, area_id)
+
+        if country and not country.area_id and 'area_id' in country_Dict and country_Dict['area_id']:
+            country.area_id = country_item['area_id']
+            country.save(update_fields=['area_id'])
+
+
+
+        if not country:
+            country_kwargs = {
+                'name_ru': country_Dict['name:ru'],
+                'name_en': country_Dict['name:en'],
+                'international_name': country_Dict['int_name'],
+                'official_name': country_Dict['official_name'],
+
+                'short_code': country_Dict['ISO3166-1'],
+                'code': country_Dict['ISO3166-1:alpha3'],
+                'num_code': country_Dict['ISO3166-1:numeric'],
+
+                'flag_img_url': country_Dict['flag'],
+
+                'geo_lat': str(country_Dict['@lat']),
+                'geo_lon': str(country_Dict['@lon']),
+            }
+            if 'area_id' in country_Dict:
+                country_kwargs.update({'area_id': country_Dict['area_id']})
+
+            country = Country.objects.create(**country_kwargs)
+
+        if 'cities' in country_Dict:
+            for city_Dict in country_Dict['cities']:
+
+                city = None
+                try:
+                    kwargs = {}
+                    if country_Dict['name:en']:
+                        kwargs.update({'name_en': country_Dict['name:en']})
+                    elif country_Dict['name:ru']:
+                        kwargs.update({'name_ru': country_Dict['name:ru']})
+                    city = City.objects.get(**kwargs)
+                except Exception as e:
+                    print(f'error = {str(e)}')
+
+                if not city:
+
+                    city_kwargs = {
+                        'country': country,
+
+                        'name_ru': city_Dict['name:ru'],
+                        'name_en': city_Dict['name:en'],
+
+                        'geo_lat': str(city_Dict['@lat']),
+                        'geo_lon': str(city_Dict['@lon']),
+                    }
+
+                    if 'area_id' in city_Dict:
+                        city_kwargs.update({'area_id': city_Dict['area_id']})
+
+                    city = City.objects.create(**city_kwargs)
+
+
+
+                if 'airports' in city_Dict:
+                    create_airports_by_airportsList(city_Dict['airports'], city)
+
+                if 'parsing_status' in city_Dict and city_Dict['parsing_status'] == 'finished':
+                    city.parsing_finished_DT = datetime.now()
+                    city.save(update_fields=['parsing_finished_DT'])
+
+        if airports_wo_city:
+            create_airports_by_airportsList(airports_wo_city)
+
+        hash_data = hashlib.md5(json.dumps(country_Dict, sort_keys=True, ensure_ascii=True).encode('utf-8')).hexdigest()
+        country.add_node_to_json_data({'hash': hash_data})
+
+        if 'parsing_status' in country_Dict and country_Dict['parsing_status'] == 'finished':
+            country.parsing_finished_DT = datetime.now()
+            country.save(update_fields=['parsing_finished_DT'])
+
+    return True

ReferenceDataApp/migrations/0001_initial.py

@@ -0,0 +1,95 @@
+# Generated by Django 4.2.2 on 2023-07-19 16:37
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Country',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')),
+                ('name_ru', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')),
+                ('name_en', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')),
+                ('name_plural', models.TextField(blank=True, null=True, verbose_name='Название (множественное число)')),
+                ('order', models.IntegerField(blank=True, null=True, verbose_name='Очередность отображения')),
+                ('createDT', models.DateTimeField(auto_now_add=True, verbose_name='Дата и время создания')),
+                ('modifiedDT', models.DateTimeField(blank=True, null=True, verbose_name='Дата и время последнего изменения')),
+                ('enable', models.BooleanField(db_index=True, default=True, verbose_name='Включено')),
+                ('json_data', models.JSONField(blank=True, default=dict, verbose_name='Дополнительные данные')),
+                ('international_name', models.CharField(blank=True, max_length=250, null=True, verbose_name='Международное название')),
+                ('official_name', models.CharField(blank=True, max_length=250, null=True, verbose_name='Официальное название')),
+                ('short_code', models.CharField(max_length=2, verbose_name='Код страны по ISO3166-1:alpha2')),
+                ('code', models.CharField(max_length=3, verbose_name='Код страны по ISO3166-1:alpha3')),
+                ('num_code', models.CharField(blank=True, max_length=3, null=True, verbose_name='Код страны по ISO3166-1:numeric')),
+                ('flag_img_url', models.URLField(blank=True, null=True, verbose_name='Ссылка на изображение флага')),
+                ('geo_lat', models.CharField(blank=True, max_length=20, null=True, verbose_name='GPS широта')),
+                ('geo_lon', models.CharField(blank=True, max_length=20, null=True, verbose_name='GPS долгота')),
+                ('area_id', models.IntegerField(blank=True, null=True)),
+            ],
+            options={
+                'verbose_name': 'Страна',
+                'verbose_name_plural': 'Страны',
+                'ordering': ('name', 'code'),
+            },
+        ),
+        migrations.CreateModel(
+            name='City',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')),
+                ('name_ru', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')),
+                ('name_en', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')),
+                ('name_plural', models.TextField(blank=True, null=True, verbose_name='Название (множественное число)')),
+                ('order', models.IntegerField(blank=True, null=True, verbose_name='Очередность отображения')),
+                ('createDT', models.DateTimeField(auto_now_add=True, verbose_name='Дата и время создания')),
+                ('modifiedDT', models.DateTimeField(blank=True, null=True, verbose_name='Дата и время последнего изменения')),
+                ('enable', models.BooleanField(db_index=True, default=True, verbose_name='Включено')),
+                ('json_data', models.JSONField(blank=True, default=dict, verbose_name='Дополнительные данные')),
+                ('geo_lat', models.CharField(blank=True, max_length=20, null=True, verbose_name='GPS широта')),
+                ('geo_lon', models.CharField(blank=True, max_length=20, null=True, verbose_name='GPS долгота')),
+                ('area_id', models.IntegerField(blank=True, null=True)),
+                ('country', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rel_cities_for_country', to='ReferenceDataApp.country', verbose_name='Страна')),
+            ],
+            options={
+                'verbose_name': 'Город',
+                'verbose_name_plural': 'Города',
+                'ordering': ('name',),
+            },
+        ),
+        migrations.CreateModel(
+            name='Airport',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')),
+                ('name_ru', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')),
+                ('name_en', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')),
+                ('name_plural', models.TextField(blank=True, null=True, verbose_name='Название (множественное число)')),
+                ('order', models.IntegerField(blank=True, null=True, verbose_name='Очередность отображения')),
+                ('createDT', models.DateTimeField(auto_now_add=True, verbose_name='Дата и время создания')),
+                ('modifiedDT', models.DateTimeField(blank=True, null=True, verbose_name='Дата и время последнего изменения')),
+                ('enable', models.BooleanField(db_index=True, default=True, verbose_name='Включено')),
+                ('json_data', models.JSONField(blank=True, default=dict, verbose_name='Дополнительные данные')),
+                ('international_name', models.CharField(blank=True, max_length=250, null=True, verbose_name='Международное название')),
+                ('iata_code', models.CharField(max_length=3, verbose_name='IATA')),
+                ('icao_code', models.CharField(max_length=4, verbose_name='ICAO')),
+                ('geo_lat', models.CharField(blank=True, max_length=20, null=True, verbose_name='GPS широта')),
+                ('geo_lon', models.CharField(blank=True, max_length=20, null=True, verbose_name='GPS долгота')),
+                ('area_id', models.IntegerField(blank=True, null=True)),
+                ('city', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rel_airports_for_city', to='ReferenceDataApp.city', verbose_name='Город')),
+            ],
+            options={
+                'verbose_name': 'Аэропорт',
+                'verbose_name_plural': 'Аэропорты',
+                'ordering': ('name', 'iata_code'),
+            },
+        ),
+    ]

ReferenceDataApp/migrations/0002_alter_airport_area_id_alter_city_area_id_and_more.py

@@ -0,0 +1,28 @@
+# Generated by Django 4.2.2 on 2023-07-19 17:09
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ReferenceDataApp', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='airport',
+            name='area_id',
+            field=models.BigIntegerField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='city',
+            name='area_id',
+            field=models.BigIntegerField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='country',
+            name='area_id',
+            field=models.BigIntegerField(blank=True, null=True),
+        ),
+    ]

ReferenceDataApp/migrations/0003_airport_parsing_finished_city_parsing_finished_and_more.py

@@ -0,0 +1,28 @@
+# Generated by Django 4.2.2 on 2023-07-19 19:41
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ReferenceDataApp', '0002_alter_airport_area_id_alter_city_area_id_and_more'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='airport',
+            name='parsing_finished',
+            field=models.BooleanField(default=False, verbose_name='Парсинг завершен'),
+        ),
+        migrations.AddField(
+            model_name='city',
+            name='parsing_finished',
+            field=models.BooleanField(default=False, verbose_name='Парсинг завершен'),
+        ),
+        migrations.AddField(
+            model_name='country',
+            name='parsing_finished',
+            field=models.BooleanField(default=False, verbose_name='Парсинг завершен'),
+        ),
+    ]

ReferenceDataApp/migrations/0004_alter_airport_city.py

@@ -0,0 +1,19 @@
+# Generated by Django 4.2.2 on 2023-07-19 19:52
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ReferenceDataApp', '0003_airport_parsing_finished_city_parsing_finished_and_more'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='airport',
+            name='city',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='rel_airports_for_city', to='ReferenceDataApp.city', verbose_name='Город'),
+        ),
+    ]

ReferenceDataApp/migrations/0005_remove_airport_parsing_finished_and_more.py

@@ -0,0 +1,40 @@
+# Generated by Django 4.2.2 on 2023-07-20 02:09
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ReferenceDataApp', '0004_alter_airport_city'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='airport',
+            name='parsing_finished',
+        ),
+        migrations.RemoveField(
+            model_name='city',
+            name='parsing_finished',
+        ),
+        migrations.RemoveField(
+            model_name='country',
+            name='parsing_finished',
+        ),
+        migrations.AddField(
+            model_name='airport',
+            name='parsing_finished_DT',
+            field=models.DateTimeField(blank=True, null=True, verbose_name='Дата и время завершения парсинга'),
+        ),
+        migrations.AddField(
+            model_name='city',
+            name='parsing_finished_DT',
+            field=models.DateTimeField(blank=True, null=True, verbose_name='Дата и время завершения парсинга'),
+        ),
+        migrations.AddField(
+            model_name='country',
+            name='parsing_finished_DT',
+            field=models.DateTimeField(blank=True, null=True, verbose_name='Дата и время завершения парсинга'),
+        ),
+    ]

ReferenceDataApp/models.py

@@ -0,0 +1,98 @@
+from django.db import models
+from BaseModels.base_models import BaseModel
+from django.utils.translation import gettext_lazy as _
+
+class Country(BaseModel):
+    international_name = models.CharField(max_length=250, verbose_name='Международное название', blank=True, null=True)
+    official_name = models.CharField(max_length=250, verbose_name='Официальное название', blank=True, null=True)
+
+    short_code = models.CharField(max_length=2, verbose_name='Код страны по ISO3166-1:alpha2')
+    code = models.CharField(max_length=3, verbose_name='Код страны по ISO3166-1:alpha3')
+    num_code = models.CharField(max_length=3, verbose_name='Код страны по ISO3166-1:numeric', blank=True, null=True)
+
+    flag_img_url = models.URLField(verbose_name='Ссылка на изображение флага', blank=True, null=True)
+
+    geo_lat = models.CharField(max_length=20, verbose_name='GPS широта', blank=True, null=True)
+    geo_lon = models.CharField(max_length=20, verbose_name='GPS долгота', blank=True, null=True)
+
+    area_id = models.BigIntegerField(blank=True, null=True)
+
+    parsing_finished_DT = models.DateTimeField(verbose_name='Дата и время завершения парсинга', blank=True, null=True)
+
+    def __str__(self):
+        if self.name:
+            return f'{self.name}'
+        elif self.international_name:
+            return f'{self.international_name}'
+        elif self.official_name:
+            return f'{self.official_name}'
+        elif self.code:
+            return f'{self.code}'
+
+
+    class Meta:
+        verbose_name = _('Страна')
+        verbose_name_plural = _('Страны')
+        ordering = ('name', 'code')
+
+
+class City(BaseModel):
+
+    country = models.ForeignKey(
+        Country, verbose_name='Страна', related_name='rel_cities_for_country', on_delete=models.CASCADE)
+
+    geo_lat = models.CharField(max_length=20, verbose_name='GPS широта', blank=True, null=True)
+    geo_lon = models.CharField(max_length=20, verbose_name='GPS долгота', blank=True, null=True)
+
+    area_id = models.BigIntegerField(blank=True, null=True)
+
+    parsing_finished_DT = models.DateTimeField(verbose_name='Дата и время завершения парсинга', blank=True, null=True)
+
+    def __str__(self):
+        if self.name:
+            return f'{self.name}'
+        else:
+            return f'{self.id}'
+
+
+    class Meta:
+        verbose_name = _('Город')
+        verbose_name_plural = _('Города')
+        ordering = ('name',)
+
+
+class Airport(BaseModel):
+
+    city = models.ForeignKey(
+        City, verbose_name='Город', related_name='rel_airports_for_city', on_delete=models.CASCADE,
+        blank=True, null=True)
+
+    international_name = models.CharField(max_length=250, verbose_name='Международное название', blank=True, null=True)
+
+    iata_code = models.CharField(max_length=3, verbose_name='IATA')
+    icao_code = models.CharField(max_length=4, verbose_name='ICAO')
+
+    geo_lat = models.CharField(max_length=20, verbose_name='GPS широта', blank=True, null=True)
+    geo_lon = models.CharField(max_length=20, verbose_name='GPS долгота', blank=True, null=True)
+
+    area_id = models.BigIntegerField(blank=True, null=True)
+
+    parsing_finished_DT = models.DateTimeField(verbose_name='Дата и время завершения парсинга', blank=True, null=True)
+
+    def __str__(self):
+        if self.name:
+            return f'{self.name}'
+        elif self.international_name:
+            return f'{self.international_name}'
+        elif self.iata_code:
+            return f'{self.iata_code}'
+        elif self.icao_code:
+            return f'{self.icao_code}'
+        else:
+            return f'{self.id}'
+
+
+    class Meta:
+        verbose_name = _('Аэропорт')
+        verbose_name_plural = _('Аэропорты')
+        ordering = ('name', 'iata_code')

ReferenceDataApp/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

ReferenceDataApp/translation.py

@@ -0,0 +1,22 @@
+from modeltranslation.translator import translator, TranslationOptions
+from .models import *
+
+
+class Country_TranslationOptions(TranslationOptions):
+    fields = (
+        'name',
+    )
+translator.register(Country, Country_TranslationOptions)
+
+class City_TranslationOptions(TranslationOptions):
+    fields = (
+        'name',
+    )
+translator.register(City, City_TranslationOptions)
+
+
+class Airport_TranslationOptions(TranslationOptions):
+    fields = (
+        'name',
+    )
+translator.register(Airport, Airport_TranslationOptions)

TWB/settings.py

@@ -47,6 +47,7 @@ INSTALLED_APPS = [
     'GeneralApp',
     'AuthApp',
     'RoutesApp',
+    'ReferenceDataApp',
 ]
 
 MIDDLEWARE = [

requirements.pip

@@ -4,3 +4,5 @@ psycopg2-binary==2.9.6
 requests
 Pillow
 django-modeltranslation=0.18.10
+overpass
+geopy

sets/admin.py

@@ -8,26 +8,26 @@ class AdminStacked_FAQitem(Admin_GenericBaseIconStackedInline):
     fields = ['order', 'question', 'answer']
 
 class Admin_BaseModel(Admin_BaseIconModel):
-    def get_fieldsets(self, request, obj=None):
-        fieldsets = [
-            [None, {
-                'classes': ['wide'],
-                'fields': [
-                    'name', 'enable',
-                ]
-            }]
-        ]
-
-        if request.user.is_superuser:
-            add_block = [
-                'Служебная инфа', {
-                    'classes': ['wide'],
-                    'fields': [
-                        'json_data'
-                    ]
-                }]
-            fieldsets.append(add_block)
-        return fieldsets
+    # def get_fieldsets(self, request, obj=None):
+    #     fieldsets = [
+    #         [None, {
+    #             'classes': ['wide'],
+    #             'fields': [
+    #                 'name', 'enable',
+    #             ]
+    #         }]
+    #     ]
+    #
+    #     if request.user.is_superuser:
+    #         add_block = [
+    #             'Служебная инфа', {
+    #                 'classes': ['wide'],
+    #                 'fields': [
+    #                     'json_data'
+    #                 ]
+    #             }]
+    #         fieldsets.append(add_block)
+    #     return fieldsets
 
     save_on_top = True
     list_display = ['id', 'name', 'enable', 'order',