0.0.27

osm parsed

ReferenceDataApp/funcs.py

@@ -0,0 +1,186 @@
+from django.shortcuts import render
+from .models import *
+import hashlib, json
+from datetime import datetime, timedelta
+
+def get_country_area_id_by_countryName(class_obj, name):
+    try:
+        obj = class_obj.objects.get(name=name)
+    except class_obj.DoesNotExist:
+        return None
+
+    return obj.area_id
+
+def get_countries_key_data():
+    data = Country.objects.all().values('short_code', 'area_id')
+    data = {item['short_code']: item['area_id'] for item in data}
+    return data
+
+
+def get_cities_by_country_name_en(country_name_en):
+    data = City.objects.filter(country__name_en=country_name_en).values('name_en', 'area_id')
+    data = {item['name_en']: item['area_id'] for item in data}
+    return data
+
+
+def create_airports_by_airportsList(airportsList, city=None):
+
+    airports_create_objs = None
+
+    airports_objs = []
+    for airport_Dict in airportsList:
+
+        airport = None
+        try:
+            kwargs = {}
+            if airport_Dict['iata']:
+                kwargs.update({'iata_code': airport_Dict['iata']})
+            airport = Airport.objects.get(**kwargs)
+        except Exception as e:
+            print(f'error = {str(e)}')
+
+        if not airport:
+            airport_kwargs = {
+                'city': city,
+
+                'name_ru': airport_Dict['name:ru'],
+                'name_en': airport_Dict['name:en'],
+
+                'geo_lat': str(airport_Dict['@lat']),
+                'geo_lon': str(airport_Dict['@lon']),
+
+                'international_name': airport_Dict['int_name'],
+
+                'iata_code': airport_Dict['iata'],
+                'icao_code': airport_Dict['icao'],
+            }
+
+            if 'area_id' in airport_Dict:
+                airport_kwargs.update({'area_id': airport_Dict['area_id']})
+
+            airports_objs.append(Airport(**airport_kwargs))
+
+    if airports_objs:
+        airports_create_objs = Airport.objects.bulk_create(airports_objs)
+
+    return airports_create_objs
+
+
+
+def parse_data():
+    # Country.objects.all().delete()
+
+    from BaseModels.OpenStreetMap.osm_api import osm_get_countries, osm_get_country_w_cities_n_airports
+    # data = osm_get_countries_n_cities_n_airports()
+
+    db_countries = get_countries_key_data()
+
+    countries_list = osm_get_countries()
+
+    for country_item in countries_list:
+
+        country = None
+
+        # получаем страну из БД
+        try:
+            kwargs = {}
+            if country_item['ISO3166-1']:
+                kwargs.update({'short_code': country_item['ISO3166-1']})
+
+            country = Country.objects.get(**kwargs)
+
+            if country.parsing_finished_DT and (datetime.now() - country.parsing_finished_DT).days < 30:
+                print(f' + {country.name} - существует в БД, не требует парсинга')
+                continue
+
+        except Country.DoesNotExist:
+            pass
+        except Exception as e:
+            print(f'error = {str(e)}')
+
+
+        area_id = None
+        if country_item['ISO3166-1'] in db_countries.keys():
+            area_id = db_countries[country_item['ISO3166-1']]
+
+        # country_Dict - полная версия с городами и аэропортами
+        country_Dict, airports_wo_city = osm_get_country_w_cities_n_airports(country_item, area_id)
+
+        if country and not country.area_id and 'area_id' in country_Dict and country_Dict['area_id']:
+            country.area_id = country_item['area_id']
+            country.save(update_fields=['area_id'])
+
+
+
+        if not country:
+            country_kwargs = {
+                'name_ru': country_Dict['name:ru'],
+                'name_en': country_Dict['name:en'],
+                'international_name': country_Dict['int_name'],
+                'official_name': country_Dict['official_name'],
+
+                'short_code': country_Dict['ISO3166-1'],
+                'code': country_Dict['ISO3166-1:alpha3'],
+                'num_code': country_Dict['ISO3166-1:numeric'],
+
+                'flag_img_url': country_Dict['flag'],
+
+                'geo_lat': str(country_Dict['@lat']),
+                'geo_lon': str(country_Dict['@lon']),
+            }
+            if 'area_id' in country_Dict:
+                country_kwargs.update({'area_id': country_Dict['area_id']})
+
+            country = Country.objects.create(**country_kwargs)
+
+        if 'cities' in country_Dict:
+            for city_Dict in country_Dict['cities']:
+
+                city = None
+                try:
+                    kwargs = {}
+                    if country_Dict['name:en']:
+                        kwargs.update({'name_en': country_Dict['name:en']})
+                    elif country_Dict['name:ru']:
+                        kwargs.update({'name_ru': country_Dict['name:ru']})
+                    city = City.objects.get(**kwargs)
+                except Exception as e:
+                    print(f'error = {str(e)}')
+
+                if not city:
+
+                    city_kwargs = {
+                        'country': country,
+
+                        'name_ru': city_Dict['name:ru'],
+                        'name_en': city_Dict['name:en'],
+
+                        'geo_lat': str(city_Dict['@lat']),
+                        'geo_lon': str(city_Dict['@lon']),
+                    }
+
+                    if 'area_id' in city_Dict:
+                        city_kwargs.update({'area_id': city_Dict['area_id']})
+
+                    city = City.objects.create(**city_kwargs)
+
+
+
+                if 'airports' in city_Dict:
+                    create_airports_by_airportsList(city_Dict['airports'], city)
+
+                if 'parsing_status' in city_Dict and city_Dict['parsing_status'] == 'finished':
+                    city.parsing_finished_DT = datetime.now()
+                    city.save(update_fields=['parsing_finished_DT'])
+
+        if airports_wo_city:
+            create_airports_by_airportsList(airports_wo_city)
+
+        hash_data = hashlib.md5(json.dumps(country_Dict, sort_keys=True, ensure_ascii=True).encode('utf-8')).hexdigest()
+        country.add_node_to_json_data({'hash': hash_data})
+
+        if 'parsing_status' in country_Dict and country_Dict['parsing_status'] == 'finished':
+            country.parsing_finished_DT = datetime.now()
+            country.save(update_fields=['parsing_finished_DT'])
+
+    return True