sysadminix/account_store
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

0.0.27

Browse Source 
osm parsed
This commit is contained in:
SDE
2023-07-20 14:04:32 +03:00
parent 113e576d4c
commit b768988a5d
19 changed files with 855 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
ReferenceDataApp/__init__.py Normal file
View File

16
ReferenceDataApp/admin.py Normal file
View File

@@ -0,0 +1,16 @@
from django.contrib import admin
from sets.admin import Admin_Trans_BaseModel
from .models import *
from modeltranslation.admin import TranslationAdmin
class Admin_Country(Admin_Trans_BaseModel):
pass
admin.site.register(Country, Admin_Country)
class Admin_City(Admin_Trans_BaseModel):
pass
admin.site.register(City, Admin_City)
class Admin_Airport(Admin_Trans_BaseModel):
pass
admin.site.register(Airport, Admin_Airport)

6
ReferenceDataApp/apps.py Normal file
View File

@@ -0,0 +1,6 @@
from django.apps import AppConfig
class ReferencedataappConfig(AppConfig):
default_auto_field = 'django.db.models.BigAutoField'
name = 'ReferenceDataApp'

186
ReferenceDataApp/funcs.py Normal file
View File

@@ -0,0 +1,186 @@
from django.shortcuts import render
from .models import *
import hashlib, json
from datetime import datetime, timedelta
def get_country_area_id_by_countryName(class_obj, name):
try:
obj = class_obj.objects.get(name=name)
except class_obj.DoesNotExist:
return None
return obj.area_id
def get_countries_key_data():
data = Country.objects.all().values('short_code', 'area_id')
data = {item['short_code']: item['area_id'] for item in data}
return data
def get_cities_by_country_name_en(country_name_en):
data = City.objects.filter(country__name_en=country_name_en).values('name_en', 'area_id')
data = {item['name_en']: item['area_id'] for item in data}
return data
def create_airports_by_airportsList(airportsList, city=None):
airports_create_objs = None
airports_objs = []
for airport_Dict in airportsList:
airport = None
try:
kwargs = {}
if airport_Dict['iata']:
kwargs.update({'iata_code': airport_Dict['iata']})
airport = Airport.objects.get(**kwargs)
except Exception as e:
print(f'error = {str(e)}')
if not airport:
airport_kwargs = {
'city': city,
'name_ru': airport_Dict['name:ru'],
'name_en': airport_Dict['name:en'],
'geo_lat': str(airport_Dict['@lat']),
'geo_lon': str(airport_Dict['@lon']),
'international_name': airport_Dict['int_name'],
'iata_code': airport_Dict['iata'],
'icao_code': airport_Dict['icao'],
}
if 'area_id' in airport_Dict:
airport_kwargs.update({'area_id': airport_Dict['area_id']})
airports_objs.append(Airport(**airport_kwargs))
if airports_objs:
airports_create_objs = Airport.objects.bulk_create(airports_objs)
return airports_create_objs
def parse_data():
# Country.objects.all().delete()
from BaseModels.OpenStreetMap.osm_api import osm_get_countries, osm_get_country_w_cities_n_airports
# data = osm_get_countries_n_cities_n_airports()
db_countries = get_countries_key_data()
countries_list = osm_get_countries()
for country_item in countries_list:
country = None
# получаем страну из БД
try:
kwargs = {}
if country_item['ISO3166-1']:
kwargs.update({'short_code': country_item['ISO3166-1']})
country = Country.objects.get(**kwargs)
if country.parsing_finished_DT and (datetime.now() - country.parsing_finished_DT).days < 30:
print(f' + {country.name} - существует в БД, не требует парсинга')
continue
except Country.DoesNotExist:
pass
except Exception as e:
print(f'error = {str(e)}')
area_id = None
if country_item['ISO3166-1'] in db_countries.keys():
area_id = db_countries[country_item['ISO3166-1']]
# country_Dict - полная версия с городами и аэропортами
country_Dict, airports_wo_city = osm_get_country_w_cities_n_airports(country_item, area_id)
if country and not country.area_id and 'area_id' in country_Dict and country_Dict['area_id']:
country.area_id = country_item['area_id']
country.save(update_fields=['area_id'])
if not country:
country_kwargs = {
'name_ru': country_Dict['name:ru'],
'name_en': country_Dict['name:en'],
'international_name': country_Dict['int_name'],
'official_name': country_Dict['official_name'],
'short_code': country_Dict['ISO3166-1'],
'code': country_Dict['ISO3166-1:alpha3'],
'num_code': country_Dict['ISO3166-1:numeric'],
'flag_img_url': country_Dict['flag'],
'geo_lat': str(country_Dict['@lat']),
'geo_lon': str(country_Dict['@lon']),
}
if 'area_id' in country_Dict:
country_kwargs.update({'area_id': country_Dict['area_id']})
country = Country.objects.create(**country_kwargs)
if 'cities' in country_Dict:
for city_Dict in country_Dict['cities']:
city = None
try:
kwargs = {}
if country_Dict['name:en']:
kwargs.update({'name_en': country_Dict['name:en']})
elif country_Dict['name:ru']:
kwargs.update({'name_ru': country_Dict['name:ru']})
city = City.objects.get(**kwargs)
except Exception as e:
print(f'error = {str(e)}')
if not city:
city_kwargs = {
'country': country,
'name_ru': city_Dict['name:ru'],
'name_en': city_Dict['name:en'],
'geo_lat': str(city_Dict['@lat']),
'geo_lon': str(city_Dict['@lon']),
}
if 'area_id' in city_Dict:
city_kwargs.update({'area_id': city_Dict['area_id']})
city = City.objects.create(**city_kwargs)
if 'airports' in city_Dict:
create_airports_by_airportsList(city_Dict['airports'], city)
if 'parsing_status' in city_Dict and city_Dict['parsing_status'] == 'finished':
city.parsing_finished_DT = datetime.now()
city.save(update_fields=['parsing_finished_DT'])
if airports_wo_city:
create_airports_by_airportsList(airports_wo_city)
hash_data = hashlib.md5(json.dumps(country_Dict, sort_keys=True, ensure_ascii=True).encode('utf-8')).hexdigest()
country.add_node_to_json_data({'hash': hash_data})
if 'parsing_status' in country_Dict and country_Dict['parsing_status'] == 'finished':
country.parsing_finished_DT = datetime.now()
country.save(update_fields=['parsing_finished_DT'])
return True

95
ReferenceDataApp/migrations/0001_initial.py Normal file
View File

@@ -0,0 +1,95 @@
# Generated by Django 4.2.2 on 2023-07-19 16:37
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
initial = True
dependencies = [
]
operations = [
migrations.CreateModel(
name='Country',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')),
('name_ru', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')),
('name_en', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')),
('name_plural', models.TextField(blank=True, null=True, verbose_name='Название (множественное число)')),
('order', models.IntegerField(blank=True, null=True, verbose_name='Очередность отображения')),
('createDT', models.DateTimeField(auto_now_add=True, verbose_name='Дата и время создания')),
('modifiedDT', models.DateTimeField(blank=True, null=True, verbose_name='Дата и время последнего изменения')),
('enable', models.BooleanField(db_index=True, default=True, verbose_name='Включено')),
('json_data', models.JSONField(blank=True, default=dict, verbose_name='Дополнительные данные')),
('international_name', models.CharField(blank=True, max_length=250, null=True, verbose_name='Международное название')),
('official_name', models.CharField(blank=True, max_length=250, null=True, verbose_name='Официальное название')),
('short_code', models.CharField(max_length=2, verbose_name='Код страны по ISO3166-1:alpha2')),
('code', models.CharField(max_length=3, verbose_name='Код страны по ISO3166-1:alpha3')),
('num_code', models.CharField(blank=True, max_length=3, null=True, verbose_name='Код страны по ISO3166-1:numeric')),
('flag_img_url', models.URLField(blank=True, null=True, verbose_name='Ссылка на изображение флага')),
('geo_lat', models.CharField(blank=True, max_length=20, null=True, verbose_name='GPS широта')),
('geo_lon', models.CharField(blank=True, max_length=20, null=True, verbose_name='GPS долгота')),
('area_id', models.IntegerField(blank=True, null=True)),
],
options={
'verbose_name': 'Страна',
'verbose_name_plural': 'Страны',
'ordering': ('name', 'code'),
},
),
migrations.CreateModel(
name='City',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')),
('name_ru', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')),
('name_en', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')),
('name_plural', models.TextField(blank=True, null=True, verbose_name='Название (множественное число)')),
('order', models.IntegerField(blank=True, null=True, verbose_name='Очередность отображения')),
('createDT', models.DateTimeField(auto_now_add=True, verbose_name='Дата и время создания')),
('modifiedDT', models.DateTimeField(blank=True, null=True, verbose_name='Дата и время последнего изменения')),
('enable', models.BooleanField(db_index=True, default=True, verbose_name='Включено')),
('json_data', models.JSONField(blank=True, default=dict, verbose_name='Дополнительные данные')),
('geo_lat', models.CharField(blank=True, max_length=20, null=True, verbose_name='GPS широта')),
('geo_lon', models.CharField(blank=True, max_length=20, null=True, verbose_name='GPS долгота')),
('area_id', models.IntegerField(blank=True, null=True)),
('country', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rel_cities_for_country', to='ReferenceDataApp.country', verbose_name='Страна')),
],
options={
'verbose_name': 'Город',
'verbose_name_plural': 'Города',
'ordering': ('name',),
},
),
migrations.CreateModel(
name='Airport',
fields=[
('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('name', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')),
('name_ru', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')),
('name_en', models.TextField(blank=True, help_text='Название', null=True, verbose_name='Название')),
('name_plural', models.TextField(blank=True, null=True, verbose_name='Название (множественное число)')),
('order', models.IntegerField(blank=True, null=True, verbose_name='Очередность отображения')),
('createDT', models.DateTimeField(auto_now_add=True, verbose_name='Дата и время создания')),
('modifiedDT', models.DateTimeField(blank=True, null=True, verbose_name='Дата и время последнего изменения')),
('enable', models.BooleanField(db_index=True, default=True, verbose_name='Включено')),
('json_data', models.JSONField(blank=True, default=dict, verbose_name='Дополнительные данные')),
('international_name', models.CharField(blank=True, max_length=250, null=True, verbose_name='Международное название')),
('iata_code', models.CharField(max_length=3, verbose_name='IATA')),
('icao_code', models.CharField(max_length=4, verbose_name='ICAO')),
('geo_lat', models.CharField(blank=True, max_length=20, null=True, verbose_name='GPS широта')),
('geo_lon', models.CharField(blank=True, max_length=20, null=True, verbose_name='GPS долгота')),
('area_id', models.IntegerField(blank=True, null=True)),
('city', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='rel_airports_for_city', to='ReferenceDataApp.city', verbose_name='Город')),
],
options={
'verbose_name': 'Аэропорт',
'verbose_name_plural': 'Аэропорты',
'ordering': ('name', 'iata_code'),
},
),
]

28
ReferenceDataApp/migrations/0002_alter_airport_area_id_alter_city_area_id_and_more.py Normal file
View File

@@ -0,0 +1,28 @@
# Generated by Django 4.2.2 on 2023-07-19 17:09
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('ReferenceDataApp', '0001_initial'),
]
operations = [
migrations.AlterField(
model_name='airport',
name='area_id',
field=models.BigIntegerField(blank=True, null=True),
),
migrations.AlterField(
model_name='city',
name='area_id',
field=models.BigIntegerField(blank=True, null=True),
),
migrations.AlterField(
model_name='country',
name='area_id',
field=models.BigIntegerField(blank=True, null=True),
),
]

28
ReferenceDataApp/migrations/0003_airport_parsing_finished_city_parsing_finished_and_more.py Normal file
View File

@@ -0,0 +1,28 @@
# Generated by Django 4.2.2 on 2023-07-19 19:41
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('ReferenceDataApp', '0002_alter_airport_area_id_alter_city_area_id_and_more'),
]
operations = [
migrations.AddField(
model_name='airport',
name='parsing_finished',
field=models.BooleanField(default=False, verbose_name='Парсинг завершен'),
),
migrations.AddField(
model_name='city',
name='parsing_finished',
field=models.BooleanField(default=False, verbose_name='Парсинг завершен'),
),
migrations.AddField(
model_name='country',
name='parsing_finished',
field=models.BooleanField(default=False, verbose_name='Парсинг завершен'),
),
]

19
ReferenceDataApp/migrations/0004_alter_airport_city.py Normal file
View File

@@ -0,0 +1,19 @@
# Generated by Django 4.2.2 on 2023-07-19 19:52
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('ReferenceDataApp', '0003_airport_parsing_finished_city_parsing_finished_and_more'),
]
operations = [
migrations.AlterField(
model_name='airport',
name='city',
field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='rel_airports_for_city', to='ReferenceDataApp.city', verbose_name='Город'),
),
]

40
ReferenceDataApp/migrations/0005_remove_airport_parsing_finished_and_more.py Normal file
View File

@@ -0,0 +1,40 @@
# Generated by Django 4.2.2 on 2023-07-20 02:09
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('ReferenceDataApp', '0004_alter_airport_city'),
]
operations = [
migrations.RemoveField(
model_name='airport',
name='parsing_finished',
),
migrations.RemoveField(
model_name='city',
name='parsing_finished',
),
migrations.RemoveField(
model_name='country',
name='parsing_finished',
),
migrations.AddField(
model_name='airport',
name='parsing_finished_DT',
field=models.DateTimeField(blank=True, null=True, verbose_name='Дата и время завершения парсинга'),
),
migrations.AddField(
model_name='city',
name='parsing_finished_DT',
field=models.DateTimeField(blank=True, null=True, verbose_name='Дата и время завершения парсинга'),
),
migrations.AddField(
model_name='country',
name='parsing_finished_DT',
field=models.DateTimeField(blank=True, null=True, verbose_name='Дата и время завершения парсинга'),
),
]

0
ReferenceDataApp/migrations/__init__.py Normal file
View File

98
ReferenceDataApp/models.py Normal file
View File

@@ -0,0 +1,98 @@
from django.db import models
from BaseModels.base_models import BaseModel
from django.utils.translation import gettext_lazy as _
class Country(BaseModel):
international_name = models.CharField(max_length=250, verbose_name='Международное название', blank=True, null=True)
official_name = models.CharField(max_length=250, verbose_name='Официальное название', blank=True, null=True)
short_code = models.CharField(max_length=2, verbose_name='Код страны по ISO3166-1:alpha2')
code = models.CharField(max_length=3, verbose_name='Код страны по ISO3166-1:alpha3')
num_code = models.CharField(max_length=3, verbose_name='Код страны по ISO3166-1:numeric', blank=True, null=True)
flag_img_url = models.URLField(verbose_name='Ссылка на изображение флага', blank=True, null=True)
geo_lat = models.CharField(max_length=20, verbose_name='GPS широта', blank=True, null=True)
geo_lon = models.CharField(max_length=20, verbose_name='GPS долгота', blank=True, null=True)
area_id = models.BigIntegerField(blank=True, null=True)
parsing_finished_DT = models.DateTimeField(verbose_name='Дата и время завершения парсинга', blank=True, null=True)
def __str__(self):
if self.name:
return f'{self.name}'
elif self.international_name:
return f'{self.international_name}'
elif self.official_name:
return f'{self.official_name}'
elif self.code:
return f'{self.code}'
class Meta:
verbose_name = _('Страна')
verbose_name_plural = _('Страны')
ordering = ('name', 'code')
class City(BaseModel):
country = models.ForeignKey(
Country, verbose_name='Страна', related_name='rel_cities_for_country', on_delete=models.CASCADE)
geo_lat = models.CharField(max_length=20, verbose_name='GPS широта', blank=True, null=True)
geo_lon = models.CharField(max_length=20, verbose_name='GPS долгота', blank=True, null=True)
area_id = models.BigIntegerField(blank=True, null=True)
parsing_finished_DT = models.DateTimeField(verbose_name='Дата и время завершения парсинга', blank=True, null=True)
def __str__(self):
if self.name:
return f'{self.name}'
else:
return f'{self.id}'
class Meta:
verbose_name = _('Город')
verbose_name_plural = _('Города')
ordering = ('name',)
class Airport(BaseModel):
city = models.ForeignKey(
City, verbose_name='Город', related_name='rel_airports_for_city', on_delete=models.CASCADE,
blank=True, null=True)
international_name = models.CharField(max_length=250, verbose_name='Международное название', blank=True, null=True)
iata_code = models.CharField(max_length=3, verbose_name='IATA')
icao_code = models.CharField(max_length=4, verbose_name='ICAO')
geo_lat = models.CharField(max_length=20, verbose_name='GPS широта', blank=True, null=True)
geo_lon = models.CharField(max_length=20, verbose_name='GPS долгота', blank=True, null=True)
area_id = models.BigIntegerField(blank=True, null=True)
parsing_finished_DT = models.DateTimeField(verbose_name='Дата и время завершения парсинга', blank=True, null=True)
def __str__(self):
if self.name:
return f'{self.name}'
elif self.international_name:
return f'{self.international_name}'
elif self.iata_code:
return f'{self.iata_code}'
elif self.icao_code:
return f'{self.icao_code}'
else:
return f'{self.id}'
class Meta:
verbose_name = _('Аэропорт')
verbose_name_plural = _('Аэропорты')
ordering = ('name', 'iata_code')

3
ReferenceDataApp/tests.py Normal file
View File

@@ -0,0 +1,3 @@
from django.test import TestCase
# Create your tests here.

22
ReferenceDataApp/translation.py Normal file
View File

@@ -0,0 +1,22 @@
from modeltranslation.translator import translator, TranslationOptions
from .models import *
class Country_TranslationOptions(TranslationOptions):
fields = (
'name',
)
translator.register(Country, Country_TranslationOptions)
class City_TranslationOptions(TranslationOptions):
fields = (
'name',
)
translator.register(City, City_TranslationOptions)
class Airport_TranslationOptions(TranslationOptions):
fields = (
'name',
)
translator.register(Airport, Airport_TranslationOptions)

0
ReferenceDataApp/views.py Normal file
View File