Skip to content

Commit

Permalink
Migrate from old GURS data to new format
Browse files Browse the repository at this point in the history
  • Loading branch information
@DavidKarlas
DavidKarlas committed Jul 3, 2023
1 parent 795852a commit 8041226
Show file tree
Hide file tree
Showing 4 changed files with 145 additions and 197 deletions.
215 changes: 67 additions & 148 deletions OsmGursBuildingImport/GursData.cs
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Data;
using System.Diagnostics;
using System.Globalization;
using System.IO;
Expand All @@ -19,11 +20,10 @@
namespace OsmGursBuildingImport
{
record BilingualName(string Name, string NameSecondLanguage);
record PostInfo(short Id, string Name);// Posts that are in bilingual area have "Koper - Capodistria" format
record PostInfo(short Id, BilingualName Name);
record VotingArea(Geometry Geometry, string Name, string Id);
record BuildingInfo(int Id, Geometry Geometry, string? Date, List<Address>? Addresses);
record SettlementInfo(Geometry Geometry, BilingualName Name);
record Address(int Id, Geometry Geometry, string Date, string HouseNumber, BilingualName StreetName, PostInfo PostInfo, BilingualName VillageName);
record BuildingInfo(long Id, Geometry Geometry, string? Date, List<Address>? Addresses);
record Address(long Id, Geometry Geometry, string Date, string HouseNumber, BilingualName StreetName, PostInfo PostInfo, BilingualName VillageName);
record ProcessingArea(Geometry Geometry, string Name, List<BuildingInfo> Buildings, string pathToPoly)
{
public bool Process { get; set; }
Expand All @@ -34,23 +34,17 @@ class GursData
private static GeometryFactory D96Factory = NtsGeometryServices.Instance.CreateGeometryFactory(new PrecisionModel(), 3794);
public Dictionary<int, ProcessingArea> ProcessingAreas = new();

Dictionary<int, BilingualName> Streets = new();
public Dictionary<int, SettlementInfo> Settlements = new();
Dictionary<int, PostInfo> Posts = new();
Dictionary<int, Address> Addresses = new();
Dictionary<long, Address> Addresses = new();
List<VotingArea> VotingAreas = new();
Dictionary<string, Dictionary<string, string>> Overrides = new();
STRtree<BuildingInfo> BuildingsIndex = new();

STRtree<BuildingInfo> BuildingsIndex = new();
Dictionary<long, List<Address>> BuildingToAddresses = new();

public GursData(string dir, string overridesDir, string tempDir)
{
LoadOverrides(overridesDir);
LoadStreets(dir);
LoadSettlements(dir);
LoadPosts(dir);
LoadAddresses(dir);
LoadBuildings(dir);
//LoadVotingAreas(dir);
LoadVotingAreasGeoJson();

BuildProcessingAreas(tempDir);
Expand Down Expand Up @@ -127,8 +121,7 @@ private void BuildProcessingAreas(string tempDir)
ProcessingAreas.Add(int.Parse(votingArea.Id), newArea);
}

Parallel.ForEach(ProcessingAreas.Values, (area) =>
{
Parallel.ForEach(ProcessingAreas.Values, (area) => {
foreach (var aprox in BuildingsIndex.Query(area.Geometry.EnvelopeInternal))
{
if (!area.Geometry.Intersects(aprox.Geometry))
Expand All @@ -151,53 +144,6 @@ private void LoadOverrides(string overridesDir)
}
}

void LoadStreets(string dir)
{
var dict = Overrides["UL_UIME"];
var dictBi = Overrides["UL_DJ"];
using var csv = Sylvan.Data.Csv.CsvDataReader.Create(Path.Combine(dir, "UL_VSE", "UL_VSE.csv"));
while (csv.Read())
{
if (csv.GetInt32(2) == 0)
continue;
Streets.Add(
csv.GetInt32(1),
new BilingualName(
OverrideString(dict, csv.GetString(3)),
OverrideString(dictBi, csv.GetString(4))));
}
}

void LoadSettlements(string dir)
{
var dict = Overrides["NA_UIME"];
using var shapeReader = new ShapefileDataReader(Path.Combine(dir, "NA", "NA.shp"), D96Factory);
while (shapeReader.Read())
{
shapeReader.Geometry.Apply(D96Converter.Instance);
Settlements.Add(shapeReader.GetInt32(2),
new SettlementInfo(
shapeReader.Geometry,
new BilingualName(
OverrideString(dict, shapeReader.GetString(4)),
shapeReader.GetString(5).Replace("\0", ""))));
}
}

void LoadPosts(string dir)
{
using var shapeReader = new ShapefileDataReader(Path.Combine(dir, "PT", "PT.shp"), D96Factory);
var dict = Overrides["PT_UIME"];
while (shapeReader.Read())
{
shapeReader.Geometry.Apply(D96Converter.Instance);
Posts.Add(shapeReader.GetInt32(2),
new PostInfo(
shapeReader.GetInt16(3),
OverrideString(dict, shapeReader.GetString(4))));
}
}

private static string OverrideString(Dictionary<string, string> dict, string original)
{
if (dict.TryGetValue(original, out var overriden))
Expand All @@ -207,27 +153,29 @@ private static string OverrideString(Dictionary<string, string> dict, string ori

void LoadAddresses(string dir)
{
using var shapeReader = new ShapefileDataReader(Path.Combine(dir, "HS", "HS.shp"), D96Factory);
while (shapeReader.Read())
{
var status = shapeReader.GetString(13);
if (status != "V")
continue;

shapeReader.Geometry.Apply(D96Converter.Instance);
int id = shapeReader.GetInt32(2);
var date = shapeReader.GetDateTime(11);
BilingualName settlementName = Settlements[shapeReader.GetInt32(7)].Name;
Address addr = new Address(
id,
shapeReader.Geometry,
date.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture),
shapeReader.GetString(5).ToLower(),
Streets.TryGetValue(shapeReader.GetInt32(6), out var street) ? street : settlementName,
Posts[shapeReader.GetInt32(9)],
settlementName
);
Addresses.Add(id, addr);
var streetNameOverride = Overrides["UL_UIME"];
var streetNameSecondaryLanguageOverride = Overrides["UL_DJ"];
var settlementNameOverride = Overrides["NA_UIME"];
var postalNameOverride = Overrides["PT_UIME"];

using var csvAddresses = Sylvan.Data.Csv.CsvDataReader.Create(Directory.GetFiles(Path.Combine(dir, "Addresses"), "KN_SLO_NASLOVI_HS_*.csv").Single());
var wktReader = new WKTReader(D96Factory.GeometryServices);
while (csvAddresses.Read())
{
var buildingId = csvAddresses.GetInt64("EID_STAVBA");
var id = csvAddresses.GetInt64("EID_HISNA_STEVILKA");
var geom = wktReader.Read(csvAddresses.GetString("GEOM"));
geom.Apply(D96Converter.Instance);//Convert from D96 to OSM coordinate system
var houseNumber = csvAddresses.GetString("HS_STEVILKA") + csvAddresses.GetString("HS_DODATEK");
var streetName = new BilingualName(OverrideString(streetNameOverride, csvAddresses.GetString("ULICA_NAZIV")), OverrideString(streetNameSecondaryLanguageOverride, csvAddresses.GetString("ULICA_NAZIV_DJ")));
var postInfo = new PostInfo(csvAddresses.GetInt16("POSTNI_OKOLIS_SIFRA"), new BilingualName(OverrideString(postalNameOverride, csvAddresses.GetString("POSTNI_OKOLIS_NAZIV")), csvAddresses.GetString("POSTNI_OKOLIS_NAZIV_DJ")));
var villageName = new BilingualName(OverrideString(settlementNameOverride, csvAddresses.GetString("NASELJE_NAZIV")), csvAddresses.GetString("NASELJE_NAZIV_DJ"));
var address = new Address(id, geom, null, houseNumber, streetName, postInfo, villageName);
Addresses.Add(id, address);
if (BuildingToAddresses.TryGetValue(buildingId, out var list))
list.Add(address);
else
BuildingToAddresses.Add(buildingId, new List<Address>() { address });
}
}

Expand All @@ -247,72 +195,43 @@ void LoadVotingAreasGeoJson()
}
}

void LoadVotingAreas(string dir)
{
using var shapeReader = new ShapefileDataReader(Path.Combine(dir, "VLV", "VLV.shp"), D96Factory);
while (shapeReader.Read())
{
if (shapeReader.GetString(1) != "LV")
continue;
shapeReader.Geometry.Apply(D96Converter.Instance);
VotingArea votingArea = new VotingArea(
shapeReader.Geometry,
shapeReader.GetString(4),
shapeReader.GetString(3));
VotingAreas.Add(votingArea);
}
}

void LoadBuildings(string dir)
{
var buildingToAddresses = new Dictionary<long, List<Address>>();
using var csvAddresses = Sylvan.Data.Csv.CsvDataReader.Create(Directory.GetFiles(Path.Combine(dir, "KS_SLO_CSV_A_U"), "KS_SLO_KHS_*.csv").Single());
while (csvAddresses.Read())
{
var sta_sid = csvAddresses.GetInt64(0);
var hs_mid = csvAddresses.GetInt32(1);
try
{
if (buildingToAddresses.TryGetValue(sta_sid, out var list))
list.Add(Addresses[hs_mid]);
else
buildingToAddresses.Add(sta_sid, new List<Address>() { Addresses[hs_mid] });

}
catch (KeyNotFoundException)
{
Console.WriteLine(
$"GURS has out of sync files?" +
$"Building with STA_SID:'{sta_sid}' is referencing non-existing address with HS_MID:'{hs_mid}'");
}
}

// Just date for now...
var buildingToInfo = new Dictionary<long, string?>();
using var csvInfo = Sylvan.Data.Csv.CsvDataReader.Create(Directory.GetFiles(Path.Combine(dir, "KS_SLO_CSV_A_U"), "KS_SLO_KST_*.csv").Single());
while (csvInfo.Read())
{
var sta_sid = csvInfo.GetInt64(0);
string date = csvInfo.GetString(12);
if (date == "")
buildingToInfo[sta_sid] = null;
else
buildingToInfo[sta_sid] = $"{date.Substring(6, 4)}-{date.Substring(3, 2)}-{date.Substring(0, 2)}";
}
var shapeReader = new ShapefileDataReader(Directory.GetFiles(Path.Combine(dir, "KS_SLO_SHP_G"), "KS_SLO_TLORISI_*.shp").Single(), D96Factory);
while (shapeReader.Read())
{
shapeReader.Geometry.Apply(D96Converter.Instance);
var id = shapeReader.GetInt32(1);

if (!buildingToAddresses.TryGetValue(id, out var addresses))
addresses = null;
BuildingsIndex.Insert(shapeReader.Geometry.EnvelopeInternal, new BuildingInfo(
id,
shapeReader.Geometry,
buildingToInfo[id],
addresses));
}
HashSet<long> insertedBuildings = new();
var shapeReader = new ShapefileDataReader(Directory.GetFiles(Path.Combine(dir, "Buildings", "KN_SLO_STAVBE_SLO_STAVBE_NADZEMNI_TLORIS"), "*.shp").Single(), D96Factory);
while (shapeReader.Read())
{
shapeReader.Geometry.Apply(D96Converter.Instance);
var id = shapeReader.GetInt64(2);

if (!BuildingToAddresses.TryGetValue(id, out var addresses))
addresses = null;
insertedBuildings.Add(id);
BuildingsIndex.Insert(shapeReader.Geometry.EnvelopeInternal, new BuildingInfo(
id,
shapeReader.Geometry,
null,
addresses));
}

shapeReader = new ShapefileDataReader(Directory.GetFiles(Path.Combine(dir, "Buildings", "KN_SLO_STAVBE_SLO_STAVBE_TLORIS"), "*.shp").Single(), D96Factory);
while (shapeReader.Read())
{
shapeReader.Geometry.Apply(D96Converter.Instance);
var id = shapeReader.GetInt64(2);

// Don't add if KN_SLO_STAVBE_SLO_STAVBE_NADZEMNI_TLORIS already added
if (!insertedBuildings.Add(id))
continue;

if (!BuildingToAddresses.TryGetValue(id, out var addresses))
addresses = null;
BuildingsIndex.Insert(shapeReader.Geometry.EnvelopeInternal, new BuildingInfo(
id,
shapeReader.Geometry,
null,
addresses));
}
BuildingsIndex.Build();
}
}
Expand Down
42 changes: 20 additions & 22 deletions OsmGursBuildingImport/OsmBuilder.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,7 @@ private ICompleteOsmGeo LineStringToWay(LineString lineString)
nodes.Add((Node)GeometryToOsmGeo(new Point(coord)));
}

return Add(new CompleteWay
{
return Add(new CompleteWay {
Id = newIdCounter--,
Nodes = nodes.ToArray()
});
Expand Down Expand Up @@ -151,18 +150,23 @@ public static bool SetAddressAttributes(Address address, TagsCollectionBase attr
anythingWasSet |= UpdateAttribute(attributes, "addr:street" + Suffix(address.Geometry.Coordinate), address.StreetName.NameSecondLanguage);
}

anythingWasSet |= UpdateAttribute(attributes, "addr:city", address.PostInfo.Name);
if (address.PostInfo.Name.Contains("/"))
{
anythingWasSet |= UpdateAttribute(attributes, "addr:city:sl", address.PostInfo.Name.Remove(address.PostInfo.Name.IndexOf(" / ")));
anythingWasSet |= UpdateAttribute(attributes, "addr:city" + Suffix(address.Geometry.Coordinate), address.PostInfo.Name.Substring(address.PostInfo.Name.IndexOf(" / ") + 3));
if (string.IsNullOrEmpty(address.PostInfo.Name.NameSecondLanguage))
{
anythingWasSet |= UpdateAttribute(attributes, "addr:city", address.PostInfo.Name.Name);
}
else
{
anythingWasSet |= UpdateAttribute(attributes, "addr:city", address.PostInfo.Name.Name + " / " + address.PostInfo.Name.NameSecondLanguage);
anythingWasSet |= UpdateAttribute(attributes, "addr:city:sl", address.PostInfo.Name.Name);
anythingWasSet |= UpdateAttribute(attributes, "addr:city" + Suffix(address.Geometry.Coordinate), address.PostInfo.Name.NameSecondLanguage);
}

anythingWasSet |= UpdateAttribute(attributes, "addr:postcode", address.PostInfo.Id.ToString());

// We want to add village only when it's not already mentioned, so when user enters
// some address into navigation it re-assuress them when seeing also correct village name...
// It is pretty common in Slovenia for people to be more fimiliar with village name than street names.
if (!address.PostInfo.Name.StartsWith(address.VillageName.Name) &&
// some address into navigation it re-assures them when seeing also correct village name...
// It is pretty common in Slovenia for people to be more familiar with village name than street names.
if (!address.PostInfo.Name.Name.StartsWith(address.VillageName.Name) &&
address.StreetName.Name != address.VillageName.Name)
{
if (string.IsNullOrEmpty(address.VillageName.NameSecondLanguage))
Expand Down Expand Up @@ -222,23 +226,20 @@ public ICompleteOsmGeo GeometryToOsmGeo(Geometry geometry)
{
var members = new List<CompleteRelationMember>();
var outerWay = LineStringToWay(polygon.Shell);
members.Add(new CompleteRelationMember()
{
members.Add(new CompleteRelationMember() {
Member = outerWay,
Role = "outer"
});
foreach (var pol in polygon.InteriorRings)
{
var osmGeo = LineStringToWay(pol);
members.Add(new CompleteRelationMember
{
members.Add(new CompleteRelationMember {
Member = osmGeo,
Role = "inner"
});
}

return Add(new CompleteRelation()
{
return Add(new CompleteRelation() {
Id = newIdCounter--,
Members = members.ToArray(),
Tags = new TagsCollection()
Expand All @@ -254,15 +255,13 @@ public ICompleteOsmGeo GeometryToOsmGeo(Geometry geometry)
foreach (var pol in multiPolygon.Geometries)
{
var osmGeo = GeometryToOsmGeo(pol);
members.Add(new CompleteRelationMember()
{
members.Add(new CompleteRelationMember() {
Member = osmGeo,
Role = "outer"
});
}

return Add(new CompleteRelation()
{
return Add(new CompleteRelation() {
Id = newIdCounter--,
Members = members.ToArray(),
Tags = new TagsCollection()
Expand All @@ -280,8 +279,7 @@ public Node GetOrCreateNode(Point point)
{
if (ExistingNodes.TryGetValue(point.Coordinate, out var node))
return node;
var newNode = new Node()
{
var newNode = new Node() {
Id = newIdCounter--,
Longitude = point.X,
Latitude = point.Y
Expand Down
Loading

0 comments on commit 8041226

Please sign in to comment.