automate the last manual part of the process

.github/workflows/request-linz-export.yml

@@ -0,0 +1,33 @@
+name: Request LINZ Export
+
+on:
+  schedule:
+    - cron: "6 16 * * 3" # at 04:06am on Wednesday NZST
+  workflow_dispatch:
+
+jobs:
+  changelog:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [20.x]
+
+    steps:
+      - name: ⏬ Checkout code
+        uses: actions/checkout@v3
+
+      - name: 🔢 Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - name: ⏬ Install
+        run: |
+          yarn
+
+      - name: 0️⃣ Request data export from LDS
+        run: |
+          yarn 0
+        env:
+          REACT_APP_LDS_KEY: ${{ secrets.REACT_APP_LDS_KEY }}

.github/workflows/sync.yml

@@ -0,0 +1,49 @@
+name: Weekly Sync
+
+on:
+  schedule:
+    - cron: "6 18 * * 3"
+    # at 06:06am on Wednesday NZST. This is 2 hours
+    # after we requested an export from LINZ.
+  workflow_dispatch:
+
+jobs:
+  changelog:
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        node-version: [20.x]
+
+    steps:
+      - name: ⏬ Checkout code
+        uses: actions/checkout@v3
+
+      - name: 🔢 Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v3
+        with:
+          node-version: ${{ matrix.node-version }}
+
+      - name: ⏬ Install
+        run: |
+          yarn
+
+      - name: 0️⃣.5️⃣ Download LINZ Export
+        run: |
+          yarn 0.5
+
+      - name: 1️⃣ Download OSM Planet file
+        run: |
+          yarn 1
+
+      - name: 2️⃣ Preprocess LINZ and OSM data
+        run: |
+          yarn 2
+
+      - name: 3️⃣ Conflate LINZ and OSM data
+        run: |
+          yarn 3
+
+      - name: 4️⃣ Upload result to CDN
+        run: |
+          yarn 4

README.md

@@ -1,17 +1,33 @@
 # Finds NZ streets that are missing in OpenStreetMap
 
 ![build](https://github.com/osm-nz/missing-streets/workflows/build/badge.svg)
+![](https://github.com/osm-nz/missing-streets/workflows/Request%20LINZ%20Export/badge.svg)
+![](https://github.com/osm-nz/missing-streets/workflows/Weekly%20Sync/badge.svg)
 
 Please see [the wiki page](https://wiki.osm.org/New_Zealand/Missing_Streets) for information about this project.
 
+If all three status badges above are green, then the script is automatically running once a week (on Wednesday morning NZ time)
+
 <details>
 <summary>Documentation for software developers (click to expand)</summary>
 
 Running the script:
 
 ```sh
 # first install nodejs and yarn
-# then download https://data.linz.govt.nz/layer/53382, save it as ./tmp/linz.csv
+
+# Then generate an API from https://data.linz.govt.nz/my/api
+# with "Full access to Exports Access"
+# create a file called `.env.local` in this folder, and add
+# REACT_APP_LDS_KEY=XXXXX
+# where XXXXX is the token you just generated.
+
+yarn 0 # this will request an export of the dataset from LINZ
+
+# now wait up to 1 hour for the export to be generated.
+# Login to https://data.linz.govt.nz to check progress
+
+yarn 0.5 # Downloads the LINZ export
 yarn 1
 yarn 2
 yarn 3 # generates the final file: ./public/conflationResult.geo.json

package.json

@@ -5,6 +5,8 @@
   "private": true,
   "homepage": "https://osm-nz.github.io/missing-streets",
   "scripts": {
+    "0": "ts-node --project tsconfig.backend.json script/0requestLinzExport",
+    "0.5": "ts-node --project tsconfig.backend.json script/0.5downloadLinzExport",
     "1": "sh script/1downloadPlanet.sh",
     "2": "ts-node --project tsconfig.backend.json script/2preprocess",
     "3": "ts-node --project tsconfig.backend.json script/3conflate",
@@ -16,9 +18,11 @@
   },
   "dependencies": {
     "@azure/storage-blob": "^12.9.0",
+    "adm-zip": "^0.5.10",
     "copy-text-to-clipboard": "^3.0.1",
     "csv-parser": "^3.0.0",
     "dotenv": "^16.0.3",
+    "koordinates-api": "^0.0.0",
     "leaflet": "^1.9.3",
     "osm-api": "^1.0.5",
     "pbf2json": "^6.10.0",
@@ -30,6 +34,7 @@
     "wellknown": "^0.5.0"
   },
   "devDependencies": {
+    "@types/adm-zip": "^0.5.5",
     "@types/geojson": "^7946.0.10",
     "@types/leaflet": "^1.9.0",
     "@types/node": "^18.11.18",

script/0.5downloadLinzExport.ts

@@ -0,0 +1,64 @@
+import { promises as fs } from "node:fs";
+import { parse } from "node:path";
+import Unzip from "adm-zip";
+import { api } from "./util/api";
+import { linzRawFile } from "./util";
+import { LINZ_LAYER_NAME_SUBSTR } from "../src/util";
+
+async function main() {
+  const allExports = await api.listExports();
+  const recentExports = allExports
+    .filter(
+      (item) =>
+        item.created_at &&
+        item.state !== "gone" &&
+        item.state !== "cancelled" &&
+        item.name.includes(LINZ_LAYER_NAME_SUBSTR)
+    )
+    .sort((a, b) => +new Date(b.created_at!) - +new Date(a.created_at!));
+
+  const mostRecent = recentExports[0];
+  if (!mostRecent) {
+    throw new Error("No recent exports found.");
+  }
+
+  console.log(
+    `Most recent export is #${mostRecent.id}, created at ${mostRecent.created_at}`
+  );
+
+  if (!mostRecent.download_url) {
+    throw new Error("Export has no download_url yet");
+  }
+
+  console.log(`Downloading from ${mostRecent.download_url} …`);
+
+  const tempFilePath = await api.downloadExport(mostRecent.download_url);
+  console.log(`Saved to ${tempFilePath}`);
+
+  const zip = new Unzip(tempFilePath);
+  const zipEntries = zip.getEntries();
+
+  const csvFile = zipEntries.find((file) => file.entryName.endsWith(".csv"));
+
+  if (!csvFile) {
+    throw new Error("No csv file in zip");
+  }
+
+  console.log(`Extracting ${csvFile.entryName}…`);
+
+  zip.extractEntryTo(
+    csvFile.entryName,
+    parse(linzRawFile).dir,
+    /* maintainEntryPath */ false,
+    /* overwrite */ true,
+    /* keepOriginalPermission */ undefined,
+    parse(linzRawFile).base
+  );
+
+  console.log("Deleting temp file…");
+  await fs.rm(tempFilePath, { force: true });
+
+  console.log("Done!");
+}
+
+main();

script/0requestLinzExport.ts

@@ -0,0 +1,8 @@
+import { LINZ_LAYER } from "../src/util";
+import { api } from "./util/api";
+
+async function main() {
+  const response = await api.generateExport(LINZ_LAYER);
+  console.log(`Requested export #${response.id}`);
+}
+main();

script/util/api.ts

@@ -0,0 +1,10 @@
+import { join } from "node:path";
+import { config as dotenv } from "dotenv";
+import { Koordinates } from "koordinates-api";
+
+dotenv({ path: join(__dirname, "../../.env.local") });
+
+export const api = new Koordinates({
+  host: "https://data.linz.govt.nz",
+  apiKey: process.env.REACT_APP_LDS_KEY!,
+});

src/util/const.ts

@@ -1 +1,2 @@
 export const LINZ_LAYER = 53382;
+export const LINZ_LAYER_NAME_SUBSTR = "roads-addressing";