Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 52 additions & 0 deletions docs/abroad-addresses.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
# Addresses of the abroad sections

The street address of every abroad section (election region 32, ids `32xxxxxxx`)
lives in `sections.address`. The МВнР publishes them as an xlsx on
<https://mfa.bg/bg/pvr2026/adresi> and changes them now and then; the API is the
one place they are kept, and the site reads them from `GET /sections/abroad`.

`place` of an abroad section stays the committee name from the CIK sheet
(„Посолство“, „Баркинг 2“). For domestic sections `place` is the address, and
`address` falls back to it (see `src/sections/section-address.ts`).

## Read

- `GET /sections/abroad` (public, cached 5 min): every abroad section with
`id`, `code`, `place`, `address` (null until loaded), `town {id, name}` and
`country {code, name}`.
- `GET /sections?town=`, `GET /sections/:id` and `/results/<section>.json`
include `address` too.

## Update

Only the address column of existing abroad sections is written. Unknown ids are
reported in `unknown`, never created; nothing is deleted. Every run reports
`updated` (`{id, from, to}`), `unchanged` and `unknown`.

1. Convert the xlsx:

```sh
uv run scripts/sections-preprocess/mfa_abroad_addresses.py adresi.xlsx abroad-addresses.csv
```

The CSV has `section_id,country,place,address`; only `section_id` and
`address` are used (`№ на секция` and `Адрес` work as headers too).

2. Load it, dry run first. Either on the server, in the `source` folder of the
environment (uses its `.env`):

```sh
npm run sections:update-abroad-addresses -- abroad-addresses.csv --dry-run
npm run sections:update-abroad-addresses -- abroad-addresses.csv
```

or over HTTP as an admin (`Manage Section`):

```sh
curl -X PATCH "$API/sections/abroad/addresses?dryRun=true" \
-H "Authorization: Bearer $TOKEN" -F file=@abroad-addresses.csv
curl -X PATCH "$API/sections/abroad/addresses" \
-H "Authorization: Bearer $TOKEN" -F file=@abroad-addresses.csv
```

A JSON body `{"sections": [{"id": "320902332", "address": "…"}]}` works too.
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
"seed:run": "npm run build && typeorm-ts-node-commonjs -d ./src/database/typeorm-seeds.datasource.ts migration:run",
"seed:revert": "typeorm-ts-node-commonjs -d ./src/database/typeorm-seeds.datasource.ts migration:revert",
"sections:update-domestic": "ts-node --transpile-only ./scripts/update-domestic-sections.ts",
"sections:update-abroad-addresses": "ts-node --transpile-only ./scripts/update-abroad-addresses.ts",
"firebase:token": "node ./scripts/get-token.js",
"deploy:production": "pm2 deploy ecosystem.config.json production update",
"deploy:staging": "pm2 deploy ecosystem.config.json staging update",
Expand Down
67 changes: 67 additions & 0 deletions scripts/sections-preprocess/mfa_abroad_addresses.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
# /// script
# requires-python = ">=3.10"
# dependencies = ["openpyxl"]
# ///
"""Turn the МВнР table of abroad sections (https://mfa.bg/bg/pvr2026/adresi)
into a CSV for `npm run sections:update-abroad-addresses` or
PATCH /sections/abroad/addresses.

Usage: uv run scripts/sections-preprocess/mfa_abroad_addresses.py <table.xlsx> [out.csv]

Columns: section_id, country, place, address. The address is the Bulgarian one,
else the local-language one. Writes to stdout without out.csv.
"""
import csv
import re
import sys

import openpyxl


def clean(value) -> str:
return re.sub(r"\s+", " ", str(value or "")).strip()


def read_sections(path: str) -> dict[str, dict[str, str]]:
sheet = openpyxl.load_workbook(path, read_only=True, data_only=True).worksheets[0]
sections: dict[str, dict[str, str]] = {}
country = ""
for row in sheet.iter_rows(values_only=True):
cells = [clean(cell) for cell in row[:7]]
while len(cells) < 7:
cells.append("")
_, country_cell, code, place, address, address_local, _ = cells
# The country is only on the first row of its group; the next row holds „N секции“.
if country_cell and not re.search(r"\d+\s+секци", country_cell):
country = country_cell
code = re.sub(r"\D", "", code)
if not re.fullmatch(r"32\d{7}", code):
continue
entry = {"country": country, "place": place, "address": address or address_local}
if code in sections and sections[code] != entry:
raise SystemExit(f"conflicting rows for {code}")
sections[code] = entry
return sections


def main() -> None:
if len(sys.argv) < 2:
raise SystemExit(__doc__)
sections = read_sections(sys.argv[1])
out = open(sys.argv[2], "w", newline="", encoding="utf-8") if len(sys.argv) > 2 else sys.stdout
writer = csv.writer(out, lineterminator="\n")
writer.writerow(["section_id", "country", "place", "address"])
for code, entry in sorted(sections.items()):
writer.writerow([code, entry["country"], entry["place"], entry["address"]])
if out is not sys.stdout:
out.close()
missing = [code for code, entry in sections.items() if not entry["address"]]
print(
f"{len(sections)} sections, {len(sections) - len(missing)} with an address"
+ (f"; without: {', '.join(sorted(missing))}" if missing else ""),
file=sys.stderr,
)


if __name__ == "__main__":
main()
62 changes: 62 additions & 0 deletions scripts/update-abroad-addresses.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/**
* Set the street addresses of the abroad sections (32*) from a CSV, straight
* against the database configured in .env (DATABASE_*). Same code as
* PATCH /sections/abroad/addresses: only the address of existing sections is
* written, unknown ids are reported and never created.
*
* uv run scripts/sections-preprocess/mfa_abroad_addresses.py adresi.xlsx > abroad-addresses.csv
* npm run sections:update-abroad-addresses -- abroad-addresses.csv --dry-run
* npm run sections:update-abroad-addresses -- abroad-addresses.csv
*/
import * as fs from 'fs'
import { AppDataSource } from '../src/database/typeorm.datasource'
import {
AbroadAddressesInputError,
parseAbroadAddressesCsv,
summarizeAbroadAddresses,
updateAbroadAddresses,
} from '../src/sections/abroad-addresses'

async function main(): Promise<number> {
const args = process.argv.slice(2)
const dryRun = args.includes('--dry-run')
const verbose = args.includes('--verbose')
const file = args.find((a) => !a.startsWith('--'))
if (!file) {
console.error(
'Usage: npm run sections:update-abroad-addresses -- <file.csv> [--dry-run] [--verbose]',
)
return 2
}
const input = parseAbroadAddressesCsv(fs.readFileSync(file))
console.error(
`Parsed ${input.length} abroad addresses from ${file}. Database: ${
AppDataSource.options.database as string
}${dryRun ? ' (dry run)' : ''}`,
)
await AppDataSource.initialize()
try {
const report = await updateAbroadAddresses(AppDataSource, input, {
dryRun,
})
console.error(summarizeAbroadAddresses(report))
console.log(
JSON.stringify(
verbose ? report : { updated: report.updated, unknown: report.unknown },
null,
2,
),
)
} finally {
await AppDataSource.destroy()
}
return 0
}

main().then(
(code) => process.exit(code),
(e: Error) => {
console.error(e instanceof AbroadAddressesInputError ? e.message : e)
process.exit(1)
},
)
69 changes: 69 additions & 0 deletions src/sections/abroad-addresses/abroad-addresses.parser.spec.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
import {
AbroadAddressesInputError,
parseAbroadAddressesCsv,
parseAbroadAddressesJson,
} from './abroad-addresses.parser'

describe('parseAbroadAddressesCsv', () => {
it('reads the converter output and ignores other columns', () => {
expect(
parseAbroadAddressesCsv(
'\uFEFFsection_id,country,place,address\n' +
'320902332,Норвегия,Тромсьо,"Kvaløyvegen 332,\n 9016 Tromsø"\n',
),
).toEqual([
{ sectionId: '320902332', address: 'Kvaløyvegen 332, 9016 Tromsø' },
])
})

it('accepts the Bulgarian headers', () => {
expect(
parseAbroadAddressesCsv('№ на секция,Адрес\n320010003,Moonah Pl 33\n'),
).toEqual([{ sectionId: '320010003', address: 'Moonah Pl 33' }])
})

it('drops exact repeats and refuses conflicting ones', () => {
expect(
parseAbroadAddressesCsv('id,address\n320010003,A\n320010003,A\n'),
).toHaveLength(1)
expect(() =>
parseAbroadAddressesCsv('id,address\n320010003,A\n320010003,B\n'),
).toThrow(AbroadAddressesInputError)
})

it('refuses a file without the columns or rows', () => {
expect(() => parseAbroadAddressesCsv('foo,bar\n1,2\n')).toThrow(
/header needs/,
)
expect(() => parseAbroadAddressesCsv('id,address\n')).toThrow(/no sections/)
})

it('refuses domestic ids', () => {
expect(() =>
parseAbroadAddressesCsv('id,address\n234615001,ул. Шипка 1\n'),
).toThrow(/not an abroad section id/)
})
})

describe('parseAbroadAddressesJson', () => {
it('takes {sections} or a bare array', () => {
const expected = [{ sectionId: '320902332', address: 'X 1' }]
expect(
parseAbroadAddressesJson({
sections: [{ id: '320902332', address: 'X 1' }],
}),
).toEqual(expected)
expect(
parseAbroadAddressesJson([{ section_id: '320902332', address: 'X 1' }]),
).toEqual(expected)
})

it('refuses anything else', () => {
expect(() => parseAbroadAddressesJson({})).toThrow(
AbroadAddressesInputError,
)
expect(() => parseAbroadAddressesJson(undefined)).toThrow(
AbroadAddressesInputError,
)
})
})
127 changes: 127 additions & 0 deletions src/sections/abroad-addresses/abroad-addresses.parser.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
import { parse } from 'csv-parse/sync'

/** One abroad section's street address, as sent to PATCH /sections/abroad/addresses. */
export interface AbroadAddressInput {
sectionId: string
address: string
}

export class AbroadAddressesInputError extends Error {
constructor(public readonly problems: string[]) {
super(`Invalid abroad addresses: ${problems.join('; ')}`)
}
}

const ABROAD_SECTION_ID = /^32\d{7}$/

const ID_HEADERS = [
'section_id',
'id',
'section',
'№ на секция',
'номер на секция',
]
const ADDRESS_HEADERS = ['address', 'адрес']

const normalizeHeader = (header: string): string =>
header
.replace(/^\uFEFF/, '')
.trim()
.toLowerCase()

const squash = (value: unknown): string =>
String(value ?? '')
.replace(/\s+/g, ' ')
.trim()

/**
* Checks the rows and drops repeats: every id is a 32xxxxxxx section, every
* address is non-empty, and an id repeated with another address is an error.
*/
export function validateAbroadAddresses(
rows: Array<{ sectionId?: unknown; address?: unknown }>,
): AbroadAddressInput[] {
const problems: string[] = []
const byId = new Map<string, string>()
rows.forEach((row, index) => {
const line = `row ${index + 1}`
const sectionId = String(row.sectionId ?? '').replace(/\s+/g, '')
const address = squash(row.address)
if (!ABROAD_SECTION_ID.test(sectionId)) {
problems.push(
`${line}: "${sectionId}" is not an abroad section id (32xxxxxxx)`,
)
return
}
if (!address) {
problems.push(`${line}: ${sectionId} has no address`)
return
}
const seen = byId.get(sectionId)
if (seen !== undefined && seen !== address) {
problems.push(`${line}: ${sectionId} has two different addresses`)
return
}
byId.set(sectionId, address)
})
if (byId.size === 0 && problems.length === 0) {
problems.push('no sections')
}
if (problems.length > 0) {
throw new AbroadAddressesInputError(problems)
}
return [...byId].map(([sectionId, address]) => ({ sectionId, address }))
}

/**
* CSV with a header row: an id column (`section_id`, `id` or `№ на секция`)
* and an `address` (`Адрес`) column. Other columns are ignored, so the file
* from scripts/sections-preprocess/mfa_abroad_addresses.py works as is.
*/
export function parseAbroadAddressesCsv(
input: string | Buffer,
): AbroadAddressInput[] {
let records: Record<string, string>[]
try {
records = parse(input, {
columns: (header: string[]) => header.map(normalizeHeader),
skip_empty_lines: true,
relax_column_count: true,
trim: true,
bom: true,
})
} catch (error) {
throw new AbroadAddressesInputError([`not a CSV file: ${error.message}`])
}
const first = records[0] ?? {}
const idHeader = ID_HEADERS.find((h) => h in first)
const addressHeader = ADDRESS_HEADERS.find((h) => h in first)
if (records.length > 0 && (!idHeader || !addressHeader)) {
throw new AbroadAddressesInputError([
`the header needs a section id column (${ID_HEADERS.join(
', ',
)}) and an address column (${ADDRESS_HEADERS.join(', ')})`,
])
}
return validateAbroadAddresses(
records.map((r) => ({ sectionId: r[idHeader], address: r[addressHeader] })),
)
}

/** JSON body: `{ "sections": [{ "id": "320902332", "address": "…" }] }` or the bare array. */
export function parseAbroadAddressesJson(body: unknown): AbroadAddressInput[] {
const list = Array.isArray(body)
? body
: (body as { sections?: unknown })?.sections
if (!Array.isArray(list)) {
throw new AbroadAddressesInputError([
'expected a CSV "file" upload or a JSON body {"sections": [{"id", "address"}]}',
])
}
return validateAbroadAddresses(
list.map((item: Record<string, unknown>) => ({
sectionId: item?.id ?? item?.section_id ?? item?.sectionId,
address: item?.address,
})),
)
}
Loading
Loading