From ecbad79b26fedc67ba9c7b019f17224d37096bb1 Mon Sep 17 00:00:00 2001 From: Haralan Dobrev Date: Mon, 26 Oct 2026 09:45:59 +0200 Subject: [PATCH] Keep the abroad section addresses in the API. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET /sections/abroad lists every abroad section with its town, country, committee name (place) and street address, publicly, so the site no longer needs a generated copy of the МВнР table. PATCH /sections/abroad/addresses (Manage Section) and npm run sections:update-abroad-addresses set the address of existing 32* sections from a CSV or JSON: only that column is written, unknown ids are reported and never created, nothing is deleted, dryRun rolls back. scripts/sections-preprocess/mfa_abroad_addresses.py turns the МВнР xlsx into that CSV. See docs/abroad-addresses.md. --- docs/abroad-addresses.md | 52 +++++++ package.json | 1 + .../mfa_abroad_addresses.py | 67 ++++++++ scripts/update-abroad-addresses.ts | 62 ++++++++ .../abroad-addresses.parser.spec.ts | 69 +++++++++ .../abroad-addresses.parser.ts | 127 +++++++++++++++ .../abroad-addresses.service.ts | 28 ++++ .../abroad-addresses.updater.spec.ts | 136 +++++++++++++++++ .../abroad-addresses.updater.ts | 113 ++++++++++++++ .../abroad-addresses/abroad-sections.list.ts | 46 ++++++ src/sections/abroad-addresses/index.ts | 4 + .../api/abroad-addresses.controller.spec.ts | 144 ++++++++++++++++++ .../api/abroad-addresses.controller.ts | 114 ++++++++++++++ src/sections/sections.module.ts | 5 + 14 files changed, 968 insertions(+) create mode 100644 docs/abroad-addresses.md create mode 100644 scripts/sections-preprocess/mfa_abroad_addresses.py create mode 100644 scripts/update-abroad-addresses.ts create mode 100644 src/sections/abroad-addresses/abroad-addresses.parser.spec.ts create mode 100644 src/sections/abroad-addresses/abroad-addresses.parser.ts create mode 100644 src/sections/abroad-addresses/abroad-addresses.service.ts create mode 100644 src/sections/abroad-addresses/abroad-addresses.updater.spec.ts create mode 100644 src/sections/abroad-addresses/abroad-addresses.updater.ts create mode 100644 src/sections/abroad-addresses/abroad-sections.list.ts create mode 100644 src/sections/abroad-addresses/index.ts create mode 100644 src/sections/api/abroad-addresses.controller.spec.ts create mode 100644 src/sections/api/abroad-addresses.controller.ts diff --git a/docs/abroad-addresses.md b/docs/abroad-addresses.md new file mode 100644 index 00000000..66d6f657 --- /dev/null +++ b/docs/abroad-addresses.md @@ -0,0 +1,52 @@ +# Addresses of the abroad sections + +The street address of every abroad section (election region 32, ids `32xxxxxxx`) +lives in `sections.address`. The МВнР publishes them as an xlsx on + and changes them now and then; the API is the +one place they are kept, and the site reads them from `GET /sections/abroad`. + +`place` of an abroad section stays the committee name from the CIK sheet +(„Посолство“, „Баркинг 2“). For domestic sections `place` is the address, and +`address` falls back to it (see `src/sections/section-address.ts`). + +## Read + +- `GET /sections/abroad` (public, cached 5 min): every abroad section with + `id`, `code`, `place`, `address` (null until loaded), `town {id, name}` and + `country {code, name}`. +- `GET /sections?town=`, `GET /sections/:id` and `/results/
.json` + include `address` too. + +## Update + +Only the address column of existing abroad sections is written. Unknown ids are +reported in `unknown`, never created; nothing is deleted. Every run reports +`updated` (`{id, from, to}`), `unchanged` and `unknown`. + +1. Convert the xlsx: + + ```sh + uv run scripts/sections-preprocess/mfa_abroad_addresses.py adresi.xlsx abroad-addresses.csv + ``` + + The CSV has `section_id,country,place,address`; only `section_id` and + `address` are used (`№ на секция` and `Адрес` work as headers too). + +2. Load it, dry run first. Either on the server, in the `source` folder of the + environment (uses its `.env`): + + ```sh + npm run sections:update-abroad-addresses -- abroad-addresses.csv --dry-run + npm run sections:update-abroad-addresses -- abroad-addresses.csv + ``` + + or over HTTP as an admin (`Manage Section`): + + ```sh + curl -X PATCH "$API/sections/abroad/addresses?dryRun=true" \ + -H "Authorization: Bearer $TOKEN" -F file=@abroad-addresses.csv + curl -X PATCH "$API/sections/abroad/addresses" \ + -H "Authorization: Bearer $TOKEN" -F file=@abroad-addresses.csv + ``` + + A JSON body `{"sections": [{"id": "320902332", "address": "…"}]}` works too. diff --git a/package.json b/package.json index 0f20e1e7..561a745b 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "seed:run": "npm run build && typeorm-ts-node-commonjs -d ./src/database/typeorm-seeds.datasource.ts migration:run", "seed:revert": "typeorm-ts-node-commonjs -d ./src/database/typeorm-seeds.datasource.ts migration:revert", "sections:update-domestic": "ts-node --transpile-only ./scripts/update-domestic-sections.ts", + "sections:update-abroad-addresses": "ts-node --transpile-only ./scripts/update-abroad-addresses.ts", "firebase:token": "node ./scripts/get-token.js", "deploy:production": "pm2 deploy ecosystem.config.json production update", "deploy:staging": "pm2 deploy ecosystem.config.json staging update", diff --git a/scripts/sections-preprocess/mfa_abroad_addresses.py b/scripts/sections-preprocess/mfa_abroad_addresses.py new file mode 100644 index 00000000..49dd30ad --- /dev/null +++ b/scripts/sections-preprocess/mfa_abroad_addresses.py @@ -0,0 +1,67 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = ["openpyxl"] +# /// +"""Turn the МВнР table of abroad sections (https://mfa.bg/bg/pvr2026/adresi) +into a CSV for `npm run sections:update-abroad-addresses` or +PATCH /sections/abroad/addresses. + +Usage: uv run scripts/sections-preprocess/mfa_abroad_addresses.py [out.csv] + +Columns: section_id, country, place, address. The address is the Bulgarian one, +else the local-language one. Writes to stdout without out.csv. +""" +import csv +import re +import sys + +import openpyxl + + +def clean(value) -> str: + return re.sub(r"\s+", " ", str(value or "")).strip() + + +def read_sections(path: str) -> dict[str, dict[str, str]]: + sheet = openpyxl.load_workbook(path, read_only=True, data_only=True).worksheets[0] + sections: dict[str, dict[str, str]] = {} + country = "" + for row in sheet.iter_rows(values_only=True): + cells = [clean(cell) for cell in row[:7]] + while len(cells) < 7: + cells.append("") + _, country_cell, code, place, address, address_local, _ = cells + # The country is only on the first row of its group; the next row holds „N секции“. + if country_cell and not re.search(r"\d+\s+секци", country_cell): + country = country_cell + code = re.sub(r"\D", "", code) + if not re.fullmatch(r"32\d{7}", code): + continue + entry = {"country": country, "place": place, "address": address or address_local} + if code in sections and sections[code] != entry: + raise SystemExit(f"conflicting rows for {code}") + sections[code] = entry + return sections + + +def main() -> None: + if len(sys.argv) < 2: + raise SystemExit(__doc__) + sections = read_sections(sys.argv[1]) + out = open(sys.argv[2], "w", newline="", encoding="utf-8") if len(sys.argv) > 2 else sys.stdout + writer = csv.writer(out, lineterminator="\n") + writer.writerow(["section_id", "country", "place", "address"]) + for code, entry in sorted(sections.items()): + writer.writerow([code, entry["country"], entry["place"], entry["address"]]) + if out is not sys.stdout: + out.close() + missing = [code for code, entry in sections.items() if not entry["address"]] + print( + f"{len(sections)} sections, {len(sections) - len(missing)} with an address" + + (f"; without: {', '.join(sorted(missing))}" if missing else ""), + file=sys.stderr, + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/update-abroad-addresses.ts b/scripts/update-abroad-addresses.ts new file mode 100644 index 00000000..8f5df530 --- /dev/null +++ b/scripts/update-abroad-addresses.ts @@ -0,0 +1,62 @@ +/** + * Set the street addresses of the abroad sections (32*) from a CSV, straight + * against the database configured in .env (DATABASE_*). Same code as + * PATCH /sections/abroad/addresses: only the address of existing sections is + * written, unknown ids are reported and never created. + * + * uv run scripts/sections-preprocess/mfa_abroad_addresses.py adresi.xlsx > abroad-addresses.csv + * npm run sections:update-abroad-addresses -- abroad-addresses.csv --dry-run + * npm run sections:update-abroad-addresses -- abroad-addresses.csv + */ +import * as fs from 'fs' +import { AppDataSource } from '../src/database/typeorm.datasource' +import { + AbroadAddressesInputError, + parseAbroadAddressesCsv, + summarizeAbroadAddresses, + updateAbroadAddresses, +} from '../src/sections/abroad-addresses' + +async function main(): Promise { + const args = process.argv.slice(2) + const dryRun = args.includes('--dry-run') + const verbose = args.includes('--verbose') + const file = args.find((a) => !a.startsWith('--')) + if (!file) { + console.error( + 'Usage: npm run sections:update-abroad-addresses -- [--dry-run] [--verbose]', + ) + return 2 + } + const input = parseAbroadAddressesCsv(fs.readFileSync(file)) + console.error( + `Parsed ${input.length} abroad addresses from ${file}. Database: ${ + AppDataSource.options.database as string + }${dryRun ? ' (dry run)' : ''}`, + ) + await AppDataSource.initialize() + try { + const report = await updateAbroadAddresses(AppDataSource, input, { + dryRun, + }) + console.error(summarizeAbroadAddresses(report)) + console.log( + JSON.stringify( + verbose ? report : { updated: report.updated, unknown: report.unknown }, + null, + 2, + ), + ) + } finally { + await AppDataSource.destroy() + } + return 0 +} + +main().then( + (code) => process.exit(code), + (e: Error) => { + console.error(e instanceof AbroadAddressesInputError ? e.message : e) + process.exit(1) + }, +) diff --git a/src/sections/abroad-addresses/abroad-addresses.parser.spec.ts b/src/sections/abroad-addresses/abroad-addresses.parser.spec.ts new file mode 100644 index 00000000..787727ff --- /dev/null +++ b/src/sections/abroad-addresses/abroad-addresses.parser.spec.ts @@ -0,0 +1,69 @@ +import { + AbroadAddressesInputError, + parseAbroadAddressesCsv, + parseAbroadAddressesJson, +} from './abroad-addresses.parser' + +describe('parseAbroadAddressesCsv', () => { + it('reads the converter output and ignores other columns', () => { + expect( + parseAbroadAddressesCsv( + '\uFEFFsection_id,country,place,address\n' + + '320902332,Норвегия,Тромсьо,"Kvaløyvegen 332,\n 9016 Tromsø"\n', + ), + ).toEqual([ + { sectionId: '320902332', address: 'Kvaløyvegen 332, 9016 Tromsø' }, + ]) + }) + + it('accepts the Bulgarian headers', () => { + expect( + parseAbroadAddressesCsv('№ на секция,Адрес\n320010003,Moonah Pl 33\n'), + ).toEqual([{ sectionId: '320010003', address: 'Moonah Pl 33' }]) + }) + + it('drops exact repeats and refuses conflicting ones', () => { + expect( + parseAbroadAddressesCsv('id,address\n320010003,A\n320010003,A\n'), + ).toHaveLength(1) + expect(() => + parseAbroadAddressesCsv('id,address\n320010003,A\n320010003,B\n'), + ).toThrow(AbroadAddressesInputError) + }) + + it('refuses a file without the columns or rows', () => { + expect(() => parseAbroadAddressesCsv('foo,bar\n1,2\n')).toThrow( + /header needs/, + ) + expect(() => parseAbroadAddressesCsv('id,address\n')).toThrow(/no sections/) + }) + + it('refuses domestic ids', () => { + expect(() => + parseAbroadAddressesCsv('id,address\n234615001,ул. Шипка 1\n'), + ).toThrow(/not an abroad section id/) + }) +}) + +describe('parseAbroadAddressesJson', () => { + it('takes {sections} or a bare array', () => { + const expected = [{ sectionId: '320902332', address: 'X 1' }] + expect( + parseAbroadAddressesJson({ + sections: [{ id: '320902332', address: 'X 1' }], + }), + ).toEqual(expected) + expect( + parseAbroadAddressesJson([{ section_id: '320902332', address: 'X 1' }]), + ).toEqual(expected) + }) + + it('refuses anything else', () => { + expect(() => parseAbroadAddressesJson({})).toThrow( + AbroadAddressesInputError, + ) + expect(() => parseAbroadAddressesJson(undefined)).toThrow( + AbroadAddressesInputError, + ) + }) +}) diff --git a/src/sections/abroad-addresses/abroad-addresses.parser.ts b/src/sections/abroad-addresses/abroad-addresses.parser.ts new file mode 100644 index 00000000..77eb08d7 --- /dev/null +++ b/src/sections/abroad-addresses/abroad-addresses.parser.ts @@ -0,0 +1,127 @@ +import { parse } from 'csv-parse/sync' + +/** One abroad section's street address, as sent to PATCH /sections/abroad/addresses. */ +export interface AbroadAddressInput { + sectionId: string + address: string +} + +export class AbroadAddressesInputError extends Error { + constructor(public readonly problems: string[]) { + super(`Invalid abroad addresses: ${problems.join('; ')}`) + } +} + +const ABROAD_SECTION_ID = /^32\d{7}$/ + +const ID_HEADERS = [ + 'section_id', + 'id', + 'section', + '№ на секция', + 'номер на секция', +] +const ADDRESS_HEADERS = ['address', 'адрес'] + +const normalizeHeader = (header: string): string => + header + .replace(/^\uFEFF/, '') + .trim() + .toLowerCase() + +const squash = (value: unknown): string => + String(value ?? '') + .replace(/\s+/g, ' ') + .trim() + +/** + * Checks the rows and drops repeats: every id is a 32xxxxxxx section, every + * address is non-empty, and an id repeated with another address is an error. + */ +export function validateAbroadAddresses( + rows: Array<{ sectionId?: unknown; address?: unknown }>, +): AbroadAddressInput[] { + const problems: string[] = [] + const byId = new Map() + rows.forEach((row, index) => { + const line = `row ${index + 1}` + const sectionId = String(row.sectionId ?? '').replace(/\s+/g, '') + const address = squash(row.address) + if (!ABROAD_SECTION_ID.test(sectionId)) { + problems.push( + `${line}: "${sectionId}" is not an abroad section id (32xxxxxxx)`, + ) + return + } + if (!address) { + problems.push(`${line}: ${sectionId} has no address`) + return + } + const seen = byId.get(sectionId) + if (seen !== undefined && seen !== address) { + problems.push(`${line}: ${sectionId} has two different addresses`) + return + } + byId.set(sectionId, address) + }) + if (byId.size === 0 && problems.length === 0) { + problems.push('no sections') + } + if (problems.length > 0) { + throw new AbroadAddressesInputError(problems) + } + return [...byId].map(([sectionId, address]) => ({ sectionId, address })) +} + +/** + * CSV with a header row: an id column (`section_id`, `id` or `№ на секция`) + * and an `address` (`Адрес`) column. Other columns are ignored, so the file + * from scripts/sections-preprocess/mfa_abroad_addresses.py works as is. + */ +export function parseAbroadAddressesCsv( + input: string | Buffer, +): AbroadAddressInput[] { + let records: Record[] + try { + records = parse(input, { + columns: (header: string[]) => header.map(normalizeHeader), + skip_empty_lines: true, + relax_column_count: true, + trim: true, + bom: true, + }) + } catch (error) { + throw new AbroadAddressesInputError([`not a CSV file: ${error.message}`]) + } + const first = records[0] ?? {} + const idHeader = ID_HEADERS.find((h) => h in first) + const addressHeader = ADDRESS_HEADERS.find((h) => h in first) + if (records.length > 0 && (!idHeader || !addressHeader)) { + throw new AbroadAddressesInputError([ + `the header needs a section id column (${ID_HEADERS.join( + ', ', + )}) and an address column (${ADDRESS_HEADERS.join(', ')})`, + ]) + } + return validateAbroadAddresses( + records.map((r) => ({ sectionId: r[idHeader], address: r[addressHeader] })), + ) +} + +/** JSON body: `{ "sections": [{ "id": "320902332", "address": "…" }] }` or the bare array. */ +export function parseAbroadAddressesJson(body: unknown): AbroadAddressInput[] { + const list = Array.isArray(body) + ? body + : (body as { sections?: unknown })?.sections + if (!Array.isArray(list)) { + throw new AbroadAddressesInputError([ + 'expected a CSV "file" upload or a JSON body {"sections": [{"id", "address"}]}', + ]) + } + return validateAbroadAddresses( + list.map((item: Record) => ({ + sectionId: item?.id ?? item?.section_id ?? item?.sectionId, + address: item?.address, + })), + ) +} diff --git a/src/sections/abroad-addresses/abroad-addresses.service.ts b/src/sections/abroad-addresses/abroad-addresses.service.ts new file mode 100644 index 00000000..5d265735 --- /dev/null +++ b/src/sections/abroad-addresses/abroad-addresses.service.ts @@ -0,0 +1,28 @@ +import { Injectable } from '@nestjs/common' +import { InjectDataSource } from '@nestjs/typeorm' +import { DataSource } from 'typeorm' +import { AbroadAddressInput } from './abroad-addresses.parser' +import { + AbroadAddressesReport, + updateAbroadAddresses, +} from './abroad-addresses.updater' +import { + AbroadSectionListItem, + listAbroadSections, +} from './abroad-sections.list' + +@Injectable() +export class AbroadAddressesService { + constructor(@InjectDataSource() private readonly dataSource: DataSource) {} + + list(): Promise { + return listAbroadSections(this.dataSource) + } + + updateAddresses( + input: AbroadAddressInput[], + options: { dryRun?: boolean } = {}, + ): Promise { + return updateAbroadAddresses(this.dataSource, input, options) + } +} diff --git a/src/sections/abroad-addresses/abroad-addresses.updater.spec.ts b/src/sections/abroad-addresses/abroad-addresses.updater.spec.ts new file mode 100644 index 00000000..c791d475 --- /dev/null +++ b/src/sections/abroad-addresses/abroad-addresses.updater.spec.ts @@ -0,0 +1,136 @@ +/** + * Integration test against a real PostgreSQL database with the full schema + * (all migrations). Skipped unless TEST_DATABASE_URL is set, e.g.: + * + * docker run -d --rm --name ti-broish-test-pg -e POSTGRES_PASSWORD=test \ + * -e POSTGRES_DB=ti_broish_test -p 55432:5432 postgres:13-alpine + * TEST_DATABASE_URL=postgres://postgres:test@localhost:55432/ti_broish_test \ + * npx jest src/sections/abroad-addresses + * + * The database is wiped. Its name must contain "test". + */ +import { DataSource } from 'typeorm' +import { listAbroadSections } from './abroad-sections.list' +import { updateAbroadAddresses } from './abroad-addresses.updater' + +const url = process.env.TEST_DATABASE_URL +const describeDb = url ? describe : describe.skip + +describeDb('abroad addresses (PostgreSQL)', () => { + let ds: DataSource + + const sections = async () => + ( + await ds.query( + 'select trim(id) as id, place, address, voters_count from sections order by id', + ) + ).map((r) => ({ ...r })) + + beforeAll(async () => { + if (!/test/i.test(new URL(url).pathname)) { + throw new Error('TEST_DATABASE_URL must point at a database named *test*') + } + ds = new DataSource({ + type: 'postgres', + url, + migrations: [__dirname + '/../../migrations/*.ts'], + }) + await ds.initialize() + await ds.query('drop schema public cascade; create schema public;') + await ds.runMigrations() + }, 120000) + + afterAll(async () => { + await ds?.destroy() + }) + + beforeEach(async () => { + await ds.query(` + truncate protocols, violations, checkins, sections, city_regions_towns, + city_regions, towns, election_regions_municipalities, municipalities, + election_regions, countries restart identity cascade; + + insert into countries (id, code, name, is_abroad) values + (1, '000', 'България', false), + (2, '001', 'Австралия', true), + (3, '090', 'Норвегия', true); + insert into election_regions (id, code, name, is_abroad) values + (1, '11', 'Ловеч', false), + (2, '32', 'Извън страната', true); + insert into municipalities (id, code, name) values (1, '18', 'Ловеч'); + insert into election_regions_municipalities values (1, 1); + insert into towns (id, code, name, country_id, municipality_id) values + (1, 43952, 'гр. Ловеч', 1, 1), + (2, 100001, 'Канбера', 2, null), + (3, 100320, 'Тромсьо', 3, null); + insert into sections (id, election_region_id, town_id, city_region_id, code, place, voters_count, + is_machine, is_mobile, is_ship, risk_level, population, address) values + ('111800001', 1, 1, null, '001', 'ул Търговска № 1', 600, true, false, false, 'low', 0, null), + ('320010003', 2, 2, null, '003', 'Посолство', 0, false, false, false, 'low', 0, 'Moonah Pl 33'), + ('320902332', 2, 3, null, '332', 'Тромсьо', 0, false, false, false, 'low', 0, 'Stakkevollvegen 1'); + `) + }) + + it('updates only the addresses of existing abroad sections', async () => { + const before = await sections() + const report = await updateAbroadAddresses(ds, [ + { sectionId: '320902332', address: 'Kvaløyvegen 332, 9016 Tromsø' }, + { sectionId: '320010003', address: 'Moonah Pl 33' }, + { sectionId: '320999999', address: 'Nowhere 1' }, + ]) + expect(report).toEqual({ + dryRun: false, + received: 3, + updated: [ + { + id: '320902332', + from: 'Stakkevollvegen 1', + to: 'Kvaløyvegen 332, 9016 Tromsø', + }, + ], + unchanged: 1, + unknown: ['320999999'], + }) + expect(await sections()).toEqual( + before.map((s) => + s.id === '320902332' + ? { ...s, address: 'Kvaløyvegen 332, 9016 Tromsø' } + : s, + ), + ) + }) + + it('writes nothing on a dry run', async () => { + const before = await sections() + const report = await updateAbroadAddresses( + ds, + [{ sectionId: '320902332', address: 'Kvaløyvegen 332, 9016 Tromsø' }], + { dryRun: true }, + ) + expect(report.dryRun).toBe(true) + expect(report.updated).toHaveLength(1) + expect(await sections()).toEqual(before) + }) + + it('lists abroad sections with town, country and address', async () => { + await ds.query(`update sections set address = null where id = '320010003'`) + expect(await listAbroadSections(ds)).toEqual([ + { + id: '320010003', + code: '003', + place: 'Посолство', + address: null, + town: { id: 100001, name: 'Канбера' }, + country: { code: '001', name: 'Австралия' }, + }, + { + id: '320902332', + code: '332', + place: 'Тромсьо', + address: 'Stakkevollvegen 1', + town: { id: 100320, name: 'Тромсьо' }, + country: { code: '090', name: 'Норвегия' }, + }, + ]) + }) +}) diff --git a/src/sections/abroad-addresses/abroad-addresses.updater.ts b/src/sections/abroad-addresses/abroad-addresses.updater.ts new file mode 100644 index 00000000..53b7921e --- /dev/null +++ b/src/sections/abroad-addresses/abroad-addresses.updater.ts @@ -0,0 +1,113 @@ +import { DataSource, EntityManager } from 'typeorm' +import { AbroadAddressInput } from './abroad-addresses.parser' + +export interface AbroadAddressChange { + id: string + from: string | null + to: string +} + +export interface AbroadAddressesReport { + dryRun: boolean + received: number + updated: AbroadAddressChange[] + unchanged: number + /** Ids that are not abroad sections in the database. Never created. */ + unknown: string[] +} + +const LOCK = 'ti-broish:update-abroad-addresses' + +/** + * Sets the address of existing abroad (32*) sections. Only the address + * column is written: no section is created, deleted or otherwise changed, + * and ids that are not in the database are reported back as unknown. + */ +export async function applyAbroadAddresses( + manager: EntityManager, + input: AbroadAddressInput[], + options: { dryRun?: boolean } = {}, +): Promise { + await manager.query('select pg_advisory_xact_lock(hashtext($1))', [LOCK]) + const ids = input.map((row) => row.sectionId) + const rows: Array<{ id: string; address: string | null }> = + await manager.query( + `select trim(id) as id, address from sections + where id like '32%' and trim(id) = any($1::text[])`, + [ids], + ) + const current = new Map(rows.map((row) => [row.id, row.address])) + + const updated: AbroadAddressChange[] = [] + const unknown: string[] = [] + let unchanged = 0 + for (const { sectionId, address } of input) { + if (!current.has(sectionId)) { + unknown.push(sectionId) + } else if (current.get(sectionId) === address) { + unchanged++ + } else { + updated.push({ id: sectionId, from: current.get(sectionId), to: address }) + } + } + + if (updated.length > 0) { + await manager.query( + `update sections as s set address = v.address + from unnest($1::text[], $2::text[]) as v(id, address) + where trim(s.id) = v.id and s.id like '32%'`, + [updated.map((row) => row.id), updated.map((row) => row.to)], + ) + } + + return { + dryRun: !!options.dryRun, + received: input.length, + updated, + unchanged, + unknown, + } +} + +/** applyAbroadAddresses in its own transaction, rolled back on dryRun. */ +export async function updateAbroadAddresses( + dataSource: DataSource, + input: AbroadAddressInput[], + options: { dryRun?: boolean } = {}, +): Promise { + const queryRunner = dataSource.createQueryRunner() + await queryRunner.connect() + await queryRunner.startTransaction() + try { + const report = await applyAbroadAddresses( + queryRunner.manager, + input, + options, + ) + if (options.dryRun) { + await queryRunner.rollbackTransaction() + } else { + await queryRunner.commitTransaction() + } + return report + } catch (error) { + await queryRunner.rollbackTransaction() + throw error + } finally { + await queryRunner.release() + } +} + +export function summarizeAbroadAddresses( + report: AbroadAddressesReport, +): string { + return [ + report.dryRun ? 'DRY RUN (nothing written)' : 'Applied', + `received ${report.received}`, + `updated ${report.updated.length}`, + `unchanged ${report.unchanged}`, + `unknown ${report.unknown.length}${ + report.unknown.length ? ` (${report.unknown.join(', ')})` : '' + }`, + ].join(', ') +} diff --git a/src/sections/abroad-addresses/abroad-sections.list.ts b/src/sections/abroad-addresses/abroad-sections.list.ts new file mode 100644 index 00000000..18b3384a --- /dev/null +++ b/src/sections/abroad-addresses/abroad-sections.list.ts @@ -0,0 +1,46 @@ +import { DataSource } from 'typeorm' +import { sectionAddress } from '../section-address' + +export interface AbroadSectionListItem { + id: string + code: string + /** Committee name: the city, „Посолство“, „Баркинг 2“. */ + place: string + /** Street address from the МВнР table, null until loaded. */ + address: string | null + town: { id: number; name: string } + country: { code: string; name: string } +} + +/** Every abroad (32*) section with its town, country and address, ordered by id. */ +export async function listAbroadSections( + dataSource: DataSource, +): Promise { + const rows: Array<{ + id: string + code: string + place: string + address: string | null + town_code: number + town_name: string + country_code: string + country_name: string + }> = await dataSource.query( + `select trim(s.id) as id, trim(s.code) as code, s.place, s.address, + t.code as town_code, t.name as town_name, + c.code as country_code, c.name as country_name + from sections s + join towns t on t.id = s.town_id + join countries c on c.id = t.country_id + where s.id like '32%' + order by s.id`, + ) + return rows.map((row) => ({ + id: row.id, + code: row.code, + place: row.place, + address: sectionAddress(row), + town: { id: Number(row.town_code), name: row.town_name }, + country: { code: row.country_code, name: row.country_name }, + })) +} diff --git a/src/sections/abroad-addresses/index.ts b/src/sections/abroad-addresses/index.ts new file mode 100644 index 00000000..81101136 --- /dev/null +++ b/src/sections/abroad-addresses/index.ts @@ -0,0 +1,4 @@ +export * from './abroad-addresses.parser' +export * from './abroad-addresses.updater' +export * from './abroad-addresses.service' +export * from './abroad-sections.list' diff --git a/src/sections/api/abroad-addresses.controller.spec.ts b/src/sections/api/abroad-addresses.controller.spec.ts new file mode 100644 index 00000000..964e65d6 --- /dev/null +++ b/src/sections/api/abroad-addresses.controller.spec.ts @@ -0,0 +1,144 @@ +import { INestApplication } from '@nestjs/common' +import { Test } from '@nestjs/testing' +import * as request from 'supertest' +import { PoliciesGuard } from 'src/casl/policies.guard' +import { CaslAbilityFactory } from 'src/casl/casl-ability.factory' +import { ConfigService } from '@nestjs/config' +import { User } from 'src/users/entities' +import { Role } from 'src/casl/role.enum' +import { AbroadAddressesService } from '../abroad-addresses' +import { AbroadAddressesController } from './abroad-addresses.controller' + +const CSV = + 'section_id,country,place,address\n' + + '320902332,Норвегия,Тромсьо,"Kvaløyvegen 332, 9016 Tromsø"\n' + + '320010003,Австралия,Канбера,"Moonah Pl 33, Yarralumla"\n' + +describe('/sections/abroad', () => { + let app: INestApplication + let currentUser: User | undefined + const list = jest.fn() + const updateAddresses = jest.fn() + + beforeAll(async () => { + const moduleRef = await Test.createTestingModule({ + controllers: [AbroadAddressesController], + providers: [ + { + provide: AbroadAddressesService, + useValue: { list, updateAddresses }, + }, + PoliciesGuard, + CaslAbilityFactory, + { provide: ConfigService, useValue: { get: () => undefined } }, + ], + }).compile() + app = moduleRef.createNestApplication() + // Stand-in for the global Firebase guard: attach the current user + app.use((req, _res, next) => { + req.user = currentUser + next() + }) + await app.init() + }) + + afterAll(async () => { + await app.close() + }) + + const userWithRoles = (roles: Role[]): User => { + const user = new User() + user.roles = roles + return user + } + + beforeEach(() => { + list.mockReset() + updateAddresses.mockReset() + updateAddresses.mockResolvedValue({ dryRun: false, received: 2 }) + currentUser = userWithRoles([Role.User, Role.Admin]) + }) + + describe('GET', () => { + it('lists the abroad sections without a user', async () => { + currentUser = undefined + const item = { + id: '320902332', + code: '332', + place: 'Тромсьо', + address: 'Kvaløyvegen 332, 9016 Tromsø', + town: { id: 100320, name: 'Тромсьо' }, + country: { code: '090', name: 'Норвегия' }, + } + list.mockResolvedValue([item]) + const res = await request(app.getHttpServer()) + .get('/sections/abroad') + .expect(200) + expect(res.body).toEqual([item]) + expect(res.headers['cache-control']).toBe('public, max-age=300') + }) + }) + + describe('PATCH addresses', () => { + it('accepts a CSV upload from an admin', async () => { + const res = await request(app.getHttpServer()) + .patch('/sections/abroad/addresses') + .attach('file', Buffer.from(CSV), 'abroad_addresses.csv') + .expect(200) + expect(res.body).toEqual({ dryRun: false, received: 2 }) + expect(updateAddresses).toHaveBeenCalledWith( + [ + { sectionId: '320902332', address: 'Kvaløyvegen 332, 9016 Tromsø' }, + { sectionId: '320010003', address: 'Moonah Pl 33, Yarralumla' }, + ], + { dryRun: false }, + ) + }) + + it('accepts JSON and dryRun=true', async () => { + await request(app.getHttpServer()) + .patch('/sections/abroad/addresses?dryRun=true') + .send({ sections: [{ id: '320902332', address: 'Kvaløyvegen 332' }] }) + .expect(200) + expect(updateAddresses).toHaveBeenCalledWith( + [{ sectionId: '320902332', address: 'Kvaløyvegen 332' }], + { dryRun: true }, + ) + }) + + it('rejects domestic ids and missing addresses with 400', async () => { + const res = await request(app.getHttpServer()) + .patch('/sections/abroad/addresses') + .send({ + sections: [ + { id: '234615001', address: 'ул. Шипка 1' }, + { id: '320902332', address: ' ' }, + ], + }) + .expect(400) + expect(res.body.errors).toHaveLength(2) + expect(updateAddresses).not.toHaveBeenCalled() + }) + + it('rejects a bad dryRun', async () => { + await request(app.getHttpServer()) + .patch('/sections/abroad/addresses?dryRun=maybe') + .send({ sections: [{ id: '320902332', address: 'x' }] }) + .expect(400) + }) + + it('rejects non-admins', async () => { + currentUser = userWithRoles([Role.User, Role.Validator]) + await request(app.getHttpServer()) + .patch('/sections/abroad/addresses') + .send({ sections: [{ id: '320902332', address: 'x' }] }) + .expect(403) + currentUser = undefined + await request(app.getHttpServer()) + .patch('/sections/abroad/addresses') + .send({ sections: [{ id: '320902332', address: 'x' }] }) + .expect((res) => expect([401, 403]).toContain(res.status)) + expect(updateAddresses).not.toHaveBeenCalled() + }) + }) +}) diff --git a/src/sections/api/abroad-addresses.controller.ts b/src/sections/api/abroad-addresses.controller.ts new file mode 100644 index 00000000..13026c25 --- /dev/null +++ b/src/sections/api/abroad-addresses.controller.ts @@ -0,0 +1,114 @@ +import { + BadRequestException, + Body, + Controller, + Get, + Header, + HttpCode, + Patch, + Query, + UploadedFile, + UseGuards, + UseInterceptors, +} from '@nestjs/common' +import { FileInterceptor } from '@nestjs/platform-express' +import { ApiBody, ApiConsumes, ApiQuery, ApiResponse } from '@nestjs/swagger' +import { Express } from 'express' +import 'multer' +import { Public } from 'src/auth/decorators' +import { Action } from 'src/casl/action.enum' +import { AppAbility } from 'src/casl/casl-ability.factory' +import { CheckPolicies } from 'src/casl/check-policies.decorator' +import { PoliciesGuard } from 'src/casl/policies.guard' +import { ApiFirebaseAuth } from '../../auth/decorators/ApiFirebaseAuth.decorator' +import { + AbroadAddressesInputError, + AbroadAddressesReport, + AbroadAddressesService, + AbroadAddressInput, + AbroadSectionListItem, + parseAbroadAddressesCsv, + parseAbroadAddressesJson, +} from '../abroad-addresses' +import { Section } from '../entities/section.entity' + +const parseDryRun = (value: string | undefined): boolean => { + if (value === undefined || value === '') { + return false + } + const v = value.trim().toLowerCase() + if (['1', 'true', 'yes'].includes(v)) { + return true + } + if (['0', 'false', 'no'].includes(v)) { + return false + } + throw new BadRequestException('dryRun must be true or false') +} + +/** + * Abroad sections (election region 32): a public list with their addresses, + * and an admin-only update of the addresses. Registered before + * SectionsController so /sections/abroad is not read as /sections/:section. + */ +@Controller('sections/abroad') +@ApiFirebaseAuth() +export class AbroadAddressesController { + constructor(private readonly service: AbroadAddressesService) {} + + @Get() + @HttpCode(200) + @Public() + @UseGuards(PoliciesGuard) + @CheckPolicies((ability: AppAbility) => ability.can(Action.Read, Section)) + @Header('Cache-Control', 'public, max-age=300') + @ApiResponse({ + status: 200, + description: + 'Every abroad section: id, code, place (committee name), address (null until loaded), town and country', + }) + list(): Promise { + return this.service.list() + } + + @Patch('addresses') + @HttpCode(200) + @UseGuards(PoliciesGuard) + @CheckPolicies((ability: AppAbility) => ability.can(Action.Manage, Section)) + @UseInterceptors(FileInterceptor('file')) + @ApiConsumes('multipart/form-data', 'application/json') + @ApiBody({ + description: + 'A multipart CSV upload in field "file" with a section_id (or id, № на секция) and an address (Адрес) column, or JSON {"sections": [{"id": "320902332", "address": "…"}]}. Only the address of existing abroad sections is written; unknown ids are reported, never created.', + }) + @ApiQuery({ + name: 'dryRun', + required: false, + type: Boolean, + description: 'Apply in a transaction, report, then roll back', + }) + @ApiResponse({ status: 200, description: 'Report of what changed' }) + @ApiResponse({ status: 400, description: 'Invalid input; nothing changed' }) + async updateAddresses( + @UploadedFile() file: Express.Multer.File | undefined, + @Body() body: unknown, + @Query('dryRun') dryRunParam?: string, + ): Promise { + const dryRun = parseDryRun(dryRunParam) + let input: AbroadAddressInput[] + try { + input = file + ? parseAbroadAddressesCsv(file.buffer) + : parseAbroadAddressesJson(body) + } catch (e) { + if (e instanceof AbroadAddressesInputError) { + throw new BadRequestException({ + message: e.message, + errors: e.problems, + }) + } + throw e + } + return this.service.updateAddresses(input, { dryRun }) + } +} diff --git a/src/sections/sections.module.ts b/src/sections/sections.module.ts index ad92fe4e..17d7ba81 100644 --- a/src/sections/sections.module.ts +++ b/src/sections/sections.module.ts @@ -25,6 +25,8 @@ import { IsTownExistsConstraint } from './api/town-exists.constraint' import { CaslModule } from 'src/casl/casl.module' import { MunicipalitiesRepository } from './entities/municipalities.repository' import { CityRegionsRepository } from './entities/cityRegions.repository' +import { AbroadAddressesController } from './api/abroad-addresses.controller' +import { AbroadAddressesService } from './abroad-addresses' @Module({ imports: [ @@ -47,6 +49,7 @@ import { CityRegionsRepository } from './entities/cityRegions.repository' IsSectionExistsConstraint, IsTownExistsConstraint, CityRegionsRepository, + AbroadAddressesService, ], exports: [ SectionsRepository, @@ -57,6 +60,8 @@ import { CityRegionsRepository } from './entities/cityRegions.repository' TownsRepository, ], controllers: [ + // Before SectionsController, whose GET :section would take /sections/abroad. + AbroadAddressesController, SectionsController, CountriesController, ElectionRegionsController,