From 20a18d04ec971a8253fa2446864ef9808581e57b Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Mon, 4 May 2026 14:43:15 -0400 Subject: [PATCH 01/59] start gha ci --- .github/workflows/ci.yml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..98512d1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,31 @@ +name: Tests + +on: + push: + pull_request: + +jobs: + tests: + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11.5" + + - name: Install test dependencies + run: | + python -m pip install --upgrade pip + python -m pip install pytest pyomo==6.8.0 idaes-pse==2.5.0 gridx-egret==0.5.5 + python -m pip install . + + - name: Run tests + run: | + python -m pytest \ + var_elim/algorithms/tests \ + var_elim/heuristics/tests \ + var_elim/mip_formulations/tests From b92166c60b2490e31dd173036e4d2fbbffda21f7 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Mon, 4 May 2026 14:48:36 -0400 Subject: [PATCH 02/59] install dependencies from requirements.txt --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 98512d1..7ca3c65 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: - name: Install test dependencies run: | python -m pip install --upgrade pip - python -m pip install pytest pyomo==6.8.0 idaes-pse==2.5.0 gridx-egret==0.5.5 + python -m pip install -r requirements.txt python -m pip install . - name: Run tests From d7c59d27a77d076a9e9345809a246063a3ecf439 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Mon, 4 May 2026 14:52:36 -0400 Subject: [PATCH 03/59] install extensions --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7ca3c65..1c6e6bd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,9 @@ jobs: - name: Install test dependencies run: | python -m pip install --upgrade pip - python -m pip install -r requirements.txt + python -m pip install pytest pyomo==6.8.0 idaes-pse==2.5.0 gridx-egret=0.5.5 numpy + pyomo build-extensions + idaes get-extensions python -m pip install . - name: Run tests From 6e76a4672f402354abbeb1f88adb12cd8e6c1b4c Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Mon, 4 May 2026 14:53:40 -0400 Subject: [PATCH 04/59] typo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1c6e6bd..89f8779 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: - name: Install test dependencies run: | python -m pip install --upgrade pip - python -m pip install pytest pyomo==6.8.0 idaes-pse==2.5.0 gridx-egret=0.5.5 numpy + python -m pip install pytest pyomo==6.8.0 idaes-pse==2.5.0 gridx-egret==0.5.5 numpy pyomo build-extensions idaes get-extensions python -m pip install . From 3441c7b94807544567e4bafcd692a46d11cd8d26 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Mon, 4 May 2026 15:38:25 -0400 Subject: [PATCH 05/59] update readme --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 5df8914..4dc30ee 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,10 @@ Non-PyPI dependencies are: - [`nmpc_examples`](https://github.com/robbybp/nmpc_examples) - [`pselib`](https://github.com/robbybp/pselib) +[!Note] +It should not be necessary for uses to manually install these dependencies +as they are already included in the `requirements.txt` file. + When the main branch of this repository requires a specific branch of some dependency (e.g. the Pyomo main branch, rather than the latest release), an issue should be opened. From 22abae2c1062aea1b9d2fbacf0f92865fdec094b Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 14:47:17 -0400 Subject: [PATCH 06/59] install pybind11 in CI --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 89f8779..63186f6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,9 +20,10 @@ jobs: - name: Install test dependencies run: | python -m pip install --upgrade pip - python -m pip install pytest pyomo==6.8.0 idaes-pse==2.5.0 gridx-egret==0.5.5 numpy + python -m pip install pybind11 setuptools pyomo build-extensions idaes get-extensions + pkg-config --libs --cflags ipopt python -m pip install . - name: Run tests From 9a471274e367bc5b2dc6e77bf11f9db12d6f4aaf Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 14:49:58 -0400 Subject: [PATCH 07/59] install pyomo and idaes in CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 63186f6..f7f01f5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,7 +20,7 @@ jobs: - name: Install test dependencies run: | python -m pip install --upgrade pip - python -m pip install pybind11 setuptools + python -m pip install pybind11 setuptools pyomo idaes-pse pyomo build-extensions idaes get-extensions pkg-config --libs --cflags ipopt From e004f6fe4a6d31161ea37cde38b3e77bcd442a17 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 15:21:22 -0400 Subject: [PATCH 08/59] download extensions before building --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f7f01f5..3c91013 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install pybind11 setuptools pyomo idaes-pse + pyomo download-extensions pyomo build-extensions idaes get-extensions pkg-config --libs --cflags ipopt From 181276ebfe1cecaa8a39645b6c93dfbac6241961 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 15:49:34 -0400 Subject: [PATCH 09/59] set PKG_CONFIG_PATH --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3c91013..c973af1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,8 @@ jobs: python -m pip install pybind11 setuptools pyomo idaes-pse pyomo download-extensions pyomo build-extensions - idaes get-extensions + idaes get-extensions --to .idaes + export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" pkg-config --libs --cflags ipopt python -m pip install . From e7fa94edd9ec5881575941c92bddf022d7d9ff17 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 15:54:28 -0400 Subject: [PATCH 10/59] debugging --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c973af1..028b1c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,7 @@ jobs: pyomo build-extensions idaes get-extensions --to .idaes export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" + ls $PKG_CONFIG_PATH pkg-config --libs --cflags ipopt python -m pip install . From 1bbd6e652def1f32e5bf29e69ae70550d565495f Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 15:58:20 -0400 Subject: [PATCH 11/59] set PKG_CONFIG_PATH? --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 028b1c7..e662362 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,6 +18,8 @@ jobs: python-version: "3.11.5" - name: Install test dependencies + env: + PKG_CONFIG_PATH: ${{ github.workspace }}/.idaes/lib/pkgconfig run: | python -m pip install --upgrade pip python -m pip install pybind11 setuptools pyomo idaes-pse From 6d29b77f5b4b29f63219329e110e33edefcdf788 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 16:01:52 -0400 Subject: [PATCH 12/59] bug --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e662362..3773dcf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,6 @@ jobs: pyomo build-extensions idaes get-extensions --to .idaes export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" - ls $PKG_CONFIG_PATH pkg-config --libs --cflags ipopt python -m pip install . From 6d82ddc902f1bb752d5956c0a5b45b034a78d380 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 16:08:36 -0400 Subject: [PATCH 13/59] debug --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3773dcf..7a3e34b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,8 +25,12 @@ jobs: python -m pip install pybind11 setuptools pyomo idaes-pse pyomo download-extensions pyomo build-extensions - idaes get-extensions --to .idaes + idaes get-extensions --to $GITHUB_WORKSPACE/.idaes export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" + echo "GH workspace = $GITHUB_WORKSPACE" + echo "CWD = $(pwd)" + ls $GITHUB_WORKSPACE/.idaes + ls $GITHUB_WORKSPACE/.idaes/lib/pkgconfig pkg-config --libs --cflags ipopt python -m pip install . From 7e05ff8e15a50d797964d9c98b0b53d87083d0a2 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 16:15:47 -0400 Subject: [PATCH 14/59] unpack idaes-lib tar.gz --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a3e34b..46901cb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,9 +26,10 @@ jobs: pyomo download-extensions pyomo build-extensions idaes get-extensions --to $GITHUB_WORKSPACE/.idaes + pushd $GITHUB_WORKSPACE/.idaes + tar -xf idaes-lib-ubuntu*-x86_64.tar.gz + popd export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" - echo "GH workspace = $GITHUB_WORKSPACE" - echo "CWD = $(pwd)" ls $GITHUB_WORKSPACE/.idaes ls $GITHUB_WORKSPACE/.idaes/lib/pkgconfig pkg-config --libs --cflags ipopt From 8aee0bc9801e7289892c3a3dd344e494c4664d49 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 16:18:07 -0400 Subject: [PATCH 15/59] only run on pull request --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46901cb..74cfaf1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,6 @@ name: Tests on: - push: pull_request: jobs: From 8bb00e798f190fefd1ddbee1fd8fe83022531a35 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 16:20:38 -0400 Subject: [PATCH 16/59] verbose tar --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 74cfaf1..983fc93 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: pyomo build-extensions idaes get-extensions --to $GITHUB_WORKSPACE/.idaes pushd $GITHUB_WORKSPACE/.idaes - tar -xf idaes-lib-ubuntu*-x86_64.tar.gz + tar -xvf idaes-lib-ubuntu*-x86_64.tar.gz popd export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" ls $GITHUB_WORKSPACE/.idaes From ea9cfed1765db7b782be51ad706e7e46da26bca4 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 16:24:18 -0400 Subject: [PATCH 17/59] unpack idaes-solvers tar.gz ... --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 983fc93..ad35295 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: pyomo build-extensions idaes get-extensions --to $GITHUB_WORKSPACE/.idaes pushd $GITHUB_WORKSPACE/.idaes - tar -xvf idaes-lib-ubuntu*-x86_64.tar.gz + tar -xvf idaes-solvers-ubuntu*-x86_64.tar.gz popd export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" ls $GITHUB_WORKSPACE/.idaes From 030db2c123c76dbab5261371d5c84644e829859c Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 16:29:44 -0400 Subject: [PATCH 18/59] maybe idaes-local is the right tar.gz to unpack... --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad35295..8014ee6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: pyomo build-extensions idaes get-extensions --to $GITHUB_WORKSPACE/.idaes pushd $GITHUB_WORKSPACE/.idaes - tar -xvf idaes-solvers-ubuntu*-x86_64.tar.gz + tar -xvf idaes-local-ubuntu*-x86_64.tar.gz popd export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" ls $GITHUB_WORKSPACE/.idaes From abe34a8d090eafc440de7adf3c347c65764602b1 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 16:40:18 -0400 Subject: [PATCH 19/59] on workflow_dispatch --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8014ee6..c6d1d41 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,6 +2,7 @@ name: Tests on: pull_request: + workflow_dispatch: jobs: tests: From 4dc914e24c611e68fad68ced99e593d770f63506 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 16:45:23 -0400 Subject: [PATCH 20/59] add dependencies to pyproject.toml --- pyproject.toml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 1422c71..3dbda27 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,24 @@ authors = [ {name = "Robert Parker and Sakshi Naik"}, ] requires-python = ">=3.11" +dependencies = [ + "cyipopt>=1.5.0", + "gridx-egret>=0.5.5", + "idaes-pse>=2.5.0", + "matplotlib>=3.9.2", + "networkx>=3.3", + "nmpc_examples @ git+https://github.com/robbybp/nmpc_examples.git", + "numpy>=1.26.4", + "pandas>=2.2.2", + "pselib @ git+https://github.com/Robbybp/pselib.git", + "Pyomo>=6.8.0", + "scipy>=1.14.1", +] + +[project.optional-dependencies] +test = [ + "pytest>=8.3.3", +] [tool.setuptools.packages.find] include = ["var_elim", "var_elim.*"] From cf3e50a3949be5d319dd6d1c7d4feb78dfe2adc4 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 16:46:29 -0400 Subject: [PATCH 21/59] install test dependencies --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c6d1d41..53a178a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,7 +33,7 @@ jobs: ls $GITHUB_WORKSPACE/.idaes ls $GITHUB_WORKSPACE/.idaes/lib/pkgconfig pkg-config --libs --cflags ipopt - python -m pip install . + python -m pip install ".[test]" - name: Run tests run: | From 0a0403e1f1944ebac472174cd612284e3fecfdbc Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 16:48:07 -0400 Subject: [PATCH 22/59] more concise output --- .github/workflows/ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 53a178a..6325eb7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,12 +27,9 @@ jobs: pyomo build-extensions idaes get-extensions --to $GITHUB_WORKSPACE/.idaes pushd $GITHUB_WORKSPACE/.idaes - tar -xvf idaes-local-ubuntu*-x86_64.tar.gz + tar -xf idaes-local-ubuntu*-x86_64.tar.gz popd export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" - ls $GITHUB_WORKSPACE/.idaes - ls $GITHUB_WORKSPACE/.idaes/lib/pkgconfig - pkg-config --libs --cflags ipopt python -m pip install ".[test]" - name: Run tests From 7256909b7f71e7812449b34b24ed38ef9e0c3b3e Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 16:49:57 -0400 Subject: [PATCH 23/59] just set pkgconfig path with export --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6325eb7..3f53003 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,8 +18,6 @@ jobs: python-version: "3.11.5" - name: Install test dependencies - env: - PKG_CONFIG_PATH: ${{ github.workspace }}/.idaes/lib/pkgconfig run: | python -m pip install --upgrade pip python -m pip install pybind11 setuptools pyomo idaes-pse From 1f25b1a85d6d3cb90336eaa45ef60f10610dcc6d Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 16:51:13 -0400 Subject: [PATCH 24/59] pip list --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f53003..c1ac6ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,9 +29,11 @@ jobs: popd export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" python -m pip install ".[test]" + python -m pip list - name: Run tests run: | + python -m pip list python -m pytest \ var_elim/algorithms/tests \ var_elim/heuristics/tests \ From 851a4e317b8a805295bbcc01b37430dfea3ed26e Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 16:57:36 -0400 Subject: [PATCH 25/59] remove pip list --- .github/workflows/ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c1ac6ca..3f53003 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,11 +29,9 @@ jobs: popd export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" python -m pip install ".[test]" - python -m pip list - name: Run tests run: | - python -m pip list python -m pytest \ var_elim/algorithms/tests \ var_elim/heuristics/tests \ From e6d32c7d4153db3d55b5c1e64a7d5ef05d1e24d7 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 17:21:42 -0400 Subject: [PATCH 26/59] add .idaes/bin to path --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f53003..714de6e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,7 @@ jobs: pushd $GITHUB_WORKSPACE/.idaes tar -xf idaes-local-ubuntu*-x86_64.tar.gz popd + echo "$GITHUB_WORKSPACE/.idaes/bin" >> "$GITHUB_PATH" export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" python -m pip install ".[test]" From 6ecd3aae4f90fe61e7ec11bc1e9ba88882263bbe Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 17:28:58 -0400 Subject: [PATCH 27/59] tree idaes dir --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 714de6e..93b4cd9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,7 @@ jobs: pushd $GITHUB_WORKSPACE/.idaes tar -xf idaes-local-ubuntu*-x86_64.tar.gz popd + tree $GITHUB_WORKSPACE/.idaes echo "$GITHUB_WORKSPACE/.idaes/bin" >> "$GITHUB_PATH" export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" python -m pip install ".[test]" From 0010c5f5a12c3cedefac65163a44dd693a8b5f72 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 17:33:22 -0400 Subject: [PATCH 28/59] check if ipopt is on path --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 93b4cd9..662588a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,13 +27,14 @@ jobs: pushd $GITHUB_WORKSPACE/.idaes tar -xf idaes-local-ubuntu*-x86_64.tar.gz popd - tree $GITHUB_WORKSPACE/.idaes echo "$GITHUB_WORKSPACE/.idaes/bin" >> "$GITHUB_PATH" export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" python -m pip install ".[test]" - name: Run tests run: | + which ipopt + ipopt --version python -m pytest \ var_elim/algorithms/tests \ var_elim/heuristics/tests \ From f31c0b78dbc90dcebcdd1b267fb4177006107551 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 17:40:29 -0400 Subject: [PATCH 29/59] export PATH --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 662588a..0bd3a8d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,7 @@ jobs: - name: Run tests run: | + export PATH="$GITHUB_WORKSPACE/.idaes" which ipopt ipopt --version python -m pytest \ From d3cbd1684297c50bed309484aac7f4eb848fac9e Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 17:46:35 -0400 Subject: [PATCH 30/59] dont use which --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0bd3a8d..cfdb176 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,6 @@ jobs: - name: Run tests run: | export PATH="$GITHUB_WORKSPACE/.idaes" - which ipopt ipopt --version python -m pytest \ var_elim/algorithms/tests \ From 754b363f05f39441e5c57545ce32bdac9915ab24 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 17:54:00 -0400 Subject: [PATCH 31/59] set LD_LIBRARY_PATH --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cfdb176..8ee5e47 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,7 @@ jobs: pushd $GITHUB_WORKSPACE/.idaes tar -xf idaes-local-ubuntu*-x86_64.tar.gz popd + tree $GITHUB_WORKSPACE/.idaes echo "$GITHUB_WORKSPACE/.idaes/bin" >> "$GITHUB_PATH" export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" python -m pip install ".[test]" @@ -34,6 +35,8 @@ jobs: - name: Run tests run: | export PATH="$GITHUB_WORKSPACE/.idaes" + export LD_LIBRARY_PATH="$GITHUB_WORKSPACE/.idaes" + export DYLD_LIBRARY_PATH="$GITHUB_WORKSPACE/.idaes" ipopt --version python -m pytest \ var_elim/algorithms/tests \ From 3613ea0e7e37db5f9197343f97dc07b5f47b4df9 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 18:06:50 -0400 Subject: [PATCH 32/59] install libraries, e.g., lapack --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ee5e47..7482a86 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,6 +19,7 @@ jobs: - name: Install test dependencies run: | + apt-get -y install libgfortran5 libgomp1 liblapack3 libblas3 libc6 python -m pip install --upgrade pip python -m pip install pybind11 setuptools pyomo idaes-pse pyomo download-extensions From ecee872a3e7e2e24ff0e326ea6c75422a4988a93 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 18:22:34 -0400 Subject: [PATCH 33/59] sudo for installing system packages --- .github/workflows/ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7482a86..5389047 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,9 +17,13 @@ jobs: with: python-version: "3.11.5" + - name: Install system dependencies + run: | + sudo apt-get update + sudo apt-get install -y liblapack3 libblas3 gfortran + - name: Install test dependencies run: | - apt-get -y install libgfortran5 libgomp1 liblapack3 libblas3 libc6 python -m pip install --upgrade pip python -m pip install pybind11 setuptools pyomo idaes-pse pyomo download-extensions @@ -36,8 +40,6 @@ jobs: - name: Run tests run: | export PATH="$GITHUB_WORKSPACE/.idaes" - export LD_LIBRARY_PATH="$GITHUB_WORKSPACE/.idaes" - export DYLD_LIBRARY_PATH="$GITHUB_WORKSPACE/.idaes" ipopt --version python -m pytest \ var_elim/algorithms/tests \ From fb57d8d011c346c85fa392aa8d1f085a15c17293 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 18:26:53 -0400 Subject: [PATCH 34/59] append to path, dont replace --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5389047..eb10508 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -39,7 +39,7 @@ jobs: - name: Run tests run: | - export PATH="$GITHUB_WORKSPACE/.idaes" + export PATH="$PATH:$GITHUB_WORKSPACE/.idaes" ipopt --version python -m pytest \ var_elim/algorithms/tests \ From 8f3ddf5da9c925c9766c973fbdcea7cd9739cf6c Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 23:02:36 -0400 Subject: [PATCH 35/59] start python script to reproduce results --- reproduce.py | 194 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 reproduce.py diff --git a/reproduce.py b/reproduce.py new file mode 100644 index 0000000..9df7352 --- /dev/null +++ b/reproduce.py @@ -0,0 +1,194 @@ +# ___________________________________________________________________________ +# +# Variable Elimination: Research code for variable elimination in NLPs +# +# Copyright (c) 2023. Triad National Security, LLC. All rights reserved. +# +# This program was produced under U.S. Government contract 89233218CNA000001 +# for Los Alamos National Laboratory (LANL), which is operated by Triad +# National Security, LLC for the U.S. Department of Energy/National Nuclear +# Security Administration. All rights in the program are reserved by Triad +# National Security, LLC, and the U.S. Department of Energy/National Nuclear +# Security Administration. The Government is granted for itself and others +# acting on its behalf a nonexclusive, paid-up, irrevocable worldwide license +# in this material to reproduce, prepare derivative works, distribute copies +# to the public, perform publicly and display publicly, and to permit others +# to do so. +# +# This software is distributed under the 3-clause BSD license. +# ___________________________________________________________________________ + +import argparse +import datetime +import os +import shlex +import subprocess + + +def default_run_dir(): + date = datetime.date.today().strftime("%Y%m%d") + return os.path.join("runs", date) + + +def script_path(script_name): + return os.path.join("var_elim", "scripts", script_name) + + +def require_repo_root(): + expected = [ + "pyproject.toml", + os.path.join("var_elim", "scripts"), + ] + missing = [path for path in expected if not os.path.exists(path)] + if missing: + raise RuntimeError( + "reproduce.py must be run from the repository root. " + f"Missing expected path: {missing[0]}" + ) + + +def run_command(cmd, dry_run=False): + print("+ " + shlex.join(cmd)) + if not dry_run: + subprocess.run(cmd, check=True) + + +def make_dir(dirname, dry_run=False): + print("+ mkdir -p " + shlex.quote(dirname)) + if not dry_run: + os.makedirs(dirname, exist_ok=True) + + +def add_common_args(parser): + parser.add_argument( + "--run-dir", + default=None, + help="Run directory. Defaults to runs/YYYYMMDD.", + ) + parser.add_argument( + "--results-dir", + default=None, + help="Directory for CSV and table outputs. Defaults to RUN_DIR/results.", + ) + parser.add_argument( + "--image-dir", + default=None, + help="Directory for plot outputs. Defaults to RUN_DIR/images.", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print commands without running them.", + ) + + +def add_structure_args(parser): + add_common_args(parser) + parser.add_argument( + "--smoke", + action="store_true", + help="Run only the distillation structural workflow for a quicker check.", + ) + parser.add_argument( + "--skip-tables", + action="store_true", + help="Skip LaTeX table generation.", + ) + parser.add_argument( + "--skip-plots", + action="store_true", + help="Skip plot generation.", + ) + + +def resolve_output_dirs(args): + run_dir = args.run_dir if args.run_dir is not None else default_run_dir() + results_dir = ( + args.results_dir + if args.results_dir is not None + else os.path.join(run_dir, "results") + ) + image_dir = ( + args.image_dir + if args.image_dir is not None + else os.path.join(run_dir, "images") + ) + return run_dir, results_dir, image_dir + + +def run_structure(args): + _, results_dir, image_dir = resolve_output_dirs(args) + + make_dir(results_dir, dry_run=args.dry_run) + make_dir(image_dir, dry_run=args.dry_run) + + analyze_cmd = [ + "python", + script_path("analyze_structure.py"), + "--results-dir", + results_dir, + ] + if args.smoke: + analyze_cmd.extend(["--model", "distill"]) + + run_command(analyze_cmd, dry_run=args.dry_run) + + csv_name = "structure-distill.csv" if args.smoke else "structure.csv" + structure_csv = os.path.join(results_dir, csv_name) + + if not args.skip_tables: + table_cmd = [ + "python", + script_path("write_latex_table.py"), + structure_csv, + "--results-dir", + results_dir, + ] + run_command(table_cmd, dry_run=args.dry_run) + + matching_table_cmd = table_cmd + ["--which", "matching-bounds"] + run_command(matching_table_cmd, dry_run=args.dry_run) + + if not args.skip_plots: + plot_cmd = [ + "python", + script_path("plot_structure_bargraphs.py"), + structure_csv, + "--image-dir", + image_dir, + ] + run_command(plot_cmd, dry_run=args.dry_run) + + if not args.smoke: + sparsity_cmd = [ + "python", + script_path("plot_sparsity.py"), + "--model", + "mb-steady", + "--image-dir", + image_dir, + ] + run_command(sparsity_cmd, dry_run=args.dry_run) + + +def main(): + require_repo_root() + + parser = argparse.ArgumentParser( + description="Run end-to-end result reproduction workflows." + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + structure_parser = subparsers.add_parser( + "structure", + help="Run structural analysis, tables, and plots.", + ) + add_structure_args(structure_parser) + structure_parser.set_defaults(func=run_structure) + + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() From 4ca65a24cd7ac2d70c171a23d9d56b182841a343 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 23:16:26 -0400 Subject: [PATCH 36/59] add solvetime subcommand to reproduce.py --- reproduce.py | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/reproduce.py b/reproduce.py index 9df7352..95f0e5f 100644 --- a/reproduce.py +++ b/reproduce.py @@ -101,6 +101,30 @@ def add_structure_args(parser): ) +def add_solvetime_args(parser): + add_common_args(parser) + parser.add_argument( + "--smoke", + action="store_true", + help="Run only the distillation matching workflow for a quicker check.", + ) + parser.add_argument( + "--skip-tables", + action="store_true", + help="Skip LaTeX table generation.", + ) + parser.add_argument( + "--skip-plots", + action="store_true", + help="Skip plot generation.", + ) + parser.add_argument( + "--tee", + action="store_true", + help="Stream solver logs from the solve-time analysis.", + ) + + def resolve_output_dirs(args): run_dir = args.run_dir if args.run_dir is not None else default_run_dir() results_dir = ( @@ -171,6 +195,49 @@ def run_structure(args): run_command(sparsity_cmd, dry_run=args.dry_run) +def run_solvetime(args): + _, results_dir, image_dir = resolve_output_dirs(args) + + make_dir(results_dir, dry_run=args.dry_run) + make_dir(image_dir, dry_run=args.dry_run) + + analyze_cmd = [ + "python", + script_path("analyze_solvetime.py"), + "--results-dir", + results_dir, + ] + if args.smoke: + analyze_cmd.extend(["--model", "distill", "--method", "matching"]) + if args.tee: + analyze_cmd.append("--tee") + + run_command(analyze_cmd, dry_run=args.dry_run) + + csv_name = "solvetime-distill-matching.csv" if args.smoke else "solvetime.csv" + solvetime_csv = os.path.join(results_dir, csv_name) + + if not args.skip_tables: + table_cmd = [ + "python", + script_path("write_latex_table.py"), + solvetime_csv, + "--results-dir", + results_dir, + ] + run_command(table_cmd, dry_run=args.dry_run) + + if not args.skip_plots: + plot_cmd = [ + "python", + script_path("plot_timing_bargraphs.py"), + solvetime_csv, + "--image-dir", + image_dir, + ] + run_command(plot_cmd, dry_run=args.dry_run) + + def main(): require_repo_root() @@ -186,6 +253,13 @@ def main(): add_structure_args(structure_parser) structure_parser.set_defaults(func=run_structure) + solvetime_parser = subparsers.add_parser( + "solvetime", + help="Run solve-time analysis, table, and plot.", + ) + add_solvetime_args(solvetime_parser) + solvetime_parser.set_defaults(func=run_solvetime) + args = parser.parse_args() args.func(args) From 267b7d79d7e7569f3f9a4b84657364e4a60542ca Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 23:20:10 -0400 Subject: [PATCH 37/59] structural analysis smoketest in CI --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb10508..84b2edf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,3 +45,8 @@ jobs: var_elim/algorithms/tests \ var_elim/heuristics/tests \ var_elim/mip_formulations/tests + + - name: Run structural analysis + run: | + export PATH="$PATH:$GITHUB_WORKSPACE/.idaes" + python reproduce.py structure --smoke --run-dir=. From f4029986c42c82a7ad96ff495e015a8269cab44d Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 23:43:13 -0400 Subject: [PATCH 38/59] install tex --- .github/workflows/ci.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 84b2edf..df7d0d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,6 +22,15 @@ jobs: sudo apt-get update sudo apt-get install -y liblapack3 libblas3 gfortran + - name: Install latex + run: | + sudo apt-get install -y \ + texlive-latex-base \ + texlive-latex-recommended \ + texlive-fonts-recommended \ + dvipng \ + cm-super + - name: Install test dependencies run: | python -m pip install --upgrade pip From cca075bd9e3ee71705e3a65b71e60def13685815 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 23:53:28 -0400 Subject: [PATCH 39/59] try texlive-latex-extra --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df7d0d0..8d8d5a0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,7 @@ jobs: sudo apt-get install -y \ texlive-latex-base \ texlive-latex-recommended \ + texlien-latex-extra \ texlive-fonts-recommended \ dvipng \ cm-super From e6c46fb0180da06fae700d50cdffff92f9a24199 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 11 Jul 2026 23:55:22 -0400 Subject: [PATCH 40/59] typo --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d8d5a0..fcbe46d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,7 +27,7 @@ jobs: sudo apt-get install -y \ texlive-latex-base \ texlive-latex-recommended \ - texlien-latex-extra \ + texlive-latex-extra \ texlive-fonts-recommended \ dvipng \ cm-super From 71d99bed1a3fc2ff1f9c47e889164b3b4c7c52e4 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sun, 12 Jul 2026 00:12:06 -0400 Subject: [PATCH 41/59] solvetime smoketest in CI --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fcbe46d..1b2f572 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,3 +60,8 @@ jobs: run: | export PATH="$PATH:$GITHUB_WORKSPACE/.idaes" python reproduce.py structure --smoke --run-dir=. + + - name: Run solvetime analysis + run: | + export PATH="$PATH:$GITHUB_WORKSPACE/.idaes" + python reproduce.py solvetime --smoke --run-dir=. From 6a86b4e4ecfc27504df7aa41e4f8570254f9a99a Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sun, 12 Jul 2026 00:25:41 -0400 Subject: [PATCH 42/59] set LD_LIBRARY_PATH --- .github/workflows/ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1b2f572..b5c20fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,5 +63,7 @@ jobs: - name: Run solvetime analysis run: | + python -m pip list export PATH="$PATH:$GITHUB_WORKSPACE/.idaes" + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/.idaes" python reproduce.py solvetime --smoke --run-dir=. From 5a65710632f35fc0a21d348b092f4d881a91d452 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sun, 12 Jul 2026 00:37:38 -0400 Subject: [PATCH 43/59] maybe we need to set pkgcconfig path? --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b5c20fc..7d10723 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,5 +65,7 @@ jobs: run: | python -m pip list export PATH="$PATH:$GITHUB_WORKSPACE/.idaes" - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/.idaes" + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/.idaes/lib" + export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" + pkg-config --libs --cflags ipopt python reproduce.py solvetime --smoke --run-dir=. From f8e904a8d112b9ec9e3acb34efd6ff7a87d01b19 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sun, 12 Jul 2026 00:53:57 -0400 Subject: [PATCH 44/59] was it the library path or the pkgconfig path --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d10723..d9a167f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,6 +66,5 @@ jobs: python -m pip list export PATH="$PATH:$GITHUB_WORKSPACE/.idaes" export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/.idaes/lib" - export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" pkg-config --libs --cflags ipopt python reproduce.py solvetime --smoke --run-dir=. From e2d3749b74b17b14e12aa0a1cb3d39420e40e26a Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sun, 12 Jul 2026 01:09:06 -0400 Subject: [PATCH 45/59] pkgconfig path --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9a167f..5e17282 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,6 @@ jobs: run: | python -m pip list export PATH="$PATH:$GITHUB_WORKSPACE/.idaes" - export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/.idaes/lib" + export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" pkg-config --libs --cflags ipopt python reproduce.py solvetime --smoke --run-dir=. From 118a7d3a76198d5895f1aa12f5996789d2b03696 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sun, 12 Jul 2026 01:18:22 -0400 Subject: [PATCH 46/59] pkgconfig path and library path --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e17282..8a3bfca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,6 +65,7 @@ jobs: run: | python -m pip list export PATH="$PATH:$GITHUB_WORKSPACE/.idaes" + #export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/.idaes/lib" export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" pkg-config --libs --cflags ipopt python reproduce.py solvetime --smoke --run-dir=. From a06e9ec098b06f5f4596a00b0314625658342b59 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sun, 12 Jul 2026 01:24:41 -0400 Subject: [PATCH 47/59] uncomment library path --- reproduce.py | 125 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/reproduce.py b/reproduce.py index 95f0e5f..fdcae47 100644 --- a/reproduce.py +++ b/reproduce.py @@ -25,6 +25,10 @@ import subprocess +SWEEP_MODELS = ["distill", "mb-steady", "pipeline"] +SWEEP_METHODS = ["no-elim", "d1", "ecd2", "linear-d2", "d2", "greedy", "matching"] + + def default_run_dir(): date = datetime.date.today().strftime("%Y%m%d") return os.path.join("runs", date) @@ -125,6 +129,34 @@ def add_solvetime_args(parser): ) +def add_convergence_args(parser): + add_common_args(parser) + parser.add_argument( + "--nsamples", + type=int, + default=None, + help=( + "Number of samples per parameter. Defaults to 2 with --smoke, " + "otherwise 11." + ), + ) + parser.add_argument( + "--smoke", + action="store_true", + help="Run only the distillation matching sweep for a quicker check.", + ) + parser.add_argument( + "--skip-tables", + action="store_true", + help="Skip LaTeX table generation.", + ) + parser.add_argument( + "--skip-plots", + action="store_true", + help="Skip plot generation.", + ) + + def resolve_output_dirs(args): run_dir = args.run_dir if args.run_dir is not None else default_run_dir() results_dir = ( @@ -238,6 +270,92 @@ def run_solvetime(args): run_command(plot_cmd, dry_run=args.dry_run) +def run_convergence(args): + _, results_dir, image_dir = resolve_output_dirs(args) + sweep_dir = os.path.join(results_dir, "sweep") + nsamples = ( + args.nsamples + if args.nsamples is not None + else (2 if args.smoke else 11) + ) + + make_dir(results_dir, dry_run=args.dry_run) + make_dir(sweep_dir, dry_run=args.dry_run) + make_dir(image_dir, dry_run=args.dry_run) + + sweep_jobs = ( + [("distill", "matching")] + if args.smoke + else [(model, None) for model in SWEEP_MODELS] + ) + for model, method in sweep_jobs: + sweep_cmd = [ + "python", + script_path("run_param_sweep.py"), + "--results-dir", + sweep_dir, + "--model", + model, + "--nsamples", + str(nsamples), + ] + if method is not None: + sweep_cmd.extend(["--method", method]) + run_command(sweep_cmd, dry_run=args.dry_run) + + if args.smoke: + summary_cmd = [ + "python", + script_path("summarize_sweep_results.py"), + "--results-dir", + sweep_dir, + "--model", + "distill", + "--method", + "matching", + ] + else: + summary_cmd = [ + "python", + script_path("summarize_sweep_results.py"), + "--results-dir", + sweep_dir, + ] + run_command(summary_cmd, dry_run=args.dry_run) + + if not args.skip_tables and not args.smoke: + summary_csv = os.path.join(results_dir, "sweep-summary.csv") + table_cmd = [ + "python", + script_path("write_latex_table.py"), + summary_csv, + "--results-dir", + results_dir, + ] + run_command(table_cmd, dry_run=args.dry_run) + + if not args.skip_plots: + plot_jobs = ( + [("distill", "matching")] + if args.smoke + else [ + (model, method) + for model in SWEEP_MODELS + for method in SWEEP_METHODS + ] + ) + for model, method in plot_jobs: + sweep_csv = os.path.join(sweep_dir, f"{model}-{method}-sweep.csv") + plot_cmd = [ + "python", + script_path("plot_sweep_results.py"), + sweep_csv, + "--image-dir", + image_dir, + ] + run_command(plot_cmd, dry_run=args.dry_run) + + def main(): require_repo_root() @@ -260,6 +378,13 @@ def main(): add_solvetime_args(solvetime_parser) solvetime_parser.set_defaults(func=run_solvetime) + convergence_parser = subparsers.add_parser( + "convergence", + help="Run parameter sweeps, summaries, and convergence plots.", + ) + add_convergence_args(convergence_parser) + convergence_parser.set_defaults(func=run_convergence) + args = parser.parse_args() args.func(args) From fb72df4e3e6ad96eaa98b85a62adb21b98fd2f54 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sun, 12 Jul 2026 01:24:54 -0400 Subject: [PATCH 48/59] uncomment library path --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a3bfca..640a71e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,7 +65,9 @@ jobs: run: | python -m pip list export PATH="$PATH:$GITHUB_WORKSPACE/.idaes" - #export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/.idaes/lib" + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/.idaes/lib" export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" pkg-config --libs --cflags ipopt python reproduce.py solvetime --smoke --run-dir=. + + From 702bab90575440fff18c224f62f4d34f9335c642 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sun, 12 Jul 2026 01:26:52 -0400 Subject: [PATCH 49/59] convergence smoketest in CI --- .github/workflows/ci.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 640a71e..1e4bd84 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,11 +63,16 @@ jobs: - name: Run solvetime analysis run: | - python -m pip list export PATH="$PATH:$GITHUB_WORKSPACE/.idaes" export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/.idaes/lib" export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" pkg-config --libs --cflags ipopt python reproduce.py solvetime --smoke --run-dir=. - + - name: Run convergence analysis + run: | + export PATH="$PATH:$GITHUB_WORKSPACE/.idaes" + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$GITHUB_WORKSPACE/.idaes/lib" + export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" + pkg-config --libs --cflags ipopt + python reproduce.py convergence --smoke --run-dir=. From 38de6be09ca09c5ab0f583a896777985d0fdd6f0 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sun, 12 Jul 2026 12:28:14 -0400 Subject: [PATCH 50/59] test that expected files exist --- .github/workflows/ci.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1e4bd84..0dcb350 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,6 +60,11 @@ jobs: run: | export PATH="$PATH:$GITHUB_WORKSPACE/.idaes" python reproduce.py structure --smoke --run-dir=. + test -s results/structure-distill.csv + test -s results/structure-distill.txt + test -s results/structure-distill-matching.txt + test -s images/fraction-elim.pdf + test -s images/nnz-per-con.pdf - name: Run solvetime analysis run: | @@ -68,6 +73,9 @@ jobs: export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" pkg-config --libs --cflags ipopt python reproduce.py solvetime --smoke --run-dir=. + test -s results/solvetime-distill-matching.csv + test -s results/solvetime-distill-matching.txt + test -s images/fraction-solvetime.pdf - name: Run convergence analysis run: | @@ -76,3 +84,6 @@ jobs: export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" pkg-config --libs --cflags ipopt python reproduce.py convergence --smoke --run-dir=. + test -s results/sweep/distill-matching-sweep.csv + test -s results/sweep/distill-matching-sweep-summary.csv + test -s images/distill-matching-sweep-convergence.pdf From 379b00a4f9a0763e3c06b87cc6889f3366570aa1 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 18 Jul 2026 11:37:45 -0400 Subject: [PATCH 51/59] was appending to GITHUB_PATH necessary? --- .github/workflows/ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0dcb350..6767a2f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,7 +43,6 @@ jobs: tar -xf idaes-local-ubuntu*-x86_64.tar.gz popd tree $GITHUB_WORKSPACE/.idaes - echo "$GITHUB_WORKSPACE/.idaes/bin" >> "$GITHUB_PATH" export PKG_CONFIG_PATH="$GITHUB_WORKSPACE/.idaes/lib/pkgconfig:$PKG_CONFIG_PATH" python -m pip install ".[test]" From 3067ce2ff8cad61b014e206b595445b7813a1a7e Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 18 Jul 2026 11:49:29 -0400 Subject: [PATCH 52/59] move header.txt to copyright.txt --- COPYRIGHT.txt | 19 +++++++++++++++++++ header.txt | 19 ------------------- 2 files changed, 19 insertions(+), 19 deletions(-) create mode 100644 COPYRIGHT.txt delete mode 100644 header.txt diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt new file mode 100644 index 0000000..dcd9a55 --- /dev/null +++ b/COPYRIGHT.txt @@ -0,0 +1,19 @@ +___________________________________________________________________________ + +Variable Elimination: Research code for variable elimination in NLPs + +Copyright (c) 2023. Triad National Security, LLC. All rights reserved. + +This program was produced under U.S. Government contract 89233218CNA000001 +for Los Alamos National Laboratory (LANL), which is operated by Triad +National Security, LLC for the U.S. Department of Energy/National Nuclear +Security Administration. All rights in the program are reserved by Triad +National Security, LLC, and the U.S. Department of Energy/National Nuclear +Security Administration. The Government is granted for itself and others +acting on its behalf a nonexclusive, paid-up, irrevocable worldwide license +in this material to reproduce, prepare derivative works, distribute copies +to the public, perform publicly and display publicly, and to permit others +to do so. + +This software is distributed under the 3-clause BSD license. +___________________________________________________________________________ diff --git a/header.txt b/header.txt deleted file mode 100644 index 5b1c230..0000000 --- a/header.txt +++ /dev/null @@ -1,19 +0,0 @@ -# ___________________________________________________________________________ -# -# Variable Elimination: Research code for variable elimination in NLPs -# -# Copyright (c) 2023. Triad National Security, LLC. All rights reserved. -# -# This program was produced under U.S. Government contract 89233218CNA000001 -# for Los Alamos National Laboratory (LANL), which is operated by Triad -# National Security, LLC for the U.S. Department of Energy/National Nuclear -# Security Administration. All rights in the program are reserved by Triad -# National Security, LLC, and the U.S. Department of Energy/National Nuclear -# Security Administration. The Government is granted for itself and others -# acting on its behalf a nonexclusive, paid-up, irrevocable worldwide license -# in this material to reproduce, prepare derivative works, distribute copies -# to the public, perform publicly and display publicly, and to permit others -# to do so. -# -# This software is distributed under the 3-clause BSD license. -# ___________________________________________________________________________ From a2f4845597eb95b2b98ae72820282ea3f8cece28 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 18 Jul 2026 12:27:03 -0400 Subject: [PATCH 53/59] start updating readme --- README.md | 138 +++++++++++++++++++++++++++++------------------------- 1 file changed, 74 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index 0539b33..678935f 100644 --- a/README.md +++ b/README.md @@ -1,60 +1,36 @@ -# variable-elimination -Repository for collaborative exploratory work on variable elimination in NLPs - -# Copyright -The copyright header in `header.txt` must be added to every source (Python) -file in this repository. +Variable Elimination +-------------------- +This repository started as a place for collaborative exploratory work +on variable elimination in nonlinear optimization problems. It now primarily +serves to host the code necessary to reproduce the results in the paper +"Variable aggregation in nonlinear optimization problems": +```bibtex +@misc{naik2026aggregation, + title={Variable aggregation for nonlinear optimization problems}, + author={Sakshi Naik and Lorenz Biegler and Russell Bent and Robert Parker}, + year={2026}, + eprint={2502.13869}, + archivePrefix={arXiv}, + primaryClass={math.OC}, + url={https://arxiv.org/abs/2502.13869}, +} +``` # Installation This repository is structured as a small Python package to facilitate code -organization and testing. It can be installed with: -```bash -git clone https://github.com/Robbybp/variable-elimination.git -cd variable-elimination -pip install -r requirements.txt -pip install -e . -``` -Then functionality can be imported in Python: -```python -from var_elim.models.distillation.distill import create_instance -model = create_instance() -``` +organization and testing. To install, you must first install our non-Python +dependencies +([PyNumero](https://pyomo.readthedocs.io/en/stable/explanation/solvers/pynumero/index.html) +and [IPOPT](https://github.com/coin-or/ipopt)) +then install this repository as a Python package. -# Dependencies +## Installing non-Python dependencies -## Python dependencies +### PyNumero -This repository was developed and tested using Python 3.11.5. -See `requirements.txt` for a list of Python dependencies. For stability, we pin to -specific versions of our dependencies, e.g. Pyomo and IDAES. -Non-PyPI dependencies are: -- [`nmpc_examples`](https://github.com/robbybp/nmpc_examples) -- [`pselib`](https://github.com/robbybp/pselib) - -These can be installed with -```bash -pip install git+https://github.com/Robbybp/nmpc_examples.git -pip install git+https://github.com/Robbybp/pselib.git +We use PyNumero to interface Pyomo, CyIpopt, and the AMPL Solver Library (ASL). +PyNumero can be installed by the Pyomo Python package: ``` - -> [!NOTE] -> Users who only wish to reproduce our results should not need to install these -> dependencies manually as they are already included in the `requirements.txt` -> file. (I.e., they are installed by the `pip install -r requirements.txt` step -> above.) These are listed only for users who wish to use our code in their own -> projects with different versions of our dependencies. - -When the main branch of this repository requires a specific branch of some dependency -(e.g. the Pyomo main branch, rather than the latest release), an issue should be opened. - -## Non-Python dependencies -Results are generated with Ipopt 3.14.17, using the Pyomo-CyIpopt interface. This -interface depends on the following non-Python dependencies: - -1. `libpynumero_ASL`. This is a library that allows us to make calls to the AMPL -solver library (ASL) from Python. It can be installed on most systems with the -following commands: -```bash pip install pyomo pyomo build-extensions ``` @@ -75,25 +51,54 @@ INFO: The following extensions were built: As long as `pynumero` has status `OK`, this step was successful for the purpose of this repository. -2. [Ipopt 3.14](https://github.com/coin-or/ipopt), with linear solver MA27. +### IPOPT +We use the IPOPT solver (version 3.14.10 with linear solver MA27) +to compare the impact of our aggregation techniques. +IPOPT can be installed by following instructions on the github repository, +[](https://github.com/coin-or/ipopt). See the following instructions to compile an Ipopt-compatible HSL library (which contains MA27): https://github.com/coin-or-tools/ThirdParty-HSL. +> [!NOTE] +> Depending on your IPOPT install location, you may have to set some environment +> variables for `libipopt` to be discoverable by the CyIpopt Python package. +> Commonly, `PKG_CONFIG_PATH=INSTALL_DIR/lib/pkgconfig`, +> `LD_LIBRARY_PATH=INSTALL_DIR/lib`, and/or `DYLD_LIBRARY_PATH=INSTALL_DIR/lib` +> must be set. + +## Installing this package +Download this repository and navigate into it with: +``` +git clone https://github.com/Robbybp/variable-elimination.git +cd variable-elimination +``` +The following command, run from the root of this repository, will install this +package and the latest versions of all its Python dependencies: +``` +pip install -e .[test] +``` +(The `[test]` flag installs `pytest`, which is only necessary to run tests.) +Verify that the installation was successful by running `pytest` from the +repository root. +The functionality of this package can then be imported in Python, e.g.: +```python +from var_elim.models.distillation.distill import create_instance +model = create_instance() +``` + +> [!NOTE] +> The above command installs the *latest* version of dependencies. +> The exact versions of all dependencies used to produce the results +> in our paper are provided in `requirements.txt`. Install these +> dependencies with +> ``` +> pip install -r requirements.txt +> ``` +> before or after installing this package. + # Reproducing our results -This repository was used to generate results for the paper: -```bibtex -@misc{naik2025aggregation, - title={Variable aggregation for nonlinear optimization problems}, - author={Sakshi Naik and Lorenz Biegler and Russell Bent and Robert Parker}, - year={2025}, - eprint={2502.13869}, - archivePrefix={arXiv}, - primaryClass={math.OC}, - url={https://arxiv.org/abs/2502.13869}, -} -``` -This paper presents three kinds of results: +Our paper presents three kinds of results: 1. Structural results describe the size and shape of models before loading any variable values. 2. Numerical results, in our case, are solution objective values, solve times, @@ -272,3 +277,8 @@ nvar = length(igraph.variables) ncon = length(igraph.constraints) nnz = igraph.n_edges ``` + +# License +This code is copyrighted by Los Alamos National Laboratory and distributed under a +BSD 3-clause license. The license and copyright are provided in `LICENSE.md` and +`COPYRIGHT.txt`. From 8615c7b5fa2f4f81c07947044bc524da13d57350 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 18 Jul 2026 13:07:44 -0400 Subject: [PATCH 54/59] more readme --- README.md | 188 +++++++++++++++--------------------------------------- 1 file changed, 52 insertions(+), 136 deletions(-) diff --git a/README.md b/README.md index 678935f..9427ef7 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,8 @@ model = create_instance() > pip install -r requirements.txt > ``` > before or after installing this package. +> +> The results for our paper were generated with Python 3.11.5. # Reproducing our results @@ -106,149 +108,63 @@ and solve time breakdowns. 3. Convergence results are the success or failure of many instances when performing a sweep over model parameter values. -Scripts to produce these results are located in the `var_elim/scripts` subdirectory -of this repository. +Scripts to run analyses, summarize results, and generate plots and tables are +located in the `var_elim/scripts` subdirectory of this repository. You can view the command line interface for each script with: ```bash -python myscript.py --help -``` -In particular, the `--results-dir=RESULTS_DIR` option specifies the directory -where results are written, and the `--image-dir=IMAGE_DIR` option specifies the -directory where images are saved (for scripts that plot figures). -These default to `results` and `images` respectively, but it might be useful -to set them to custom values to avoid overwriting previous results -(produced with, e.g., a different Pyomo version). - -TL;DR: Reproduce the bulk of our results with the following commands: -```bash -python analyze_structure.py -python write_latex_table.py results/structure.csv -python analyze_solvetime.py -python write_latex_table.py results/solvetime.csv -python run_param_sweep.py --model=distill -python run_param_sweep.py --model=mb-steady -python run_param_sweep.py --model=pipeline -python summarize_sweep_results.py --model=distill -python summarize_sweep_results.py --model=mb-steady -python summarize_sweep_results.py --model=pipeline -``` -See below for more details, especially on how to speed this up in an HPC environment. - -### Producing results in parallel on HPC -The results can be time-consuming to reproduce, so we typically run them in parallel -on multiple-node/core HPC systems. This repository includes scripts to write command -lines that can be run in parallel with utilities like Slurm and GNU Parallel. - -## Reproducing structural results -Structural results are produced by the `analyze_structure.py` script: -```bash -python analyze_structure.py --results-dir=RESULTS_DIR -``` -Results are written to `RESULTS_DIR/structure.csv`. -Display these results as a Latex table, similar to that displayed in the paper, with: -```bash -python write_latex_table.py RESULTS_DIR/structure.csv +python var_elim/scripts/myscript.py --help ``` -To write structure-analysis commands that can be run in parallel: -```bash -python write_command_lines.py structure --results-dir=RESULTS_DIR -``` +## Using the `reproduce.py` script -To run these commands in parallel with multiple subprocesses: -```bash -# This may require installing GNU Parallel for the `parallel` command -parallel -a structure-commands.txt -``` - -To collect the results into a single file: -```bash -# Only necessary if we have written many small files in parallel runs! -python collect_results.py structure --results-dir=RESULTS_DIR -``` -The results are now combined into `RESULTS_DIR/structure.csv` and can be -displayed as above. -Figures may be produced with: -```bash -python plot_structure_bargraphs.py RESULTS_DIR/structure.csv --image-dir=IMAGE_DIR -``` - -## Reproducing numerical results -Numerical results are produced using the `analyze_solvetime.py` script: -```bash -python analyze_solvetime.py --results-dir=RESULTS_DIR -``` -Results are written to `RESULTS_DIR/solvetime.csv`, and can be displayed +For convenience, we have provided a Python script, `reproduce.py`, that allows +each set of results to be produced with a single command. +This script should be run from the root of this repository. +To make sure this script is working properly, run the "smoke tests" with: -```bash -python write_latex_table.py RESULTS_DIR/solvetime.csv -``` - -Independent commands for parallel runs can be written with: -```bash -python write_command_lines.py solvetime --results-dir=RESULTS_DIR -``` - -We typically prefer to run scripts that measure solvetimes on independent, -identical compute nodes rather than using multiple processes on the same -node. We do this with a Slurm batch script and the `sbatch` command. -The exact contents of this batch script depends on your HPC environment. - -Collect results into a single file with: -```bash -# Only necessary if we have written many small files in parallel runs! -python collect_results.py solvetime --results-dir=RESULTS_DIR -``` - -The breakdown of solve time can be plotted with: -```bash -python plot_timing_bargraphs.py RESULTS_DIR/solvetime.csv --image-dir=IMAGE_DIR -``` - -## Reproducing convergence results -Parameter sweeps are run with the `run_param_sweep.py` script: -```bash -python run_param_sweep.py --results-dir=RESULTS_DIR -``` -This writes a CSV file of convergence results for each model-method combination, -e.g., `mb-steady-matching-sweep.csv`, into the `RESULTS_DIR/sweep` subdirectory. - -A summary of sweep results may be displayed with: -```bash -python summarize_sweep_results.py --results-dir=RESULTS_DIR --model=MODEL -``` -where `MODEL` is one of `mb-steady`, `distill`, or `pipeline`. - -Parameter sweep success/failure results may be plotted in a grid with: -```bash -python plot_sweep_results.py SWEEP_CSV --image-dir=IMAGE_DIR -``` - -We typically run the parameter sweep for each model-method combination -on a different compute node (managed by e.g., Slurm), and use multiprocess -parallelism to run individual parameter samples in parallel within each node -(using GNU Parallel). - -Commands can be written with: -```bash -python write_sweep_command_lines.py --results-dir=RESULTS_DIR -``` -The commands for each model-method combination, written to -`commands/parallel-sweep-commands.txt`, can then be run on multiple compute -nodes with, for example, the Slurm `sbatch` command. - -Parameter sweep results for each model-method combination may be combined -into CSV files, e.g., `mb-steady-matching-sweep.csv` with: -```bash -# Alternatively, we could run each command in this file manually -parallel -a commands/collect-sweep-commands.txt -``` - -Plots may be generated with: -```bash -# Alternatively, we could run each command in this file manually -parallel -a commands/plot-sweep-commands.txt ``` +python reproduce.py structure --smoke +python reproduce.py solvetime --smoke +python reproduce.py convergence --smoke +``` +By default, this script saves results in the `runs/DATE` subdirectory, where +`DATE` is today's date in YYYYMMDD format. After running the smoke tests, +we should have the following files: +``` +runs +└── DATE + ├── images + │   ├── distill-matching-sweep-convergence.pdf + │   ├── fraction-elim.pdf + │   ├── fraction-solvetime.pdf + │   └── nnz-per-con.pdf + └── results + ├── solvetime-distill-matching.csv + ├── solvetime-distill-matching.txt + ├── structure-distill-matching.txt + ├── structure-distill.csv + ├── structure-distill.txt + └── sweep + ├── distill-matching-sweep-summary.csv + └── distill-matching-sweep.csv +``` +The full results can be produced with: +``` +python reproduce.py structure +python reproduce.py solvetime +python reproduce.py convergence +``` +The convergence results use 11 samples per parameter by default. +To run an abridged parameter sweep, use e.g., `--nsamples=2`. +After running these commands, we should have the following files: +``` +TODO +``` + +## Producing results in parallel on HPC +The results can be time-consuming to reproduce, so we typically run them in parallel +on multiple-node/core HPC systems. We have provided a makefile with the commands +used to run these results in parallel. # Using these methods on your own models The functionality to identify sets of variables and constraints to eliminate, perform From a5ab8b07dab7f3609d843d4afdaf628609d42d4d Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 18 Jul 2026 13:12:26 -0400 Subject: [PATCH 55/59] correct IPOPT version --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9427ef7..a93c972 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ As long as `pynumero` has status `OK`, this step was successful for the purpose of this repository. ### IPOPT -We use the IPOPT solver (version 3.14.10 with linear solver MA27) +We use the IPOPT solver (version 3.14.17 with linear solver MA27) to compare the impact of our aggregation techniques. IPOPT can be installed by following instructions on the github repository, [](https://github.com/coin-or/ipopt). From 4c8844f249fa50a848ec565be3e49b318d0c6b51 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 18 Jul 2026 13:13:21 -0400 Subject: [PATCH 56/59] typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a93c972..66b9e72 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ of this repository. We use the IPOPT solver (version 3.14.17 with linear solver MA27) to compare the impact of our aggregation techniques. IPOPT can be installed by following instructions on the github repository, -[](https://github.com/coin-or/ipopt). +https://github.com/coin-or/ipopt. See the following instructions to compile an Ipopt-compatible HSL library (which contains MA27): https://github.com/coin-or-tools/ThirdParty-HSL. From c1c95772002d53b58e19f9127cf5f6e4b7bcd67a Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 18 Jul 2026 13:19:58 -0400 Subject: [PATCH 57/59] update --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 66b9e72..8837cb3 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ Variable Elimination --------------------- +==================== This repository started as a place for collaborative exploratory work on variable elimination in nonlinear optimization problems. It now primarily serves to host the code necessary to reproduce the results in the paper @@ -25,6 +25,7 @@ and [IPOPT](https://github.com/coin-or/ipopt)) then install this repository as a Python package. ## Installing non-Python dependencies +If you already have PyNumero and IPOPT installed, you may skip this section. ### PyNumero @@ -56,8 +57,9 @@ We use the IPOPT solver (version 3.14.17 with linear solver MA27) to compare the impact of our aggregation techniques. IPOPT can be installed by following instructions on the github repository, https://github.com/coin-or/ipopt. -See the following instructions to compile an Ipopt-compatible HSL library -(which contains MA27): https://github.com/coin-or-tools/ThirdParty-HSL. +See the following repository for instructions to compile an Ipopt-compatible +HSL library (which contains MA27): +https://github.com/coin-or-tools/ThirdParty-HSL. > [!NOTE] > Depending on your IPOPT install location, you may have to set some environment From e7084ce9545ad730eadf8476b7c667f95f986e85 Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 18 Jul 2026 11:29:53 -0600 Subject: [PATCH 58/59] remove superfluous commands from makefile --- Makefile | 59 +++----------------------------------------------------- 1 file changed, 3 insertions(+), 56 deletions(-) diff --git a/Makefile b/Makefile index 7dd0ff0..741e096 100644 --- a/Makefile +++ b/Makefile @@ -1,64 +1,8 @@ -# TODO: Variable for results dir and/or suffix DIR=var_elim/scripts COMDIR=commands RESDIR=results IMDIR=images -all: - echo "WARNING: Reproducing all results in serial via `make all` is time-consuming." - echo "We recommend reproducing these results in an HPC environment with a resource" - echo "manager like Slurm, if available." - # Generate results for Table 3, Table 4, Figure 5, Figure 6, and Figure 7 - python $(DIR)/analyze_structure.py - # Table 3 - python $(DIR)/write_latex_table.py results/structure.csv - # Table 4 - python $(DIR)/write_latex_table.py results/structure.csv --which=matching-bounds - # Figures 5 and 7 - python $(DIR)/plot_structure_bargraphs.py results/structure.csv --image-dir=$(IMDIR) - # Figure 6 - python $(DIR)/plot_sparsity.py --model=mb-steady --image-dir=$(IMDIR) - # Generate results for Figure 8 and Table 5 - python $(DIR)/analyze_solvetime.py - # Table 5 - python $(DIR)/write_latex_table.py results/solvetime.csv - # Figure 8 - python $(DIR)/plot_timing_bargraphs.py results/solvetime.csv --image-dir=$(IMDIR) - # Generate results for Table 6 and Figures 9, 10, 11, and 12 - python $(DIR)/run_param_sweep.py --model=distill - python $(DIR)/run_param_sweep.py --model=mb-steady - python $(DIR)/run_param_sweep.py --model=pipeline - # Table 6 - python $(DIR)/summarize_sweep_results.py --model=distill - python $(DIR)/summarize_sweep_results.py --model=mb-steady - python $(DIR)/summarize_sweep_results.py --model=pipeline - # - # Each parameter sweep plot is produced individually... - # Figure 9 - python $(DIR)/plot_sweep_results.py $(DIR)/results/sweep/distill-no-elim-sweep.csv - python $(DIR)/plot_sweep_results.py $(DIR)/results/sweep/distill-d1-sweep.csv - python $(DIR)/plot_sweep_results.py $(DIR)/results/sweep/distill-trivial-sweep.csv - python $(DIR)/plot_sweep_results.py $(DIR)/results/sweep/distill-linear-d2-sweep.csv - python $(DIR)/plot_sweep_results.py $(DIR)/results/sweep/distill-d2-sweep.csv - python $(DIR)/plot_sweep_results.py $(DIR)/results/sweep/distill-ampl-sweep.csv - python $(DIR)/plot_sweep_results.py $(DIR)/results/sweep/distill-matching-sweep.csv - # Figure 10 - python $(DIR)/plot_sweep_results.py $(DIR)/results/sweep/mb-steady-no-elim-sweep.csv - python $(DIR)/plot_sweep_results.py $(DIR)/results/sweep/mb-steady-d1-sweep.csv - python $(DIR)/plot_sweep_results.py $(DIR)/results/sweep/mb-steady-trivial-sweep.csv - python $(DIR)/plot_sweep_results.py $(DIR)/results/sweep/mb-steady-linear-d2-sweep.csv - python $(DIR)/plot_sweep_results.py $(DIR)/results/sweep/mb-steady-d2-sweep.csv - python $(DIR)/plot_sweep_results.py $(DIR)/results/sweep/mb-steady-ampl-sweep.csv - python $(DIR)/plot_sweep_results.py $(DIR)/results/sweep/mb-steady-matching-sweep.csv - # Figure 11 - python $(DIR)/plot_sweep_results.py $(DIR)/results/sweep/pipeline-no-elim-sweep.csv - python $(DIR)/plot_sweep_results.py $(DIR)/results/sweep/pipeline-d1-sweep.csv - python $(DIR)/plot_sweep_results.py $(DIR)/results/sweep/pipeline-trivial-sweep.csv - python $(DIR)/plot_sweep_results.py $(DIR)/results/sweep/pipeline-linear-d2-sweep.csv - python $(DIR)/plot_sweep_results.py $(DIR)/results/sweep/pipeline-d2-sweep.csv - python $(DIR)/plot_sweep_results.py $(DIR)/results/sweep/pipeline-ampl-sweep.csv - python $(DIR)/plot_sweep_results.py $(DIR)/results/sweep/pipeline-matching-sweep.csv - structure-parallel: mkdir -p $(RESDIR)/structure # Letting this get created by the parallel scripts below leads to a race condition python $(DIR)/write_command_lines.py structure --results-dir=$(RESDIR) --commands-dir=$(COMDIR) @@ -81,15 +25,18 @@ solvetime-collect: python $(DIR)/plot_timing_bargraphs.py $(RESDIR)/solvetime.csv --image-dir=$(IMDIR) sweep-batch: + mkdir -p $(COMDIR) mkdir -p $(RESDIR)/sweep python $(DIR)/write_sweep_command_lines.py --results-dir=$(RESDIR)/sweep --commands-dir=$(COMDIR) --image-dir=$(IMDIR) # This is a custom command I use to submit batch jobs on multiple HPC nodes submit-batch.sh $(COMDIR)/parallel-sweep-commands.txt sweep-collect: + mkdir -p $(IMDIR) parallel -a $(COMDIR)/collect-sweep-commands.txt parallel -a $(COMDIR)/plot-sweep-commands.txt python $(DIR)/summarize_sweep_results.py --results-dir=$(RESDIR)/sweep --model=distill python $(DIR)/summarize_sweep_results.py --results-dir=$(RESDIR)/sweep --model=mb-steady python $(DIR)/summarize_sweep_results.py --results-dir=$(RESDIR)/sweep --model=pipeline + python $(DIR)/summarize_sweep_results.py --results-dir=$(RESDIR)/sweep python $(DIR)/write_latex_table.py $(RESDIR)/sweep-summary.csv --results-dir=$(RESDIR) From df4f3a5ee830c07527ef20866e05b781613a428d Mon Sep 17 00:00:00 2001 From: Robert Parker Date: Sat, 18 Jul 2026 14:30:41 -0400 Subject: [PATCH 59/59] add full sweep files to readme --- README.md | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8837cb3..ddb0355 100644 --- a/README.md +++ b/README.md @@ -159,10 +159,74 @@ python reproduce.py convergence The convergence results use 11 samples per parameter by default. To run an abridged parameter sweep, use e.g., `--nsamples=2`. After running these commands, we should have the following files: + +
+ + runs/DATE + ``` -TODO +runs +└── DATE + ├── images + │   ├── distill-d1-sweep-convergence.pdf + │   ├── distill-d2-sweep-convergence.pdf + │   ├── distill-ecd2-sweep-convergence.pdf + │   ├── distill-greedy-sweep-convergence.pdf + │   ├── distill-linear-d2-sweep-convergence.pdf + │   ├── distill-matching-sweep-convergence.pdf + │   ├── distill-no-elim-sweep-convergence.pdf + │   ├── fraction-elim.pdf + │   ├── fraction-solvetime.pdf + │   ├── mb-steady-d1-sweep-convergence.pdf + │   ├── mb-steady-d2-sweep-convergence.pdf + │   ├── mb-steady-ecd2-sweep-convergence.pdf + │   ├── mb-steady-greedy-sweep-convergence.pdf + │   ├── mb-steady-linear-d2-sweep-convergence.pdf + │   ├── mb-steady-matching-sparsity.pdf + │   ├── mb-steady-matching-sweep-convergence.pdf + │   ├── mb-steady-no-elim-sweep-convergence.pdf + │   ├── nnz-per-con.pdf + │   ├── pipeline-d1-sweep-convergence.pdf + │   ├── pipeline-d2-sweep-convergence.pdf + │   ├── pipeline-ecd2-sweep-convergence.pdf + │   ├── pipeline-greedy-sweep-convergence.pdf + │   ├── pipeline-linear-d2-sweep-convergence.pdf + │   ├── pipeline-matching-sweep-convergence.pdf + │   └── pipeline-no-elim-sweep-convergence.pdf + └── results + ├── solvetime.csv + ├── solvetime.txt + ├── structure-matching.txt + ├── structure.csv + ├── structure.txt + ├── sweep + │   ├── distill-d1-sweep.csv + │   ├── distill-d2-sweep.csv + │   ├── distill-ecd2-sweep.csv + │   ├── distill-greedy-sweep.csv + │   ├── distill-linear-d2-sweep.csv + │   ├── distill-matching-sweep.csv + │   ├── distill-no-elim-sweep.csv + │   ├── mb-steady-d1-sweep.csv + │   ├── mb-steady-d2-sweep.csv + │   ├── mb-steady-ecd2-sweep.csv + │   ├── mb-steady-greedy-sweep.csv + │   ├── mb-steady-linear-d2-sweep.csv + │   ├── mb-steady-matching-sweep.csv + │   ├── mb-steady-no-elim-sweep.csv + │   ├── pipeline-d1-sweep.csv + │   ├── pipeline-d2-sweep.csv + │   ├── pipeline-ecd2-sweep.csv + │   ├── pipeline-greedy-sweep.csv + │   ├── pipeline-linear-d2-sweep.csv + │   ├── pipeline-matching-sweep.csv + │   └── pipeline-no-elim-sweep.csv + ├── sweep-summary.csv + └── sweep-summary.txt ``` +
+ ## Producing results in parallel on HPC The results can be time-consuming to reproduce, so we typically run them in parallel on multiple-node/core HPC systems. We have provided a makefile with the commands