Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog/+flow-runs-task-manager-retention.changed.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
`infrahub-taskmanager flush flow-runs` works with Infrahub 1.13 and later, where `infrahub tasks flush flow-runs` takes no number of days. With those releases it deletes the runs older than the task history retention set on the task manager (`INFRAHUB_TASK_MANAGER_RETENTION_TASK_HISTORY`), warns when `days_to_keep` or `batch_size` is given because they are ignored, and with the new `--rewrite` flag rewrites the tables that are mostly free space to return their disk space. With earlier releases the command behaves as before.
18 changes: 15 additions & 3 deletions src/cmd/infrahub-taskmanager/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,14 @@ func main() {
Long: "Cleanup operations for Prefect resources.",
}

var rewrite bool
flowRunsCmd := &cobra.Command{
Use: "flow-runs [days_to_keep] [batch_size]",
Short: "Delete completed/failed/cancelled flow runs older than the retention period",
Use: "flow-runs [days_to_keep] [batch_size]",
Short: "Delete completed/failed/cancelled flow runs older than the retention period",
Long: "Delete completed/failed/cancelled flow runs older than the retention period.\n\n" +
"From Infrahub 1.13, the task manager deletes the runs older than its task history retention " +
"(INFRAHUB_TASK_MANAGER_RETENTION_TASK_HISTORY), so days_to_keep and batch_size are ignored: " +
"set the retention instead. --rewrite then returns the disk space of the deleted runs.",
Args: cobra.RangeArgs(0, 2),
SilenceUsage: true,
RunE: func(cmd *cobra.Command, args []string) error {
Expand All @@ -56,9 +61,16 @@ func main() {
return err
}
}
return iops.FlushFlowRuns(days, batch)
return iops.FlushFlowRuns(app.FlowRunsRequest{
DaysToKeep: days,
BatchSize: batch,
ArgumentsGiven: len(args) > 0,
Rewrite: rewrite,
})
},
}
flowRunsCmd.Flags().BoolVar(&rewrite, "rewrite", false,
"After the deletes, rewrite the tables that are mostly free space to return it to the disk (Infrahub 1.13 or later)")

staleRunsCmd := &cobra.Command{
Use: "stale-runs [days_to_keep] [batch_size]",
Expand Down
3 changes: 1 addition & 2 deletions src/internal/app/collect_extras.go
Original file line number Diff line number Diff line change
Expand Up @@ -235,8 +235,7 @@ func commandErrorLine(output string) string {
// ported from the Python collect_benchmark parser.
var benchmarkResultRe = regexp.MustCompile(`(\w+(?: \w+)*): (\d+)(?: MB)? - Required: (\d+)(?: MB)? : (\w+)`)

// ansiEscapeRe matches the ANSI SGR color sequences the benchmark entrypoint
// wraps its result lines in.
// ansiEscapeRe matches ANSI SGR color sequences in command output.
var ansiEscapeRe = regexp.MustCompile(`\x1b\[[0-9;]*m`)

// benchmarkMeasure is one parsed benchmark result; the field names mirror the
Expand Down
70 changes: 68 additions & 2 deletions src/internal/app/taskmanager.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,75 @@ var (
}
)

// FlowRunsRequest is what the user asked `flush flow-runs` for.
type FlowRunsRequest struct {
DaysToKeep int
BatchSize int
// ArgumentsGiven is true when the user passed days_to_keep or batch_size.
ArgumentsGiven bool
Rewrite bool
}

// FlushFlowRuns removes completed Prefect runs beyond the retention window.
func (iops *InfrahubOps) FlushFlowRuns(daysToKeep, batchSize int) error {
return iops.flushTaskRuns(flowRunsConfig, daysToKeep, batchSize)
//
// From Infrahub 1.13, `infrahub tasks flush flow-runs` takes no number of days: the task manager deletes up to its
// own task history retention, and `--rewrite` returns the disk space. Earlier releases take the number of days.
func (iops *InfrahubOps) FlushFlowRuns(request FlowRunsRequest) error {
if err := iops.checkPrerequisites(); err != nil {
return err
}
if err := iops.DetectEnvironment(); err != nil {
return err
}

// Wide enough that the Infrahub CLI prints its help and summary lines without wrapping them.
execOpts := iops.buildTaskWorkerExecOpts(&ExecOptions{Env: map[string]string{"COLUMNS": "200"}})
help, helpErr := iops.Exec("task-worker", []string{"infrahub", "tasks", "flush", "flow-runs", "--help"}, execOpts)
if helpErr != nil || !taskManagerOwnsRetention(help) {
if request.Rewrite {
logrus.Warn("This Infrahub release cannot rewrite the task history tables; --rewrite is ignored")
}
return iops.flushTaskRuns(flowRunsConfig, request.DaysToKeep, request.BatchSize)
}

if warning := ignoredArgumentsWarning(request); warning != "" {
logrus.Warn(warning)
}
command := flowRunsCommand(request)
logrus.Infof("Running %s in the task worker...", strings.Join(command, " "))
if _, err := iops.ExecStream("task-worker", command, execOpts); err != nil {
return fmt.Errorf("failed to execute %s: %w", strings.Join(command, " "), err)
}
logrus.Info("Flow runs cleanup completed")
return nil
}

// taskManagerOwnsRetention reports whether the help of `infrahub tasks flush flow-runs` is the one of Infrahub 1.13
// and later, which takes no number of days and offers --rewrite.
func taskManagerOwnsRetention(help string) bool {
help = ansiEscapeRe.ReplaceAllString(help, "")
return strings.Contains(help, "--rewrite") && !strings.Contains(help, "--days-to-keep")
}

// flowRunsCommand is the `infrahub tasks flush flow-runs` command of Infrahub 1.13 and later for the request.
func flowRunsCommand(request FlowRunsRequest) []string {
command := []string{"infrahub", "tasks", "flush", "flow-runs"}
if request.Rewrite {
command = append(command, "--rewrite")
}
return command
}

// ignoredArgumentsWarning explains that Infrahub 1.13 and later ignore days_to_keep and batch_size, when they were given.
func ignoredArgumentsWarning(request FlowRunsRequest) string {
if !request.ArgumentsGiven {
return ""
}
return fmt.Sprintf(
"This Infrahub release deletes the flow runs older than the task history retention set on the task manager, "+
"not the %d days given; to keep %d days, set INFRAHUB_TASK_MANAGER_RETENTION_TASK_HISTORY=%dd on the task manager",
request.DaysToKeep, request.DaysToKeep, request.DaysToKeep,
)
}

// FlushStaleRuns cancels running Prefect flow runs that exceeded retention.
Expand Down
123 changes: 123 additions & 0 deletions src/internal/app/taskmanager_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
package app

import (
"reflect"
"testing"
)

// Help texts of `infrahub tasks flush flow-runs`, printed with COLUMNS=100 by the Infrahub 1.11.4 image and by a
// build of the change that ships in Infrahub 1.13.
const (
flowRunsHelpBefore113 = ` Usage: infrahub tasks flush flow-runs [OPTIONS] [CONFIG_FILE]

Flush old task runs.

╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮
│ config_file [CONFIG_FILE] [env var: INFRAHUB_CONFIG] [default: infrahub.toml] │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
│ --days-to-keep INTEGER [default: 30] │
│ --batch-size INTEGER [default: 100] │
│ --help Show this message and exit. │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯`

flowRunsHelpFrom113 = ` Usage: infrahub tasks flush flow-runs [OPTIONS] [CONFIG_FILE]

Delete the finished task runs older than the task history retention, with their logs and
artifacts.

Raises: typer.Exit: With code 1 when the cleanup fails.

╭─ Arguments ──────────────────────────────────────────────────────────────────────────────────────╮
│ config_file [CONFIG_FILE] [env var: INFRAHUB_CONFIG] [default: infrahub.toml] │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯
╭─ Options ────────────────────────────────────────────────────────────────────────────────────────╮
│ --rewrite After the deletes, rewrite each task history and activity log table whose │
│ live rows hold less than half of its disk space, to return that space to the │
│ disk. Tasks that write to a table wait while it is rewritten. │
│ --help Show this message and exit. │
╰──────────────────────────────────────────────────────────────────────────────────────────────────╯`
)

func TestTaskManagerOwnsRetention(t *testing.T) {
cases := []struct {
name string
help string
want bool
}{
{name: "before 1.13", help: flowRunsHelpBefore113, want: false},
{name: "from 1.13", help: flowRunsHelpFrom113, want: true},
{
name: "colored help from 1.13",
help: "│ \x1b[1;36m-\x1b[0m\x1b[1;36m-rewrite\x1b[0m Return unused disk space. │",
want: true,
},
{
name: "colored legacy option still owns retention",
help: "--rewrite \x1b[1;36m-\x1b[0m\x1b[1;36m-days-to-keep\x1b[0m INTEGER",
want: false,
},
{name: "no help", help: "", want: false},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if got := taskManagerOwnsRetention(tc.help); got != tc.want {
t.Fatalf("taskManagerOwnsRetention() = %v, want %v", got, tc.want)
}
})
}
}

func TestFlowRunsCommand(t *testing.T) {
cases := []struct {
name string
request FlowRunsRequest
want []string
}{
{
name: "without rewrite",
request: FlowRunsRequest{DaysToKeep: 7, BatchSize: 200, ArgumentsGiven: true},
want: []string{"infrahub", "tasks", "flush", "flow-runs"},
},
{
name: "with rewrite",
request: FlowRunsRequest{DaysToKeep: 30, BatchSize: 200, Rewrite: true},
want: []string{"infrahub", "tasks", "flush", "flow-runs", "--rewrite"},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if got := flowRunsCommand(tc.request); !reflect.DeepEqual(got, tc.want) {
t.Fatalf("flowRunsCommand() = %v, want %v", got, tc.want)
}
})
}
}

func TestIgnoredArgumentsWarning(t *testing.T) {
cases := []struct {
name string
request FlowRunsRequest
want string
}{
{
name: "days given",
request: FlowRunsRequest{DaysToKeep: 7, BatchSize: 200, ArgumentsGiven: true},
want: "This Infrahub release deletes the flow runs older than the task history retention set on the task " +
"manager, not the 7 days given; to keep 7 days, set INFRAHUB_TASK_MANAGER_RETENTION_TASK_HISTORY=7d " +
"on the task manager",
},
{
name: "nothing given",
request: FlowRunsRequest{DaysToKeep: 30, BatchSize: 200},
want: "",
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if got := ignoredArgumentsWarning(tc.request); got != tc.want {
t.Fatalf("ignoredArgumentsWarning() = %q, want %q", got, tc.want)
}
})
}
}
Loading