Skip to content

Commit a09ef17

Browse files
dbpolitodbpolito
committed
fix: rollback failed proxy listener expansion
- recreate the original proxy after expansion failures - restore prior ports, networks, readiness, and Caddy apps - join expansion and container rollback failures
1 parent e381e9e commit a09ef17

2 files changed

Lines changed: 90 additions & 35 deletions

File tree

‎services/proxy/manager.go‎

Lines changed: 81 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -428,13 +428,32 @@ func (m *DefaultManager) createAliasOverride(network string, routes []route) (st
428428
return file.Name(), nil
429429
}
430430

431-
func (m *DefaultManager) ensureCaddy(network string, routes []route) error {
431+
func (m *DefaultManager) ensureCaddy(network string, routes []route) (err error) {
432432
inspect := builder.NewCommand("docker", "inspect", "--format", "{{.State.Running}}", caddyContainer)
433433
running, err := m.shell.Exec(inspect)
434434
ports := make(map[int]bool)
435435
var preservedApps []byte
436436
var preservedAppsExist bool
437437
var preservedNetworks []string
438+
var originalPorts map[int]bool
439+
expanding := false
440+
defer func() {
441+
if err == nil || !expanding {
442+
return
443+
}
444+
_ = m.shell.Interactive(builder.NewCommand("docker", "rm", "--force"), caddyContainer)
445+
rollbackErr := m.createCaddy(originalPorts)
446+
if rollbackErr == nil {
447+
rollbackErr = m.connectCaddyNetworks(preservedNetworks)
448+
}
449+
if rollbackErr == nil {
450+
rollbackErr = m.waitForCaddy()
451+
}
452+
if rollbackErr == nil && preservedAppsExist {
453+
rollbackErr = m.restoreApps(preservedApps, true)
454+
}
455+
err = errors.Join(err, rollbackErr)
456+
}()
438457
if err == nil {
439458
compatibility, inspectErr := m.shell.Exec(builder.NewCommand("docker", "inspect", "--format", "{{json .NetworkSettings.Networks}}|{{json .Config.Entrypoint}}|{{json .Config.Cmd}}", caddyContainer))
440459
if inspectErr != nil {
@@ -462,6 +481,7 @@ func (m *DefaultManager) ensureCaddy(network string, routes []route) error {
462481
for port := range parseContainerPorts(bindings) {
463482
ports[port] = true
464483
}
484+
originalPorts = copyPortSet(ports)
465485
networks, inspectErr := m.shell.Exec(builder.NewCommand("docker", "inspect", "--format", "{{json .NetworkSettings.Networks}}", caddyContainer))
466486
if inspectErr != nil {
467487
return inspectErr
@@ -470,44 +490,17 @@ func (m *DefaultManager) ensureCaddy(network string, routes []route) error {
470490
if err = m.shell.Interactive(builder.NewCommand("docker", "rm", "--force"), caddyContainer); err != nil {
471491
return err
472492
}
493+
expanding = true
473494
err = errors.New("proxy listener expansion required")
474495
break
475496
}
476497
}
477498
}
478499
if err != nil {
479-
configPath, configErr := m.ensureBaseConfig()
480-
if configErr != nil {
481-
return configErr
482-
}
483-
if _, networkErr := m.shell.Exec(builder.NewCommand("docker", "network", "inspect", caddyAdminNet)); networkErr != nil {
484-
if networkErr = m.shell.Interactive(builder.NewCommand("docker", "network", "create"), caddyAdminNet); networkErr != nil {
485-
return networkErr
486-
}
487-
}
488-
args := []string{"run", "-d", "--name", caddyContainer, "--restart", "unless-stopped", "--network", caddyAdminNet, "--network-alias", caddyAdminHost, "-p", "127.0.0.1:2019:2019"}
489500
for _, route := range routes {
490501
ports[route.Listen] = true
491502
}
492-
var sortedPorts []int
493-
for port := range ports {
494-
sortedPorts = append(sortedPorts, port)
495-
}
496-
sort.Ints(sortedPorts)
497-
for _, port := range sortedPorts {
498-
mapping := fmt.Sprintf("%d:%d", port, port)
499-
args = append(args, "-p", mapping)
500-
}
501-
args = append(args,
502-
"-v", configPath+":/etc/caddy/caddy.json:ro",
503-
"-v", caddyVolume+":/var/lib/caddy",
504-
"-e", "XDG_CONFIG_HOME=/var/lib/caddy/config",
505-
"-e", "XDG_DATA_HOME=/var/lib/caddy/data",
506-
"--entrypoint", "/bin/sh",
507-
caddyImage,
508-
"-c", caddyStartCmd,
509-
)
510-
if err = m.shell.Interactive(builder.NewCommand("docker"), args...); err != nil {
503+
if err = m.createCaddy(ports); err != nil {
511504
return err
512505
}
513506
} else if running != "true" {
@@ -538,14 +531,59 @@ func (m *DefaultManager) ensureCaddy(network string, routes []route) error {
538531
return fmt.Errorf("kool proxy does not publish port %d; remove %s and retry", route.Listen, caddyContainer)
539532
}
540533
}
534+
if err = m.waitForCaddy(); err != nil {
535+
return err
536+
}
537+
if preservedAppsExist {
538+
if err = m.restoreApps(preservedApps, true); err != nil {
539+
return err
540+
}
541+
}
542+
expanding = false
543+
return nil
544+
}
545+
546+
func (m *DefaultManager) createCaddy(ports map[int]bool) error {
547+
configPath, err := m.ensureBaseConfig()
548+
if err != nil {
549+
return err
550+
}
551+
if _, err = m.shell.Exec(builder.NewCommand("docker", "network", "inspect", caddyAdminNet)); err != nil {
552+
if err = m.shell.Interactive(builder.NewCommand("docker", "network", "create"), caddyAdminNet); err != nil {
553+
return err
554+
}
555+
}
556+
args := []string{"run", "-d", "--name", caddyContainer, "--restart", "unless-stopped", "--network", caddyAdminNet, "--network-alias", caddyAdminHost, "-p", "127.0.0.1:2019:2019"}
557+
var sortedPorts []int
558+
for port := range ports {
559+
sortedPorts = append(sortedPorts, port)
560+
}
561+
sort.Ints(sortedPorts)
562+
for _, port := range sortedPorts {
563+
args = append(args, "-p", fmt.Sprintf("%d:%d", port, port))
564+
}
565+
args = append(args, "-v", configPath+":/etc/caddy/caddy.json:ro", "-v", caddyVolume+":/var/lib/caddy", "-e", "XDG_CONFIG_HOME=/var/lib/caddy/config", "-e", "XDG_DATA_HOME=/var/lib/caddy/data", "--entrypoint", "/bin/sh", caddyImage, "-c", caddyStartCmd)
566+
return m.shell.Interactive(builder.NewCommand("docker"), args...)
567+
}
568+
569+
func (m *DefaultManager) connectCaddyNetworks(networks []string) error {
570+
for _, network := range networks {
571+
if network == caddyAdminNet {
572+
continue
573+
}
574+
if err := m.shell.Interactive(builder.NewCommand("docker", "network", "connect", network), caddyContainer); err != nil {
575+
return err
576+
}
577+
}
578+
return nil
579+
}
580+
581+
func (m *DefaultManager) waitForCaddy() error {
541582
for attempt := 0; attempt < 30; attempt++ {
542-
response, requestErr := m.request(http.MethodGet, m.adminURL+"/config/", nil)
543-
if requestErr == nil {
583+
response, err := m.request(http.MethodGet, m.adminURL+"/config/", nil)
584+
if err == nil {
544585
_ = response.Body.Close()
545586
if response.StatusCode < 500 {
546-
if preservedAppsExist {
547-
return m.restoreApps(preservedApps, true)
548-
}
549587
return nil
550588
}
551589
}
@@ -554,6 +592,14 @@ func (m *DefaultManager) ensureCaddy(network string, routes []route) error {
554592
return errors.New("kool proxy did not become ready")
555593
}
556594

595+
func copyPortSet(ports map[int]bool) map[int]bool {
596+
copy := make(map[int]bool, len(ports))
597+
for port := range ports {
598+
copy[port] = true
599+
}
600+
return copy
601+
}
602+
557603
func parseContainerPorts(raw string) map[int]bool {
558604
ports := make(map[int]bool)
559605
var bindings map[string]interface{}

‎services/proxy/manager_test.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -770,6 +770,15 @@ func TestParseExistingProxyPortsAndNetworks(t *testing.T) {
770770
}
771771
}
772772

773+
func TestCopyPortSetPreservesOriginalExpansionPorts(t *testing.T) {
774+
original := map[int]bool{80: true}
775+
expanded := copyPortSet(original)
776+
expanded[3001] = true
777+
if !original[80] || original[3001] || !expanded[3001] {
778+
t.Fatalf("expected rollback ports to remain independent, original=%v expanded=%v", original, expanded)
779+
}
780+
}
781+
773782
func TestRestoreAppsIfUnchangedPreservesConcurrentUpdate(t *testing.T) {
774783
current := []byte(`{"http":{"servers":{"current":{}}}}`)
775784
restored := false

0 commit comments

Comments
 (0)