coreos · tixxdz · Apr 14, 2016 · Apr 4, 2016 · Apr 13, 2016 · Apr 5, 2016
diff --git a/api/server.go b/api/server.go
@@ -38,6 +38,10 @@ type Server struct {
 	cur       http.Handler
 }
 
+func (s *Server) GetListeners() []net.Listener {
+	return s.listeners
+}
+
 func (s *Server) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
 	s.cur.ServeHTTP(rw, req)
 }
@@ -48,7 +52,7 @@ func (s *Server) Serve() {
 		go func() {
 			err := http.Serve(l, s)
 			if err != nil {
-				log.Errorf("Failed serving HTTP on listener: %v", l.Addr())
+				log.Errorf("Failed serving HTTP on listener: addr: %v, err: %v", l.Addr(), err)
 			}
 		}()
 	}

diff --git a/fleetd/fleetd.go b/fleetd/fleetd.go
@@ -99,7 +99,7 @@ func main() {
 	}
 
 	log.Debugf("Creating Server")
-	srv, err := server.New(*cfg)
+	srv, err := server.New(*cfg, nil)
 	if err != nil {
 		log.Fatalf("Failed creating Server: %v", err.Error())
 	}
@@ -119,13 +119,22 @@ func main() {
 		}
 
 		log.Infof("Restarting server components")
+		srv.SetReconfigServer(true)
+
+		// Get Server.listeners[] to keep it for a new server,
+		// before killing the old server.
+		oldListeners := srv.GetApiServerListeners()
+
 		srv.Kill()
 
-		srv, err = server.New(*cfg)
+		// The new server takes the original listeners.
+		srv, err = server.New(*cfg, oldListeners)
 		if err != nil {
 			log.Fatalf(err.Error())
 		}
+
 		srv.Run()
+		srv.SetReconfigServer(false)
 	}
 
 	shutdown := func() {

diff --git a/functional/server_test.go b/functional/server_test.go
@@ -0,0 +1,133 @@
+// Copyright 2016 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package functional
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/coreos/fleet/functional/platform"
+	"github.com/coreos/fleet/functional/util"
+)
+
+// TestReconfigureServer checks whether fleetd managed to keep its listeners
+// across reconfiguration of fleetd after receiving SIGHUP.
+func TestReconfigureServer(t *testing.T) {
+	cluster, err := platform.NewNspawnCluster("smoke")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer cluster.Destroy()
+
+	m0, err := cluster.CreateMember()
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = cluster.WaitForNMachines(m0, 1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// NOTE: we need to sleep once here to get reliable results.
+	// Without this sleep, the entire fleetd test always ends up succeeding
+	// no matter whether SIGHUP came or not.
+	_, _ = cluster.MemberCommand(m0, "sh", "-c", `'sleep 2'`)
+
+	err = waitForFleetdSocket(cluster, m0)
+	if err != nil {
+		t.Fatalf("Failed to get a list of fleetd sockets: %v", err)
+	}
+
+	// send a SIGHUP to fleetd, and periodically checks if a message
+	// "Reloading configuration" appears in fleet's journal, up to timeout (15) seconds.
+	stdout, _ := cluster.MemberCommand(m0, "sudo", "systemctl", "kill", "-s", "SIGHUP", "fleet")
+	if strings.TrimSpace(stdout) != "" {
+		t.Fatalf("Sending SIGHUP to fleetd returned: %s", stdout)
+	}
+
+	err = waitForReloadConfig(cluster, m0)
+	if err != nil {
+		t.Fatalf("Failed to get log about reconfiguration: %v", err)
+	}
+
+	// check if fleetd is still running correctly, by running fleetctl status
+	_, _, err = cluster.Fleetctl(m0, "list-units")
+	if err != nil {
+		t.Fatalf("Unable to check list-units. Please check for fleetd socket. err:%v", err)
+	}
+
+	// Check for HTTP listener error looking into the fleetd journal
+	stdout, _ = cluster.MemberCommand(m0, "journalctl _PID=$(pidof fleetd)")
+	if strings.Contains(strings.TrimSpace(stdout), "Failed serving HTTP on listener:") {
+		t.Fatalf("Fleetd log returned error on HTTP listeners: %s", stdout)
+	}
+
+	// Check expected state after reconfiguring fleetd
+	stdout, _ = cluster.MemberCommand(m0, "systemctl", "show", "--property=ActiveState", "fleet")
+	if strings.TrimSpace(stdout) != "ActiveState=active" {
+		t.Fatalf("Fleet unit not reported as active: %s", stdout)
+	}
+	stdout, _ = cluster.MemberCommand(m0, "systemctl", "show", "--property=Result", "fleet")
+	if strings.TrimSpace(stdout) != "Result=success" {
+		t.Fatalf("Result for fleet unit not reported as success: %s", stdout)
+	}
+}
+
+// waitForReloadConfig returns if a message "Reloading configuration" exists
+// in the journal, periodically checking for the journal up to the timeout.
+func waitForReloadConfig(cluster platform.Cluster, m0 platform.Member) (err error) {
+	_, err = util.WaitForState(
+		func() bool {
+			// NOTE: journalctl should run just simply like "journalctl -u fleet",
+			// without being piped with grep. Doing
+			// "journalctl -u fleet | grep \"Reloading configuration\"" is racy
+			// in a subtle way, so that it sometimes fails only on semaphoreci.
+			// - dpark 20160408
+			stdout, _ := cluster.MemberCommand(m0, "journalctl _PID=$(pidof fleetd)")
+			journalfleet := strings.TrimSpace(stdout)
+			if !strings.Contains(journalfleet, "Reloading configuration") {
+				fmt.Errorf("Fleetd is not fully reconfigured, retrying... entire fleet journal:\n%v", journalfleet)
+				return false
+			}
+			return true
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("Reloading configuration log not found: %v", err)
+	}
+
+	return nil
+}
+
+// waitForFleetdSocket returns if /var/run/fleet.sock exists, periodically
+// checking for states.
+func waitForFleetdSocket(cluster platform.Cluster, m0 platform.Member) (err error) {
+	_, err = util.WaitForState(
+		func() bool {
+			stdout, _ := cluster.MemberCommand(m0, "test -S /var/run/fleet.sock && echo 1")
+			if strings.TrimSpace(stdout) == "" {
+				fmt.Errorf("Fleetd is not fully started, retrying...")
+				return false
+			}
+			return true
+		},
+	)
+	if err != nil {
+		return fmt.Errorf("Fleetd socket not found: %v", err)
+	}
+
+	return nil
+}
diff --git a/server/server.go b/server/server.go
@@ -17,6 +17,7 @@ package server
 import (
 	"encoding/json"
 	"errors"
+	"net"
 	"net/http"
 	"sync"
 	"time"
@@ -48,16 +49,17 @@ const (
 )
 
 type Server struct {
-	agent         *agent.Agent
-	aReconciler   *agent.AgentReconciler
-	usPub         *agent.UnitStatePublisher
-	usGen         *unit.UnitStateGenerator
-	engine        *engine.Engine
-	mach          *machine.CoreOSMachine
-	hrt           heart.Heart
-	mon           *Monitor
-	api           *api.Server
-	disableEngine bool
+	agent          *agent.Agent
+	aReconciler    *agent.AgentReconciler
+	usPub          *agent.UnitStatePublisher
+	usGen          *unit.UnitStateGenerator
+	engine         *engine.Engine
+	mach           *machine.CoreOSMachine
+	hrt            heart.Heart
+	mon            *Monitor
+	api            *api.Server
+	disableEngine  bool
+	reconfigServer bool
 
 	engineReconcileInterval time.Duration
 
@@ -66,7 +68,7 @@ type Server struct {
 	wg    sync.WaitGroup // used to co-ordinate shutdown
 }
 
-func New(cfg config.Config) (*Server, error) {
+func New(cfg config.Config, listeners []net.Listener) (*Server, error) {
 	agentTTL, err := time.ParseDuration(cfg.AgentTTL)
 	if err != nil {
 		return nil, err
@@ -115,9 +117,11 @@ func New(cfg config.Config) (*Server, error) {
 
 	e := engine.New(reg, lManager, rStream, mach)
 
-	listeners, err := activation.Listeners(false)
-	if err != nil {
-		return nil, err
+	if len(listeners) == 0 {
+		listeners, err = activation.Listeners(false)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	hrt := heart.New(reg, mach)
@@ -142,6 +146,7 @@ func New(cfg config.Config) (*Server, error) {
 		stopc:       nil,
 		engineReconcileInterval: eIval,
 		disableEngine:           cfg.DisableEngine,
+		reconfigServer:          false,
 	}
 
 	return &srv, nil
@@ -239,7 +244,9 @@ func (s *Server) Supervise() {
 
 // Kill is used to gracefully terminate the server by triggering the Monitor to shut down
 func (s *Server) Kill() {
-	close(s.killc)
+	if !s.reconfigServer {
+		close(s.killc)
+	}
 }
 
 func (s *Server) Purge() {
@@ -260,3 +267,11 @@ func (s *Server) MarshalJSON() ([]byte, error) {
 		UnitStateGenerator: s.usGen,
 	})
 }
+
+func (s *Server) GetApiServerListeners() []net.Listener {
+	return s.api.GetListeners()
+}
+
+func (s *Server) SetReconfigServer(isReconfigServer bool) {
+	s.reconfigServer = isReconfigServer
+}