succd: MODBUS library does not differentiate between TCP socket time outs and RTU time outs

The accesses to -KEC1 always time out on the attempt to update the
output values.  We noticed that this is related to the timing between
the reading of the inputs and the following write to the outputs.

Fix -KEC1 accesses by waiting before sending the next request to the
board after receiving the reply for the previous one.

succbone/succd/go.mod

@@ -4,5 +4,8 @@ go 1.22.3
 
 require (
 	github.com/coder/websocket v1.8.12
+	github.com/simonvetter/modbus v1.6.3
 	k8s.io/klog v1.0.0
 )
+
+require github.com/goburrow/serial v0.1.0 // indirect

succbone/succd/go.sum

@@ -1,5 +1,9 @@
 github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo=
 github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs=
 github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/goburrow/serial v0.1.0 h1:v2T1SQa/dlUqQiYIT8+Cu7YolfqAi3K96UmhwYyuSrA=
+github.com/goburrow/serial v0.1.0/go.mod h1:sAiqG0nRVswsm1C97xsttiYCzSLBmUZ/VSlVLZJ8haA=
+github.com/simonvetter/modbus v1.6.3 h1:kDzwVfIPczsM4Iz09il/Dij/bqlT4XiJVa0GYaOVA9w=
+github.com/simonvetter/modbus v1.6.3/go.mod h1:hh90ZaTaPLcK2REj6/fpTbiV0J6S7GWmd8q+GVRObPw=
 k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
 k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=

succbone/succd/http.go

@@ -62,6 +62,17 @@ type apiData struct {
 		// DPOn means the diffusion pump is turned on.
 		DPOn bool
 	}
+	// Temperature state.
+	Temperatures struct {
+		DPBottom float32
+		DPTop    float32
+		DPInlet  float32
+		SEM      float32
+	}
+	// Humidity state.
+	Humidity struct {
+		SEM float32
+	}
 	// Pressure feedback into evacuation board.
 	Feedback struct {
 		// RoughReached is true when the system has reached a rough vacuum
@@ -115,6 +126,11 @@ func (s *webServer) apiData(skipSystem bool) *apiData {
 	ad.Pirani.MbarFloat = mbar
 	ad.Pumps.RPOn = state.rpOn
 	ad.Pumps.DPOn = state.dpOn
+	ad.Temperatures.DPBottom = state.tempDPBottom
+	ad.Temperatures.DPTop = state.tempDPTop
+	ad.Temperatures.DPInlet = state.tempDPInlet
+	ad.Temperatures.SEM = state.tempSEM
+	ad.Humidity.SEM = state.humiditySEM
 	ad.Feedback.RoughReached = rough
 	ad.Feedback.HighReached = high
 	ad.LoopLoad = s.d.loopLoad()
@@ -217,6 +233,26 @@ func (s *webServer) viewMetrics(w http.ResponseWriter, r *http.Request) {
 	fmt.Fprintf(w, "# HELP sem_vacuum_high_reached Whether a high vacuum has been reached (boolean)\n")
 	fmt.Fprintf(w, "# TYPE sem_vacuum_high_reached gauge\n")
 	fmt.Fprintf(w, "sem_vacuum_high_reached %f\n", boolToFloat(high))
+
+	fmt.Fprintf(w, "# HELP sem_environment_temperature_celsius Environmental temperature of the SEM, in degrees celsius\n")
+	fmt.Fprintf(w, "# TYPE sem_environment_temperature_celsius gauge\n")
+	fmt.Fprintf(w, "sem_environment_temperature_celsius %f\n", state.tempSEM)
+
+	fmt.Fprintf(w, "# HELP sem_environment_humidity_percent Environmental relative humidity of the SEM, in percent\n")
+	fmt.Fprintf(w, "# TYPE sem_environment_humidity_percent gauge\n")
+	fmt.Fprintf(w, "sem_environment_humidity_percent %f\n", state.humiditySEM)
+
+	fmt.Fprintf(w, "# HELP sem_dp_bottom_temperature_celsius Temperature of the DP bottom, in degrees celsius\n")
+	fmt.Fprintf(w, "# TYPE sem_dp_bottom_temperature_celsius gauge\n")
+	fmt.Fprintf(w, "sem_dp_bottom_temperature_celsius %f\n", state.tempDPBottom)
+
+	fmt.Fprintf(w, "# HELP sem_dp_top_temperature_celsius Temperature of the DP top, in degrees celsius\n")
+	fmt.Fprintf(w, "# TYPE sem_dp_top_temperature_celsius gauge\n")
+	fmt.Fprintf(w, "sem_dp_top_temperature_celsius %f\n", state.tempDPTop)
+
+	fmt.Fprintf(w, "# HELP sem_dp_inlet_temperature_celsius Temperature of the DP inlet flange, in degrees celsius\n")
+	fmt.Fprintf(w, "# TYPE sem_dp_inlet_temperature_celsius gauge\n")
+	fmt.Fprintf(w, "sem_dp_inlet_temperature_celsius %f\n", state.tempDPInlet)
 }
 
 func (s *webServer) viewRoughingPumpEnable(w http.ResponseWriter, r *http.Request) {

succbone/succd/index.html

@@ -9,19 +9,19 @@ body {
     padding: 2em;
 }
 table {
-    font-size: 40px;
+    font-size: 30px;
 }
 table.status td {
     width: 2em;
 }
 th, td {
     background-color: #e8e8e8;
-    padding: 0.4em;
+    padding: 0.3em;
 }
 th {
     font-weight: 100;
     text-align: right;
-    font-size: 30px;
+    font-size: 25px;
 }
 td {
     text-align: left;
@@ -34,7 +34,7 @@ h2 {
     font-weight: 100;
 }
 button {
-    height: 4.5em;
+    height: 3.3em;
     padding-left: 1.5em;
     padding-right: 1.5em;
 }
@@ -76,6 +76,7 @@ td > span {
 
 .has-hidden:hover .hidden-text {
     display: block;
+}
 
 @media only screen and (max-width: 700px) {
     body {
@@ -120,19 +121,6 @@ td > span {
         </tr>
     </table>
 
-    <table>
-        <tr>
-            <th>Pirani Pressure</th>
-            <td class="has-hidden">
-                <div id="mbar">{{ .Pirani.Mbar }}</div>
-                <div class="hidden-text" style="color: #606060;">
-                    <span>Voltage: </span><span id="volts">{{ .Pirani.Volts }}</span>
-                </div>
-            </td>
-        </tr>
-    </table>
-
-
     <table>
         <tr>
             <th rowspan="3">Control</th>
@@ -163,6 +151,41 @@ td > span {
         </tr>
     </table>
 
+    <table>
+        <tr>
+            <th>Pirani Pressure</th>
+            <td colspan="2" class="has-hidden">
+                <div id="mbar">{{ .Pirani.Mbar }}</div>
+                <div class="hidden-text" style="color: #606060;">
+                    <span>Voltage: </span><span id="volts">{{ .Pirani.Volts }}</span>
+                </div>
+            </td>
+        </tr>
+        <tr>
+            <th rowspan="4">Temperatures</th>
+            <th>DP Bottom</th>
+            <td id="temp-dp-bottom">{{ .Temperatures.DPBottom }}&nbsp;°C</td>
+        </tr>
+        <tr>
+            <th>DP Top</th>
+            <td id="temp-dp-top">{{ .Temperatures.DPTop }}&nbsp;°C</td>
+        </tr>
+        <tr>
+            <th>DP Inlet</th>
+            <td id="temp-dp-inlet">{{ .Temperatures.DPInlet }}&nbsp;°C</td>
+        </tr>
+        <tr>
+            <th>SEM Environment</th>
+            <td id="temp-sem">{{ .Temperatures.SEM }}&nbsp;°C</td>
+        </tr>
+        <tr>
+            <th>Humidity</th>
+            <th>SEM Environment</th>
+            <td id="humidity-sem">{{ .Humidity.SEM }}%</td>
+        </tr>
+    </table>
+
+
     <div class="graph-container">
         <canvas id="graph" width="1024" height="512" style="max-width: 100%;"></canvas>
     </div>
@@ -316,6 +339,11 @@ window.addEventListener("load", (_) => {
     let trough = document.querySelector("#trough");
     let thigh = document.querySelector("#thigh");
     let load = document.querySelector("#load");
+    let tempSEM = document.querySelector("#temp-sem");
+    let tempDPBottom = document.querySelector("#temp-dp-bottom");
+    let tempDPTop = document.querySelector("#temp-dp-top");
+    let tempDPInlet = document.querySelector("#temp-dp-inlet");
+    let humiditySEM = document.querySelector('#humidity-sem');
 
     // Buttons
     let pd = document.querySelector("#pd");
@@ -388,6 +416,30 @@ window.addEventListener("load", (_) => {
                 dp.style = colors.default;
             }
 
+            tempSEM.innerHTML = data.Temperatures.SEM + "&nbsp;°C";
+            tempSEM.style = (data.Temperatures.SEM > 30) ?
+                colors.highlightCaution : colors.default;
+            humiditySEM.innerHTML = data.Humidity.SEM + "%";
+            humiditySEM.style = (data.Humidity.SEM > 59) ?
+                colors.highlightCaution : colors.default;
+
+            tempDPTop.innerHTML = data.Temperatures.DPTop + "&nbsp;°C";
+            tempDPTop.style = (data.Temperatures.DPTop > 30) ?
+                colors.highlightCaution : colors.default;
+
+            tempDPInlet.innerHTML = data.Temperatures.DPInlet + "&nbsp;°C";
+            tempDPInlet.style = (data.Temperatures.DPInlet > 30) ?
+                colors.highlightCaution : colors.default;
+
+            tempDPBottom.innerHTML = data.Temperatures.DPBottom + "&nbsp;°C";
+            if (data.Temperatures.DPBottom > 200) {
+                tempDPBottom.style = colors.highlightFault;
+            } else if (data.Temperatures.DPBottom > 50) {
+                tempDPBottom.style = colors.highlightNeutral;
+            } else {
+                tempDPBottom.style = colors.default;
+            }
+
             let t = [];
             if (data.Feedback.RoughReached) {
                 trough.innerHTML = "OK";
@@ -413,7 +465,7 @@ window.addEventListener("load", (_) => {
 
             // Indicate all process values as unknown
 
-            [failsafe, highpressure, rp, dp, trough, thigh, volts, mbar].forEach((el) => {
+            [failsafe, highpressure, rp, dp, trough, thigh, volts, mbar, tempDPBottom, tempDPTop, tempDPInlet].forEach((el) => {
                 if (!el.innerHTML.includes("??")) {
                     el.innerHTML += "??";
                 }

succbone/succd/main.go

@@ -35,6 +35,12 @@ func main() {
 	d.daemonState.piraniVolts3.limit = 3
 	d.daemonState.piraniVolts100.limit = 100
 
+	d.tempDPBottom = 420.6
+	d.tempDPTop = 69.0
+	d.tempDPInlet = 42.0
+	d.tempSEM = 42.5
+	d.humiditySEM = 66.6
+
 	d.aboveRough.threshold = float64(flagPressureThresholdRough)
 	d.aboveRough.hysteresis = float64(flagPressureThresholdRoughHysteresis)
 	d.aboveHigh.threshold = float64(flagPressureThresholdHigh)
@@ -42,12 +48,6 @@ func main() {
 	if flagFake {
 		klog.Infof("Starting with fake peripherals")
 		d.adcPirani = &fakeADC{}
-		d.gpioRoughingPump = &fakeGPIO{desc: "rp"}
-		d.gpioDiffusionPump = &fakeGPIO{desc: "~dp"}
-		d.gpioBtnPumpDown = &fakeGPIO{desc: "~pd"}
-		d.gpioBtnVent = &fakeGPIO{desc: "~vent"}
-		d.gpioBelowRough = &fakeGPIO{desc: "~rough"}
-		d.gpioBelowHigh = &fakeGPIO{desc: "~high"}
 	} else {
 		adc, err := newBBADC(0)
 		if err != nil {
@@ -55,21 +55,9 @@ func main() {
 		}
 		d.adcPirani = adc
 
-		for _, c := range []struct {
-			out *gpio
-			num int
-		}{
-			{&d.gpioRoughingPump, 115},
-			{&d.gpioDiffusionPump, 49},
-			{&d.gpioBtnPumpDown, 48},
-			{&d.gpioBtnVent, 60},
-			{&d.gpioBelowRough, 30},
-			{&d.gpioBelowHigh, 7},
-		} {
-			*c.out, err = newBBGPIO(c.num, true)
-			if err != nil {
-				klog.Exitf("Failed to setup GPIO: %v", err)
-			}
+		err = d.modbusConnect()
+		if err != nil {
+			klog.Exitf("Failed to connect to modbus %v", err)
 		}
 	}
 
@@ -86,5 +74,9 @@ func main() {
 	}()
 
 	go d.process(ctx)
+	if !flagFake {
+		go d.modbusProcess(ctx)
+	}
+
 	<-ctx.Done()
 }

succbone/succd/modbus.go

@@ -0,0 +1,186 @@
+package main
+
+import (
+	"context"
+	"time"
+
+	"github.com/simonvetter/modbus"
+	"k8s.io/klog"
+)
+
+func modbusValuesToFloat(v uint16) float32 {
+	return float32(v) / 10.0
+}
+
+func (d *daemon) modbusConnect() error {
+	var err error
+
+	d.mu.Lock()
+	defer d.mu.Unlock()
+
+	// Setup modbus client
+	d.modbusClient, err = modbus.NewClient(&modbus.ClientConfiguration{
+		URL:     "tcp://10.250.241.20:8887",
+		Timeout: 1 * time.Second,
+	})
+	if err != nil {
+		return err
+	}
+	// Connect to modbus client
+	err = d.modbusClient.Open()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (d *daemon) modbusRestart() error {
+	d.modbusClient.Close()
+	return d.modbusClient.Open()
+}
+
+// There are currently two devices connected to the modbus.
+// The first one (slave 1) is a temperature/humidity sensor.
+// The second one (slave 2) is a PTA8D08 transmitter
+//
+// Returns whether modbus should restart (only in case of an underlying network error)
+func (d *daemon) modbusUpdate() bool {
+	var err error
+	var numDevicesNotResponding int
+
+	// Switch to slave 1 (BTA1)
+	d.modbusClient.SetUnitId(1)
+
+	// Read temperature and humidity
+	var registersBTA1 []uint16 // temperature, humidity
+	registersBTA1, err = d.modbusClient.ReadRegisters(1, 2, modbus.INPUT_REGISTER)
+	if err != nil {
+		numDevicesNotResponding += 1
+		klog.Warningf("error while reading registers from BTA1 %v", err)
+	} else if len(registersBTA1) != 2 {
+		klog.Warningf("expected two registers from modbus slave 1, but got %d", len(registersBTA1))
+	} else {
+		d.mu.Lock()
+		d.daemonState.tempSEM = modbusValuesToFloat(registersBTA1[0])
+		d.daemonState.humiditySEM = modbusValuesToFloat(registersBTA1[1])
+		d.mu.Unlock()
+	}
+
+	// Switch to slave 2 (KEC2)
+	d.modbusClient.SetUnitId(2)
+
+	// PT100 mapping
+	// Channel 0: Cable -WGA6, Sensor "dp bottom"
+	// Channel 1: Cable -WGA8, Sensor "dp inlet"
+	// Channel 2: Cable WGA7, Sensor "dp top"
+	var registersKEC2 []uint16 // temperatures dp
+	registersKEC2, err = d.modbusClient.ReadRegisters(0, 3, modbus.HOLDING_REGISTER)
+	if err != nil {
+		numDevicesNotResponding += 1
+		klog.Warningf("error while reading registers from KEC2 %v", err)
+	} else if len(registersKEC2) != 3 {
+		klog.Warningf("expected three registers from modbus slave 2, but got %d", len(registersKEC2))
+	} else {
+		d.mu.Lock()
+		d.daemonState.tempDPBottom = modbusValuesToFloat(registersKEC2[0])
+		d.daemonState.tempDPInlet = modbusValuesToFloat(registersKEC2[1])
+		d.daemonState.tempDPTop = modbusValuesToFloat(registersKEC2[2])
+		d.mu.Unlock()
+	}
+
+	// Switch to slave 3 (KEC1)
+	d.modbusClient.SetUnitId(3)
+
+	// Do a read first to avoid side-effects from the subsequent write to the relay states
+	var digitalInputs [8]bool
+	var digitalInputRegisters []uint16
+
+	digitalInputRegisters, err = d.modbusClient.ReadRegisters(0x81, 8, modbus.HOLDING_REGISTER)
+	if err != nil {
+		numDevicesNotResponding += 1
+		klog.Warningf("error while reading digital inputs from KEC1 %v", err)
+	} else {
+		// Convert MODBUS words into bools
+		for idx, value := range digitalInputRegisters {
+			if value != 0 {
+				digitalInputs[idx] = true
+			} else {
+				digitalInputs[idx] = false
+			}
+		}
+		// TODO: Input mapping goes here
+	}
+
+	// We must wait between reading and writing to the -KEC1 relay board
+	// because otherwise it chokes and times out the write registers
+	// command.
+	time.Sleep(time.Millisecond * 10)
+
+	// KFA1-KFA8
+	var relayState [8]bool
+	d.mu.Lock()
+	// -KFA1 Roughing Pump (normally closed contact)
+	relayState[0] = !d.daemonState.rpOn
+	// -KFA2 Diffusion Pump
+	relayState[1] = d.daemonState.dpOn
+	// -KFA4 Button Vent
+	relayState[3] = d.daemonState.vent.output
+	// -KFA5 Button Pump-Down
+	relayState[4] = d.daemonState.pumpdown.output
+	// -KFA6 Fake-Pirani Rough (normally closed contact)
+	relayState[5] = !d.aboveRough.output
+	// -KFA7 Fake-Pirani High (normally closed contact)
+	relayState[6] = !d.aboveHigh.output
+	d.mu.Unlock()
+
+	// The KEC1 module uses a non-standard MODBUS interface
+	// instead of coils
+	// 0x0100 is the open command
+	// 0x0200 is the close command
+	// We write 8 words (16-bit) to address 0x01 to update the relays
+	var registerValuesKEC1 [8]uint16
+	// Convert the boolean values to the commands
+	for idx, state := range relayState {
+		if state {
+			registerValuesKEC1[idx] = 0x0100
+		} else {
+			registerValuesKEC1[idx] = 0x0200
+		}
+	}
+
+	err = d.modbusClient.WriteRegisters(0x01, registerValuesKEC1[:])
+	if err != nil {
+		numDevicesNotResponding += 1
+		klog.Warningf("error while updating registers %v", err)
+	}
+
+	if numDevicesNotResponding >= 4 {
+		klog.Warningf("no device did respond to our request. Probably a network timeout.")
+		return true
+	}
+
+	return false
+}
+
+// Call modbusUpdate every 100 milliseconds
+func (d *daemon) modbusProcess(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+			shouldRestart := d.modbusUpdate()
+			// the modbus library does not reopen the tcp socket in case of
+			// a connection loss.
+			if shouldRestart {
+				klog.Infof("restarting modbus connection...")
+				err := d.modbusRestart()
+				if err != nil {
+					klog.Warningf("failed to restart modbus %v", err)
+				}
+			}
+			time.Sleep(time.Millisecond * 100)
+		}
+	}
+}

succbone/succd/process.go

@@ -8,22 +8,17 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/simonvetter/modbus"
 	"k8s.io/klog"
 )
 
 // daemon is the main service of the succdaemon.
 type daemon struct {
+	modbusClient *modbus.ModbusClient
 	// adcPirani is the adc implementation returning the voltage of the Pfeiffer
 	// Pirani gauge.
 	adcPirani adc
 
-	gpioDiffusionPump gpio
-	gpioRoughingPump  gpio
-	gpioBtnPumpDown   gpio
-	gpioBtnVent       gpio
-	gpioBelowRough    gpio
-	gpioBelowHigh     gpio
-
 	load atomic.Int64
 
 	// mu guards the state below.
@@ -55,6 +50,13 @@ type daemonState struct {
 	pumpdown   momentaryOutput
 	aboveRough thresholdOutput
 	aboveHigh  thresholdOutput
+
+	tempDPBottom float32
+	tempDPTop    float32
+	tempDPInlet  float32
+
+	tempSEM     float32
+	humiditySEM float32
 }
 
 func (d *daemonState) vacuumStatus() (rough, high bool) {
@@ -162,33 +164,5 @@ func (d *daemon) processOnce(_ context.Context) error {
 		d.dpOn = false
 	}
 
-	// Update relay outputs.
-	for _, rel := range []struct {
-		name string
-		gpio gpio
-		// activeHigh means the relay is active high, ie. a true source will
-		// mean that NO/COM get connected, and a false source means that NC/COM
-		// get connected.
-		activeHigh bool
-		source     bool
-	}{
-		{"rp", d.gpioRoughingPump, false, d.rpOn},
-		{"dp", d.gpioDiffusionPump, true, d.dpOn},
-		{"pumpdown", d.gpioBtnPumpDown, true, d.pumpdown.output},
-		{"vent", d.gpioBtnVent, true, d.vent.output},
-		{"rough", d.gpioBelowRough, false, d.aboveRough.output},
-		{"high", d.gpioBelowHigh, false, d.aboveHigh.output},
-	} {
-		val := rel.source
-		if rel.activeHigh {
-			// Invert because the relays go through logical inversion (ie. a
-			// GPIO false is a relay trigger).
-			val = !val
-		}
-		if err := rel.gpio.set(val); err != nil {
-			return fmt.Errorf("when outputting %s: %w", rel.name, err)
-		}
-	}
-
 	return nil
 }