From 4edabc5c566fd76ce0cdbdc3a552235af23f6f5f Mon Sep 17 00:00:00 2001 From: hmelder Date: Sun, 10 Nov 2024 01:22:18 +0100 Subject: [PATCH 01/13] Add temperature and humidity stuff --- succbone/succd/http.go | 16 ++++++++++++++++ succbone/succd/index.html | 29 ++++++++++++++++++++++++++++- succbone/succd/main.go | 6 ++++++ succbone/succd/process.go | 7 +++++++ 4 files changed, 57 insertions(+), 1 deletion(-) diff --git a/succbone/succd/http.go b/succbone/succd/http.go index f1503e7..9a43281 100644 --- a/succbone/succd/http.go +++ b/succbone/succd/http.go @@ -62,6 +62,17 @@ type apiData struct { // DPOn means the diffusion pump is turned on. DPOn bool } + // Temperature state. + Temperatures struct { + DPBottom float32 + DPTop float32 + DPInlet float32 + SEM float32 + } + // Humidity state. + Humidity struct { + SEM float32 + } // Pressure feedback into evacuation board. Feedback struct { // RoughReached is true when the system has reached a rough vacuum @@ -115,6 +126,11 @@ func (s *webServer) apiData(skipSystem bool) *apiData { ad.Pirani.MbarFloat = mbar ad.Pumps.RPOn = state.rpOn ad.Pumps.DPOn = state.dpOn + ad.Temperatures.DPBottom = state.tempDPBottom + ad.Temperatures.DPTop = state.tempDPTop + ad.Temperatures.DPInlet = state.tempDPInlet + ad.Temperatures.SEM = state.tempSEM + ad.Humidity.SEM = state.humiditySEM ad.Feedback.RoughReached = rough ad.Feedback.HighReached = high ad.LoopLoad = s.d.loopLoad() diff --git a/succbone/succd/index.html b/succbone/succd/index.html index abd5695..c598f0c 100644 --- a/succbone/succd/index.html +++ b/succbone/succd/index.html @@ -111,6 +111,22 @@ td > span { DP {{ if .Pumps.DPOn }}ON{{ else }}OFF{{ end }} + + Temperatures + SEM + {{ .Temperatures.SEM }} + DPBottom + {{ .Temperatures.DPBottom }} + DPTop + {{ .Temperatures.DPTop }} + DPInlet + {{ .Temperatures.DPInlet }} + + + Humidity + SEM + {{ .Humidity.SEM }} + Safety Pirani
Failsafe @@ -316,6 +332,11 @@ window.addEventListener("load", (_) => { let trough = document.querySelector("#trough"); let thigh = document.querySelector("#thigh"); let load = document.querySelector("#load"); + let tempSEM = document.querySelector("#temp-sem"); + let tempDPBottom = document.querySelector("#temp-dp-bottom"); + let tempDPTop = document.querySelector("#temp-dp-top"); + let tempDPInlet = document.querySelector("#temp-dp-inlet"); + let humiditySEM = document.querySelector('#humidity-sem'); // Buttons let pd = document.querySelector("#pd"); @@ -388,6 +409,12 @@ window.addEventListener("load", (_) => { dp.style = colors.default; } + tempSEM.innerHTML = data.Temperatures.SEM + " °C"; + tempDPBottom.innerHTML = data.Temperatures.DPBottom + " °C"; + tempDPTop.innerHTML = data.Temperatures.DPTop + " °C"; + tempDPInlet.innerHTML = data.Temperatures.DPInlet + " °C"; + humiditySEM.innerHTML = data.Humidity.SEM + " %"; + let t = []; if (data.Feedback.RoughReached) { trough.innerHTML = "OK"; @@ -413,7 +440,7 @@ window.addEventListener("load", (_) => { // Indicate all process values as unknown - [failsafe, highpressure, rp, dp, trough, thigh, volts, mbar].forEach((el) => { + [failsafe, highpressure, rp, dp, trough, thigh, volts, mbar, tempDPBottom, tempDPTop, tempDPInlet].forEach((el) => { if (!el.innerHTML.includes("??")) { el.innerHTML += "??"; } diff --git a/succbone/succd/main.go b/succbone/succd/main.go index 64663a8..a56f2cc 100644 --- a/succbone/succd/main.go +++ b/succbone/succd/main.go @@ -35,6 +35,12 @@ func main() { d.daemonState.piraniVolts3.limit = 3 d.daemonState.piraniVolts100.limit = 100 + d.tempDPBottom = 420.6 + d.tempDPTop = 69.0 + d.tempDPInlet = 42.0 + d.tempSEM = 42.5 + d.humiditySEM = 66.6 + d.aboveRough.threshold = float64(flagPressureThresholdRough) d.aboveRough.hysteresis = float64(flagPressureThresholdRoughHysteresis) d.aboveHigh.threshold = float64(flagPressureThresholdHigh) diff --git a/succbone/succd/process.go b/succbone/succd/process.go index de99e84..c886260 100644 --- a/succbone/succd/process.go +++ b/succbone/succd/process.go @@ -55,6 +55,13 @@ type daemonState struct { pumpdown momentaryOutput aboveRough thresholdOutput aboveHigh thresholdOutput + + tempDPBottom float32 + tempDPTop float32 + tempDPInlet float32 + + tempSEM float32 + humiditySEM float32 } func (d *daemonState) vacuumStatus() (rough, high bool) { -- 2.45.2 From 2e6a3be1004044561064b0549ee5cae6bd36980e Mon Sep 17 00:00:00 2001 From: hmelder Date: Sun, 10 Nov 2024 01:47:01 +0100 Subject: [PATCH 02/13] Add modbus integration --- succbone/succd/go.mod | 3 ++ succbone/succd/go.sum | 4 ++ succbone/succd/main.go | 9 ++++ succbone/succd/modbus.go | 101 ++++++++++++++++++++++++++++++++++++++ succbone/succd/process.go | 2 + 5 files changed, 119 insertions(+) create mode 100644 succbone/succd/modbus.go diff --git a/succbone/succd/go.mod b/succbone/succd/go.mod index 574985d..d35e107 100644 --- a/succbone/succd/go.mod +++ b/succbone/succd/go.mod @@ -4,5 +4,8 @@ go 1.22.3 require ( github.com/coder/websocket v1.8.12 + github.com/simonvetter/modbus v1.6.3 k8s.io/klog v1.0.0 ) + +require github.com/goburrow/serial v0.1.0 // indirect diff --git a/succbone/succd/go.sum b/succbone/succd/go.sum index c8b5195..ab9ca54 100644 --- a/succbone/succd/go.sum +++ b/succbone/succd/go.sum @@ -1,5 +1,9 @@ github.com/coder/websocket v1.8.12 h1:5bUXkEPPIbewrnkU8LTCLVaxi4N4J8ahufH2vlo4NAo= github.com/coder/websocket v1.8.12/go.mod h1:LNVeNrXQZfe5qhS9ALED3uA+l5pPqvwXg3CKoDBB2gs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/goburrow/serial v0.1.0 h1:v2T1SQa/dlUqQiYIT8+Cu7YolfqAi3K96UmhwYyuSrA= +github.com/goburrow/serial v0.1.0/go.mod h1:sAiqG0nRVswsm1C97xsttiYCzSLBmUZ/VSlVLZJ8haA= +github.com/simonvetter/modbus v1.6.3 h1:kDzwVfIPczsM4Iz09il/Dij/bqlT4XiJVa0GYaOVA9w= +github.com/simonvetter/modbus v1.6.3/go.mod h1:hh90ZaTaPLcK2REj6/fpTbiV0J6S7GWmd8q+GVRObPw= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= diff --git a/succbone/succd/main.go b/succbone/succd/main.go index a56f2cc..cfd93a9 100644 --- a/succbone/succd/main.go +++ b/succbone/succd/main.go @@ -61,6 +61,11 @@ func main() { } d.adcPirani = adc + err = d.modbusConnect() + if err != nil { + klog.Exitf("Failed to connect to modbus %v", err) + } + for _, c := range []struct { out *gpio num int @@ -92,5 +97,9 @@ func main() { }() go d.process(ctx) + if !flagFake { + go d.modbusProcess(ctx) + } + <-ctx.Done() } diff --git a/succbone/succd/modbus.go b/succbone/succd/modbus.go new file mode 100644 index 0000000..15c3697 --- /dev/null +++ b/succbone/succd/modbus.go @@ -0,0 +1,101 @@ +package main + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/simonvetter/modbus" +) + +func modbusValuesToFloat(v uint16) float32 { + return float32(v) / 10.0 +} + +func (d *daemon) modbusConnect() error { + var err error + + d.mu.Lock() + defer d.mu.Unlock() + + // Setup modbus client + d.modbusClient, err = modbus.NewClient(&modbus.ClientConfiguration{ + URL: "tcp://10.250.241.20:8887", + Timeout: 5 * time.Second, + }) + if err != nil { + return err + } + // Connect to modbus client + err = d.modbusClient.Open() + if err != nil { + return err + } + + return nil +} + +// There are currently two devices connected to the modbus. +// The first one (slave 1) is a temperature/humidity sensor. +// The second one (slave 2) is a PTA8D08 transmitter +func (d *daemon) modbusUpdate() error { + d.mu.Lock() + defer d.mu.Unlock() + + var err error + var registers []uint16 // temperature, humidity + + // Switch to slave 1 + d.modbusClient.SetUnitId(1) + + // Read temperature and humidity + registers, err = d.modbusClient.ReadRegisters(1, 2, modbus.INPUT_REGISTER) + if err != nil { + return err + } + + if len(registers) != 2 { + msg := fmt.Sprintf("Expected two registers from modbus slave 1, but got %d", len(registers)) + return errors.New(msg) + } + + d.daemonState.tempSEM = modbusValuesToFloat(registers[0]) + d.daemonState.humiditySEM = modbusValuesToFloat(registers[1]) + + // Switch to slave 2 + d.modbusClient.SetUnitId(2) + + // PT100 mapping + // Channel 0: Cable -WGA6, Sensor "dp bottom" + // Channel 1: Cable -WGA8, Sensor "dp inlet" + // Channel 2: Cable WGA7, Sensor "dp top" + registers, err = d.modbusClient.ReadRegisters(0, 3, modbus.HOLDING_REGISTER) + if err != nil { + return err + } + + if len(registers) != 3 { + msg := fmt.Sprintf("Expected three registers from modbus slave 2, but got %d", len(registers)) + return errors.New(msg) + } + + d.daemonState.tempDPBottom = modbusValuesToFloat(registers[0]) + d.daemonState.tempDPInlet = modbusValuesToFloat(registers[1]) + d.daemonState.tempDPTop = modbusValuesToFloat(registers[2]) + + return nil +} + +// Call modbusUpdate every second +func (d *daemon) modbusProcess(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + default: + d.modbusUpdate() + time.Sleep(time.Second * 1) + } + } +} diff --git a/succbone/succd/process.go b/succbone/succd/process.go index c886260..7ade055 100644 --- a/succbone/succd/process.go +++ b/succbone/succd/process.go @@ -8,11 +8,13 @@ import ( "sync/atomic" "time" + "github.com/simonvetter/modbus" "k8s.io/klog" ) // daemon is the main service of the succdaemon. type daemon struct { + modbusClient *modbus.ModbusClient // adcPirani is the adc implementation returning the voltage of the Pfeiffer // Pirani gauge. adcPirani adc -- 2.45.2 From c02d24414faf19cc5b67fcbd619afa02dba801ea Mon Sep 17 00:00:00 2001 From: Rahix Date: Sun, 10 Nov 2024 01:38:32 +0100 Subject: [PATCH 03/13] succd: Improve styling of temperature values HMI design goes brrrr... --- succbone/succd/index.html | 73 +++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/succbone/succd/index.html b/succbone/succd/index.html index c598f0c..d700e1e 100644 --- a/succbone/succd/index.html +++ b/succbone/succd/index.html @@ -9,19 +9,19 @@ body { padding: 2em; } table { - font-size: 40px; + font-size: 30px; } table.status td { width: 2em; } th, td { background-color: #e8e8e8; - padding: 0.4em; + padding: 0.3em; } th { font-weight: 100; text-align: right; - font-size: 30px; + font-size: 25px; } td { text-align: left; @@ -34,7 +34,7 @@ h2 { font-weight: 100; } button { - height: 4.5em; + height: 3.3em; padding-left: 1.5em; padding-right: 1.5em; } @@ -76,6 +76,7 @@ td > span { .has-hidden:hover .hidden-text { display: block; +} @media only screen and (max-width: 700px) { body { @@ -111,22 +112,6 @@ td > span { DP {{ if .Pumps.DPOn }}ON{{ else }}OFF{{ end }} - - Temperatures - SEM - {{ .Temperatures.SEM }} - DPBottom - {{ .Temperatures.DPBottom }} - DPTop - {{ .Temperatures.DPTop }} - DPInlet - {{ .Temperatures.DPInlet }} - - - Humidity - SEM - {{ .Humidity.SEM }} - Safety Pirani
Failsafe @@ -136,19 +121,6 @@ td > span { - - - - - -
Pirani Pressure -
{{ .Pirani.Mbar }}
-
- Voltage: {{ .Pirani.Volts }} -
-
- - @@ -179,6 +151,41 @@ td > span {
Control
+ + + + + + + + + + + + + + + + + + + + + + + + + + + +
Pirani Pressure +
{{ .Pirani.Mbar }}
+
+ Voltage: {{ .Pirani.Volts }} +
+
TemperaturesDP Bottom{{ .Temperatures.DPBottom }}
DP Top{{ .Temperatures.DPTop }}
DP Inlet{{ .Temperatures.DPInlet }}
SEM Environment{{ .Temperatures.SEM }}
HumiditySEM Environment{{ .Humidity.SEM }}
+ +
-- 2.45.2 From 4ac1b5eb32a7ccdc4a40df8be7aeebb1eccca8b5 Mon Sep 17 00:00:00 2001 From: Rahix Date: Sun, 10 Nov 2024 01:46:29 +0100 Subject: [PATCH 04/13] succd: Add highlight colors for temperatures Highlight out-of-range temperatures for the user. The limits are currently by intuition and should be reconsidered. --- succbone/succd/index.html | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/succbone/succd/index.html b/succbone/succd/index.html index d700e1e..def9ac9 100644 --- a/succbone/succd/index.html +++ b/succbone/succd/index.html @@ -417,10 +417,28 @@ window.addEventListener("load", (_) => { } tempSEM.innerHTML = data.Temperatures.SEM + " °C"; - tempDPBottom.innerHTML = data.Temperatures.DPBottom + " °C"; - tempDPTop.innerHTML = data.Temperatures.DPTop + " °C"; - tempDPInlet.innerHTML = data.Temperatures.DPInlet + " °C"; + tempSEM.style = (data.Temperatures.SEM > 30) ? + colors.highlightCaution : colors.default; humiditySEM.innerHTML = data.Humidity.SEM + " %"; + humiditySEM.style = (data.Humidity.SEM > 59) ? + colors.highlightCaution : colors.default; + + tempDPTop.innerHTML = data.Temperatures.DPTop + " °C"; + tempDPTop.style = (data.Temperatures.DPTop > 30) ? + colors.highlightCaution : colors.default; + + tempDPInlet.innerHTML = data.Temperatures.DPInlet + " °C"; + tempDPInlet.style = (data.Temperatures.DPInlet > 30) ? + colors.highlightCaution : colors.default; + + tempDPBottom.innerHTML = data.Temperatures.DPBottom + " °C"; + if (data.Temperatures.DPBottom > 200) { + tempDPBottom.style = colors.highlightFault; + } else if (data.Temperatures.DPBottom > 50) { + tempDPBottom.style = colors.highlightNeutral; + } else { + tempDPBottom.style = colors.default; + } let t = []; if (data.Feedback.RoughReached) { -- 2.45.2 From edb905170855c05cf36e25b315bd39f072c583ab Mon Sep 17 00:00:00 2001 From: Rahix Date: Sun, 10 Nov 2024 02:04:31 +0100 Subject: [PATCH 05/13] succd: Export temperature values to prometheus Also add metrics for all the temperature and humidity measurements. --- succbone/succd/http.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/succbone/succd/http.go b/succbone/succd/http.go index 9a43281..01eab8e 100644 --- a/succbone/succd/http.go +++ b/succbone/succd/http.go @@ -233,6 +233,26 @@ func (s *webServer) viewMetrics(w http.ResponseWriter, r *http.Request) { fmt.Fprintf(w, "# HELP sem_vacuum_high_reached Whether a high vacuum has been reached (boolean)\n") fmt.Fprintf(w, "# TYPE sem_vacuum_high_reached gauge\n") fmt.Fprintf(w, "sem_vacuum_high_reached %f\n", boolToFloat(high)) + + fmt.Fprintf(w, "# HELP sem_environment_temperature_celsius Environmental temperature of the SEM, in degrees celsius\n") + fmt.Fprintf(w, "# TYPE sem_environment_temperature_celsius gauge\n") + fmt.Fprintf(w, "sem_environment_temperature_celsius %f\n", state.tempSEM) + + fmt.Fprintf(w, "# HELP sem_environment_humidity_percent Environmental relative humidity of the SEM, in percent\n") + fmt.Fprintf(w, "# TYPE sem_environment_humidity_percent gauge\n") + fmt.Fprintf(w, "sem_environment_humidity_percent %f\n", state.humiditySEM) + + fmt.Fprintf(w, "# HELP sem_dp_bottom_temperature_celsius Temperature of the DP bottom, in degrees celsius\n") + fmt.Fprintf(w, "# TYPE sem_dp_bottom_temperature_celsius gauge\n") + fmt.Fprintf(w, "sem_dp_bottom_temperature_celsius %f\n", state.tempDPBottom) + + fmt.Fprintf(w, "# HELP sem_dp_top_temperature_celsius Temperature of the DP top, in degrees celsius\n") + fmt.Fprintf(w, "# TYPE sem_dp_top_temperature_celsius gauge\n") + fmt.Fprintf(w, "sem_dp_top_temperature_celsius %f\n", state.tempDPTop) + + fmt.Fprintf(w, "# HELP sem_dp_inlet_temperature_celsius Temperature of the DP inlet flange, in degrees celsius\n") + fmt.Fprintf(w, "# TYPE sem_dp_inlet_temperature_celsius gauge\n") + fmt.Fprintf(w, "sem_dp_inlet_temperature_celsius %f\n", state.tempDPInlet) } func (s *webServer) viewRoughingPumpEnable(w http.ResponseWriter, r *http.Request) { -- 2.45.2 From 0e45650972720c1df41af010adcbd9fb5d909f64 Mon Sep 17 00:00:00 2001 From: Rahix Date: Sun, 10 Nov 2024 02:08:05 +0100 Subject: [PATCH 06/13] succd: Render temperature unit in template as well --- succbone/succd/index.html | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/succbone/succd/index.html b/succbone/succd/index.html index def9ac9..37f0849 100644 --- a/succbone/succd/index.html +++ b/succbone/succd/index.html @@ -164,24 +164,24 @@ td > span { Temperatures DP Bottom - {{ .Temperatures.DPBottom }} + {{ .Temperatures.DPBottom }} °C DP Top - {{ .Temperatures.DPTop }} + {{ .Temperatures.DPTop }} °C DP Inlet - {{ .Temperatures.DPInlet }} + {{ .Temperatures.DPInlet }} °C SEM Environment - {{ .Temperatures.SEM }} + {{ .Temperatures.SEM }} °C Humidity SEM Environment - {{ .Humidity.SEM }} + {{ .Humidity.SEM }}% @@ -419,7 +419,7 @@ window.addEventListener("load", (_) => { tempSEM.innerHTML = data.Temperatures.SEM + " °C"; tempSEM.style = (data.Temperatures.SEM > 30) ? colors.highlightCaution : colors.default; - humiditySEM.innerHTML = data.Humidity.SEM + " %"; + humiditySEM.innerHTML = data.Humidity.SEM + "%"; humiditySEM.style = (data.Humidity.SEM > 59) ? colors.highlightCaution : colors.default; -- 2.45.2 From 606d470577c8a9e69c95dc461e1095a905f44b79 Mon Sep 17 00:00:00 2001 From: hmelder Date: Sun, 10 Nov 2024 05:11:48 +0100 Subject: [PATCH 07/13] succd: Migrate to KEC1 MODBUS relay board --- succbone/succd/main.go | 23 -------------- succbone/succd/modbus.go | 66 ++++++++++++++++++++++++++++++++++++--- succbone/succd/process.go | 35 --------------------- 3 files changed, 61 insertions(+), 63 deletions(-) diff --git a/succbone/succd/main.go b/succbone/succd/main.go index cfd93a9..52a10db 100644 --- a/succbone/succd/main.go +++ b/succbone/succd/main.go @@ -48,12 +48,6 @@ func main() { if flagFake { klog.Infof("Starting with fake peripherals") d.adcPirani = &fakeADC{} - d.gpioRoughingPump = &fakeGPIO{desc: "rp"} - d.gpioDiffusionPump = &fakeGPIO{desc: "~dp"} - d.gpioBtnPumpDown = &fakeGPIO{desc: "~pd"} - d.gpioBtnVent = &fakeGPIO{desc: "~vent"} - d.gpioBelowRough = &fakeGPIO{desc: "~rough"} - d.gpioBelowHigh = &fakeGPIO{desc: "~high"} } else { adc, err := newBBADC(0) if err != nil { @@ -65,23 +59,6 @@ func main() { if err != nil { klog.Exitf("Failed to connect to modbus %v", err) } - - for _, c := range []struct { - out *gpio - num int - }{ - {&d.gpioRoughingPump, 115}, - {&d.gpioDiffusionPump, 49}, - {&d.gpioBtnPumpDown, 48}, - {&d.gpioBtnVent, 60}, - {&d.gpioBelowRough, 30}, - {&d.gpioBelowHigh, 7}, - } { - *c.out, err = newBBGPIO(c.num, true) - if err != nil { - klog.Exitf("Failed to setup GPIO: %v", err) - } - } } web := webServer{ diff --git a/succbone/succd/modbus.go b/succbone/succd/modbus.go index 15c3697..61f38de 100644 --- a/succbone/succd/modbus.go +++ b/succbone/succd/modbus.go @@ -63,7 +63,7 @@ func (d *daemon) modbusUpdate() error { d.daemonState.tempSEM = modbusValuesToFloat(registers[0]) d.daemonState.humiditySEM = modbusValuesToFloat(registers[1]) - // Switch to slave 2 + // Switch to slave 2 (KEC2) d.modbusClient.SetUnitId(2) // PT100 mapping @@ -76,18 +76,74 @@ func (d *daemon) modbusUpdate() error { } if len(registers) != 3 { - msg := fmt.Sprintf("Expected three registers from modbus slave 2, but got %d", len(registers)) - return errors.New(msg) + return fmt.Errorf("expected three registers from modbus slave 2, but got %d", len(registers)) } d.daemonState.tempDPBottom = modbusValuesToFloat(registers[0]) d.daemonState.tempDPInlet = modbusValuesToFloat(registers[1]) d.daemonState.tempDPTop = modbusValuesToFloat(registers[2]) + // Switch to slave 3 (KEC1) + d.modbusClient.SetUnitId(3) + + // Do a read first to avoid side-effects from the subsequent write to the relay states + var digitalInputs [8]bool + var digitalInputRegisters []uint16 + + digitalInputRegisters, err = d.modbusClient.ReadRegisters(0x81, 8, modbus.HOLDING_REGISTER) + if err != nil { + return err + } + // Convert MODBUS words into bools + for idx, value := range digitalInputRegisters { + if value != 0 { + digitalInputs[idx] = true + } else { + digitalInputs[idx] = false + } + } + // TODO: Input mapping goes here + + // KFA1-KFA8 + var relayState [8]bool + + // -KFA1 Roughing Pump + relayState[0] = d.daemonState.rpOn + // -KFA2 Diffusion Pump + relayState[1] = d.daemonState.dpOn + // -KFA4 Button Vent + relayState[3] = d.daemonState.vent.output + // -KFA5 Button Pump-Down + relayState[4] = d.daemonState.pumpdown.output + // -KFA6 Fake-Pirani Rough + relayState[5] = d.aboveRough.output + // -KFA7 Fake-Pirani High + relayState[6] = d.aboveHigh.output + + // The KEC1 module uses a non-standard MODBUS interface + // instead of coils + // 0x0100 is the open command + // 0x0200 is the close command + // We write 8 words (16-bit) to address 0x01 to update the relays + var registerValues [8]uint16 + // Convert the boolean values to the commands + for idx, state := range relayState { + if state { + registerValues[idx] = 0x0100 + } else { + registerValues[idx] = 0x0200 + } + } + + err = d.modbusClient.WriteRegisters(0x01, registerValues[:]) + if err != nil { + return err + } + return nil } -// Call modbusUpdate every second +// Call modbusUpdate every 100 milliseconds func (d *daemon) modbusProcess(ctx context.Context) { for { select { @@ -95,7 +151,7 @@ func (d *daemon) modbusProcess(ctx context.Context) { return default: d.modbusUpdate() - time.Sleep(time.Second * 1) + time.Sleep(time.Millisecond * 100) } } } diff --git a/succbone/succd/process.go b/succbone/succd/process.go index 7ade055..7504b19 100644 --- a/succbone/succd/process.go +++ b/succbone/succd/process.go @@ -19,13 +19,6 @@ type daemon struct { // Pirani gauge. adcPirani adc - gpioDiffusionPump gpio - gpioRoughingPump gpio - gpioBtnPumpDown gpio - gpioBtnVent gpio - gpioBelowRough gpio - gpioBelowHigh gpio - load atomic.Int64 // mu guards the state below. @@ -171,33 +164,5 @@ func (d *daemon) processOnce(_ context.Context) error { d.dpOn = false } - // Update relay outputs. - for _, rel := range []struct { - name string - gpio gpio - // activeHigh means the relay is active high, ie. a true source will - // mean that NO/COM get connected, and a false source means that NC/COM - // get connected. - activeHigh bool - source bool - }{ - {"rp", d.gpioRoughingPump, false, d.rpOn}, - {"dp", d.gpioDiffusionPump, true, d.dpOn}, - {"pumpdown", d.gpioBtnPumpDown, true, d.pumpdown.output}, - {"vent", d.gpioBtnVent, true, d.vent.output}, - {"rough", d.gpioBelowRough, false, d.aboveRough.output}, - {"high", d.gpioBelowHigh, false, d.aboveHigh.output}, - } { - val := rel.source - if rel.activeHigh { - // Invert because the relays go through logical inversion (ie. a - // GPIO false is a relay trigger). - val = !val - } - if err := rel.gpio.set(val); err != nil { - return fmt.Errorf("when outputting %s: %w", rel.name, err) - } - } - return nil } -- 2.45.2 From e7fd2dd7d769594c28359f09be4ad0495847c02a Mon Sep 17 00:00:00 2001 From: hmelder Date: Sun, 10 Nov 2024 05:20:24 +0100 Subject: [PATCH 08/13] succd: KFA{1,6,7} are normally closed --- succbone/succd/modbus.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/succbone/succd/modbus.go b/succbone/succd/modbus.go index 61f38de..ed33f7e 100644 --- a/succbone/succd/modbus.go +++ b/succbone/succd/modbus.go @@ -107,18 +107,18 @@ func (d *daemon) modbusUpdate() error { // KFA1-KFA8 var relayState [8]bool - // -KFA1 Roughing Pump - relayState[0] = d.daemonState.rpOn + // -KFA1 Roughing Pump (normally closed contact) + relayState[0] = !d.daemonState.rpOn // -KFA2 Diffusion Pump relayState[1] = d.daemonState.dpOn // -KFA4 Button Vent relayState[3] = d.daemonState.vent.output // -KFA5 Button Pump-Down relayState[4] = d.daemonState.pumpdown.output - // -KFA6 Fake-Pirani Rough - relayState[5] = d.aboveRough.output - // -KFA7 Fake-Pirani High - relayState[6] = d.aboveHigh.output + // -KFA6 Fake-Pirani Rough (normally closed contact) + relayState[5] = !d.aboveRough.output + // -KFA7 Fake-Pirani High (normally closed contact) + relayState[6] = !d.aboveHigh.output // The KEC1 module uses a non-standard MODBUS interface // instead of coils -- 2.45.2 From d8a467a0c451edd11a7a61212670bbca845b465c Mon Sep 17 00:00:00 2001 From: hmelder Date: Sun, 10 Nov 2024 05:45:32 +0100 Subject: [PATCH 09/13] succd: Split scope lock into multiple blocks We noticed huge load spikes with the latest changes. This was caused by the modbus goroutine blocking the entire daemon for long periods of time while doing its data transfer. Fix this by only holding the lock while performing data accesses. --- succbone/succd/modbus.go | 45 +++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/succbone/succd/modbus.go b/succbone/succd/modbus.go index ed33f7e..e5f3b91 100644 --- a/succbone/succd/modbus.go +++ b/succbone/succd/modbus.go @@ -40,28 +40,27 @@ func (d *daemon) modbusConnect() error { // The first one (slave 1) is a temperature/humidity sensor. // The second one (slave 2) is a PTA8D08 transmitter func (d *daemon) modbusUpdate() error { - d.mu.Lock() - defer d.mu.Unlock() - var err error - var registers []uint16 // temperature, humidity - // Switch to slave 1 + // Switch to slave 1 (BTA1) d.modbusClient.SetUnitId(1) // Read temperature and humidity - registers, err = d.modbusClient.ReadRegisters(1, 2, modbus.INPUT_REGISTER) + var registersBTA1 []uint16 // temperature, humidity + registersBTA1, err = d.modbusClient.ReadRegisters(1, 2, modbus.INPUT_REGISTER) if err != nil { return err } - if len(registers) != 2 { - msg := fmt.Sprintf("Expected two registers from modbus slave 1, but got %d", len(registers)) + if len(registersBTA1) != 2 { + msg := fmt.Sprintf("Expected two registers from modbus slave 1, but got %d", len(registersBTA1)) return errors.New(msg) } - d.daemonState.tempSEM = modbusValuesToFloat(registers[0]) - d.daemonState.humiditySEM = modbusValuesToFloat(registers[1]) + d.mu.Lock() + d.daemonState.tempSEM = modbusValuesToFloat(registersBTA1[0]) + d.daemonState.humiditySEM = modbusValuesToFloat(registersBTA1[1]) + d.mu.Unlock() // Switch to slave 2 (KEC2) d.modbusClient.SetUnitId(2) @@ -70,18 +69,21 @@ func (d *daemon) modbusUpdate() error { // Channel 0: Cable -WGA6, Sensor "dp bottom" // Channel 1: Cable -WGA8, Sensor "dp inlet" // Channel 2: Cable WGA7, Sensor "dp top" - registers, err = d.modbusClient.ReadRegisters(0, 3, modbus.HOLDING_REGISTER) + var registersKEC2 []uint16 // temperatures dp + registersKEC2, err = d.modbusClient.ReadRegisters(0, 3, modbus.HOLDING_REGISTER) if err != nil { return err } - if len(registers) != 3 { - return fmt.Errorf("expected three registers from modbus slave 2, but got %d", len(registers)) + if len(registersKEC2) != 3 { + return fmt.Errorf("expected three registers from modbus slave 2, but got %d", len(registersKEC2)) } - d.daemonState.tempDPBottom = modbusValuesToFloat(registers[0]) - d.daemonState.tempDPInlet = modbusValuesToFloat(registers[1]) - d.daemonState.tempDPTop = modbusValuesToFloat(registers[2]) + d.mu.Lock() + d.daemonState.tempDPBottom = modbusValuesToFloat(registersKEC2[0]) + d.daemonState.tempDPInlet = modbusValuesToFloat(registersKEC2[1]) + d.daemonState.tempDPTop = modbusValuesToFloat(registersKEC2[2]) + d.mu.Unlock() // Switch to slave 3 (KEC1) d.modbusClient.SetUnitId(3) @@ -106,7 +108,7 @@ func (d *daemon) modbusUpdate() error { // KFA1-KFA8 var relayState [8]bool - + d.mu.Lock() // -KFA1 Roughing Pump (normally closed contact) relayState[0] = !d.daemonState.rpOn // -KFA2 Diffusion Pump @@ -119,23 +121,24 @@ func (d *daemon) modbusUpdate() error { relayState[5] = !d.aboveRough.output // -KFA7 Fake-Pirani High (normally closed contact) relayState[6] = !d.aboveHigh.output + d.mu.Unlock() // The KEC1 module uses a non-standard MODBUS interface // instead of coils // 0x0100 is the open command // 0x0200 is the close command // We write 8 words (16-bit) to address 0x01 to update the relays - var registerValues [8]uint16 + var registerValuesKEC1 [8]uint16 // Convert the boolean values to the commands for idx, state := range relayState { if state { - registerValues[idx] = 0x0100 + registerValuesKEC1[idx] = 0x0100 } else { - registerValues[idx] = 0x0200 + registerValuesKEC1[idx] = 0x0200 } } - err = d.modbusClient.WriteRegisters(0x01, registerValues[:]) + err = d.modbusClient.WriteRegisters(0x01, registerValuesKEC1[:]) if err != nil { return err } -- 2.45.2 From 6f93b96c39d3f47cc4b781653b3d4c2b2f1bd56c Mon Sep 17 00:00:00 2001 From: hmelder Date: Sun, 10 Nov 2024 06:19:01 +0100 Subject: [PATCH 10/13] succd: do not early return on error in modbusUpdate When one device fails, this should not influence updates of the other devices. Thus, early return was the wrong strategy here. Instead, when communication with a device fails, skip the process data update and continue with the next device. --- succbone/succd/modbus.go | 67 ++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 37 deletions(-) diff --git a/succbone/succd/modbus.go b/succbone/succd/modbus.go index e5f3b91..a4d05e0 100644 --- a/succbone/succd/modbus.go +++ b/succbone/succd/modbus.go @@ -2,11 +2,10 @@ package main import ( "context" - "errors" - "fmt" "time" "github.com/simonvetter/modbus" + "k8s.io/klog" ) func modbusValuesToFloat(v uint16) float32 { @@ -39,7 +38,7 @@ func (d *daemon) modbusConnect() error { // There are currently two devices connected to the modbus. // The first one (slave 1) is a temperature/humidity sensor. // The second one (slave 2) is a PTA8D08 transmitter -func (d *daemon) modbusUpdate() error { +func (d *daemon) modbusUpdate() { var err error // Switch to slave 1 (BTA1) @@ -49,19 +48,16 @@ func (d *daemon) modbusUpdate() error { var registersBTA1 []uint16 // temperature, humidity registersBTA1, err = d.modbusClient.ReadRegisters(1, 2, modbus.INPUT_REGISTER) if err != nil { - return err + klog.Warningf("error while reading registers from BTA1 %v", err) + } else if len(registersBTA1) != 2 { + klog.Warningf("expected two registers from modbus slave 1, but got %d", len(registersBTA1)) + } else { + d.mu.Lock() + d.daemonState.tempSEM = modbusValuesToFloat(registersBTA1[0]) + d.daemonState.humiditySEM = modbusValuesToFloat(registersBTA1[1]) + d.mu.Unlock() } - if len(registersBTA1) != 2 { - msg := fmt.Sprintf("Expected two registers from modbus slave 1, but got %d", len(registersBTA1)) - return errors.New(msg) - } - - d.mu.Lock() - d.daemonState.tempSEM = modbusValuesToFloat(registersBTA1[0]) - d.daemonState.humiditySEM = modbusValuesToFloat(registersBTA1[1]) - d.mu.Unlock() - // Switch to slave 2 (KEC2) d.modbusClient.SetUnitId(2) @@ -72,19 +68,17 @@ func (d *daemon) modbusUpdate() error { var registersKEC2 []uint16 // temperatures dp registersKEC2, err = d.modbusClient.ReadRegisters(0, 3, modbus.HOLDING_REGISTER) if err != nil { - return err + klog.Warningf("error while reading registers from KEC2 %v", err) + } else if len(registersKEC2) != 3 { + klog.Warningf("expected three registers from modbus slave 2, but got %d", len(registersKEC2)) + } else { + d.mu.Lock() + d.daemonState.tempDPBottom = modbusValuesToFloat(registersKEC2[0]) + d.daemonState.tempDPInlet = modbusValuesToFloat(registersKEC2[1]) + d.daemonState.tempDPTop = modbusValuesToFloat(registersKEC2[2]) + d.mu.Unlock() } - if len(registersKEC2) != 3 { - return fmt.Errorf("expected three registers from modbus slave 2, but got %d", len(registersKEC2)) - } - - d.mu.Lock() - d.daemonState.tempDPBottom = modbusValuesToFloat(registersKEC2[0]) - d.daemonState.tempDPInlet = modbusValuesToFloat(registersKEC2[1]) - d.daemonState.tempDPTop = modbusValuesToFloat(registersKEC2[2]) - d.mu.Unlock() - // Switch to slave 3 (KEC1) d.modbusClient.SetUnitId(3) @@ -94,17 +88,18 @@ func (d *daemon) modbusUpdate() error { digitalInputRegisters, err = d.modbusClient.ReadRegisters(0x81, 8, modbus.HOLDING_REGISTER) if err != nil { - return err - } - // Convert MODBUS words into bools - for idx, value := range digitalInputRegisters { - if value != 0 { - digitalInputs[idx] = true - } else { - digitalInputs[idx] = false + klog.Warningf("error while reading digital inputs from KEC1 %v", err) + } else { + // Convert MODBUS words into bools + for idx, value := range digitalInputRegisters { + if value != 0 { + digitalInputs[idx] = true + } else { + digitalInputs[idx] = false + } } + // TODO: Input mapping goes here } - // TODO: Input mapping goes here // KFA1-KFA8 var relayState [8]bool @@ -140,10 +135,8 @@ func (d *daemon) modbusUpdate() error { err = d.modbusClient.WriteRegisters(0x01, registerValuesKEC1[:]) if err != nil { - return err + klog.Warningf("error while updating registers %v", err) } - - return nil } // Call modbusUpdate every 100 milliseconds -- 2.45.2 From 152290f5a3dd738978a91b9be6b0373efe9c716d Mon Sep 17 00:00:00 2001 From: Rahix Date: Sun, 10 Nov 2024 06:33:20 +0100 Subject: [PATCH 11/13] succd: Fix -KEC1 relay board updates The accesses to -KEC1 always time out on the attempt to update the output values. We noticed that this is related to the timing between the reading of the inputs and the following write to the outputs. Fix -KEC1 accesses by waiting before sending the next request to the board after receiving the reply for the previous one. --- succbone/succd/modbus.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/succbone/succd/modbus.go b/succbone/succd/modbus.go index a4d05e0..70fa22f 100644 --- a/succbone/succd/modbus.go +++ b/succbone/succd/modbus.go @@ -101,6 +101,11 @@ func (d *daemon) modbusUpdate() { // TODO: Input mapping goes here } + // We must wait between reading and writing to the -KEC1 relay board + // because otherwise it chokes and times out the write registers + // command. + time.Sleep(time.Millisecond * 10) + // KFA1-KFA8 var relayState [8]bool d.mu.Lock() -- 2.45.2 From 9ec580c26fbc15e276720ef0bd085827c617a42f Mon Sep 17 00:00:00 2001 From: hmelder Date: Sun, 10 Nov 2024 06:51:21 +0100 Subject: [PATCH 12/13] succd: implement auto-restarting of MODBUS connection in case of network loss --- succbone/succd/modbus.go | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/succbone/succd/modbus.go b/succbone/succd/modbus.go index 70fa22f..bc7681d 100644 --- a/succbone/succd/modbus.go +++ b/succbone/succd/modbus.go @@ -2,6 +2,7 @@ package main import ( "context" + "net" "time" "github.com/simonvetter/modbus" @@ -35,10 +36,17 @@ func (d *daemon) modbusConnect() error { return nil } +func (d *daemon) modbusRestart() error { + d.modbusClient.Close() + return d.modbusClient.Open() +} + // There are currently two devices connected to the modbus. // The first one (slave 1) is a temperature/humidity sensor. // The second one (slave 2) is a PTA8D08 transmitter -func (d *daemon) modbusUpdate() { +// +// Returns whether modbus should restart (only in case of an underlying network error) +func (d *daemon) modbusUpdate() bool { var err error // Switch to slave 1 (BTA1) @@ -48,6 +56,9 @@ func (d *daemon) modbusUpdate() { var registersBTA1 []uint16 // temperature, humidity registersBTA1, err = d.modbusClient.ReadRegisters(1, 2, modbus.INPUT_REGISTER) if err != nil { + if _, ok := err.(net.Error); ok { + return true + } klog.Warningf("error while reading registers from BTA1 %v", err) } else if len(registersBTA1) != 2 { klog.Warningf("expected two registers from modbus slave 1, but got %d", len(registersBTA1)) @@ -68,6 +79,9 @@ func (d *daemon) modbusUpdate() { var registersKEC2 []uint16 // temperatures dp registersKEC2, err = d.modbusClient.ReadRegisters(0, 3, modbus.HOLDING_REGISTER) if err != nil { + if _, ok := err.(net.Error); ok { + return true + } klog.Warningf("error while reading registers from KEC2 %v", err) } else if len(registersKEC2) != 3 { klog.Warningf("expected three registers from modbus slave 2, but got %d", len(registersKEC2)) @@ -88,6 +102,9 @@ func (d *daemon) modbusUpdate() { digitalInputRegisters, err = d.modbusClient.ReadRegisters(0x81, 8, modbus.HOLDING_REGISTER) if err != nil { + if _, ok := err.(net.Error); ok { + return true + } klog.Warningf("error while reading digital inputs from KEC1 %v", err) } else { // Convert MODBUS words into bools @@ -140,8 +157,13 @@ func (d *daemon) modbusUpdate() { err = d.modbusClient.WriteRegisters(0x01, registerValuesKEC1[:]) if err != nil { + if _, ok := err.(net.Error); ok { + return true + } klog.Warningf("error while updating registers %v", err) } + + return false } // Call modbusUpdate every 100 milliseconds @@ -151,7 +173,16 @@ func (d *daemon) modbusProcess(ctx context.Context) { case <-ctx.Done(): return default: - d.modbusUpdate() + shouldRestart := d.modbusUpdate() + // the modbus library does not reopen the tcp socket in case of + // a connection loss. + if shouldRestart { + klog.Infof("restarting modbus connection...") + err := d.modbusRestart() + if err != nil { + klog.Warningf("failed to restart modbus %v", err) + } + } time.Sleep(time.Millisecond * 100) } } -- 2.45.2 From 3b9c1ba912e765af07e0de8a63acf3db6750878f Mon Sep 17 00:00:00 2001 From: hmelder Date: Sun, 10 Nov 2024 07:06:56 +0100 Subject: [PATCH 13/13] succd: MODBUS library does not differentiate between TCP socket time outs and RTU time outs --- succbone/succd/modbus.go | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/succbone/succd/modbus.go b/succbone/succd/modbus.go index bc7681d..4aebd03 100644 --- a/succbone/succd/modbus.go +++ b/succbone/succd/modbus.go @@ -2,7 +2,6 @@ package main import ( "context" - "net" "time" "github.com/simonvetter/modbus" @@ -22,7 +21,7 @@ func (d *daemon) modbusConnect() error { // Setup modbus client d.modbusClient, err = modbus.NewClient(&modbus.ClientConfiguration{ URL: "tcp://10.250.241.20:8887", - Timeout: 5 * time.Second, + Timeout: 1 * time.Second, }) if err != nil { return err @@ -48,6 +47,7 @@ func (d *daemon) modbusRestart() error { // Returns whether modbus should restart (only in case of an underlying network error) func (d *daemon) modbusUpdate() bool { var err error + var numDevicesNotResponding int // Switch to slave 1 (BTA1) d.modbusClient.SetUnitId(1) @@ -56,9 +56,7 @@ func (d *daemon) modbusUpdate() bool { var registersBTA1 []uint16 // temperature, humidity registersBTA1, err = d.modbusClient.ReadRegisters(1, 2, modbus.INPUT_REGISTER) if err != nil { - if _, ok := err.(net.Error); ok { - return true - } + numDevicesNotResponding += 1 klog.Warningf("error while reading registers from BTA1 %v", err) } else if len(registersBTA1) != 2 { klog.Warningf("expected two registers from modbus slave 1, but got %d", len(registersBTA1)) @@ -79,9 +77,7 @@ func (d *daemon) modbusUpdate() bool { var registersKEC2 []uint16 // temperatures dp registersKEC2, err = d.modbusClient.ReadRegisters(0, 3, modbus.HOLDING_REGISTER) if err != nil { - if _, ok := err.(net.Error); ok { - return true - } + numDevicesNotResponding += 1 klog.Warningf("error while reading registers from KEC2 %v", err) } else if len(registersKEC2) != 3 { klog.Warningf("expected three registers from modbus slave 2, but got %d", len(registersKEC2)) @@ -102,9 +98,7 @@ func (d *daemon) modbusUpdate() bool { digitalInputRegisters, err = d.modbusClient.ReadRegisters(0x81, 8, modbus.HOLDING_REGISTER) if err != nil { - if _, ok := err.(net.Error); ok { - return true - } + numDevicesNotResponding += 1 klog.Warningf("error while reading digital inputs from KEC1 %v", err) } else { // Convert MODBUS words into bools @@ -157,12 +151,15 @@ func (d *daemon) modbusUpdate() bool { err = d.modbusClient.WriteRegisters(0x01, registerValuesKEC1[:]) if err != nil { - if _, ok := err.(net.Error); ok { - return true - } + numDevicesNotResponding += 1 klog.Warningf("error while updating registers %v", err) } + if numDevicesNotResponding >= 4 { + klog.Warningf("no device did respond to our request. Probably a network timeout.") + return true + } + return false } -- 2.45.2