FEMS Backport (OpenEMS#2419)

* KACO PV-Inverter: calculate production energy manually (OpenEMS#857)

Sometimes data from SunSpec was not reliable (even after fixing SunSpec scalefactors).

* GoodWe 20/30 Chargers: avoid multiple HIGH Priority mobus tasks (OpenEMS#859)

Set goodwe charger channels by the abstract goodwe component to avoid multiple HIGH Priority mobus tasks for Goodwe20/30 Chargers.

TwoStringCharger is no longer a ModbusComponent

* UI: Utils, tests and translations

* fix subtractSafely of not checking for undefined
   * explicitely checking for undefined and null
   * add unittest for utils

* UI: Fix chart wrong scaling when firstSetupProtocol is too close to end of month

* Backport FEMS Backend

- Fix compatibility with and require Odoo 16 (separate Odoo module will be updated independently)
- Fix possible NPEs
- Fix starting of Websocket servers (avoid event race conditions)
- Add COMPONENT_IDs to backend components
- Fix Websocket handshake case-insensitive
- Improve general performance
- AVoid excessive logs
- Add GenericSystemLog to Metadata: record system execute and update
- Improve InfluxDB Aggregated data handling
- Allow multiple InfluxDB servers for different periods (defined by start-/enddate)
- InfluxDB: only allow Channel-Addresses in standardized format

* Websocket Api Controller: cleanup getEdgeRequest & JUnit tests

* Cleanup getEdgeRequest to use the static method to generate the metadata for the local edge
 * added tests for getEdgeRequest/getEdgesRequest  Edge

* AppCenter: add check for peakshaving to not be compatible with home

- added invert boolean for error messages
- add test for peakshaving & CheckHome
- update translations

* AppCenter: separated AppConfiguration into tasks

This change is a prerequisite for automatically adding channels to the persistence predictor for TimeOfUseTariff Apps.

 - separated AppConfiguration into tasks
 - added Builder for AppConfiguration
 - changed AppConfiguration to a record
 - moved AggregateTasks into separate folder
 - added simple(& automatical) order for tasks
 - add tests for all AggregateTasks
 - moved validation to AggregateTasks to validate their Task configuration
 - changed AppValidateWorker to a (OSGi-)Component
 - added OnlyIfThrowing interface

* Add ENTSO-E App

---------

Co-authored-by: Michael Grill <[email protected]>
Co-authored-by: Sebastian Asen <[email protected]>
Co-authored-by: Lukas Rieger <[email protected]>
Co-authored-by: Stefan Feilmeier <[email protected]>

Author: 4 people

.gitpod.Dockerfile

@@ -12,7 +12,7 @@ ENV TRIGGER_REBUILD 4
 RUN npm install -g @angular/cli 
 
 # Install odoo
-ENV ODOO_VERSION 15.0
+ENV ODOO_VERSION 16.0
 ENV ODOO_RELEASE latest
 RUN curl -o odoo.deb -sSL http://nightly.odoo.com/${ODOO_VERSION}/nightly/deb/odoo_${ODOO_VERSION}.${ODOO_RELEASE}_all.deb \
     && sudo apt-get update \

.gitpod.yml

@@ -26,9 +26,9 @@ tasks:
       cd /workspace/odoo
       mkdir -p addons-available addons-enabled
       cd addons-available
-      git clone --depth=1 -b 15.0 https://github.com/OCA/partner-contact
-      git clone --depth=1 -b 15.0 https://github.com/OCA/web.git
-      git clone --depth=1 -b 15.0 https://github.com/OpenEMS/odoo-openems.git
+      git clone --depth=1 -b 16.0 https://github.com/OCA/partner-contact
+      git clone --depth=1 -b 16.0 https://github.com/OCA/web.git
+      git clone --depth=1 -b 16.0 https://github.com/OpenEMS/odoo-openems.git
       cd ../addons-enabled
       ln -s ../addons-available/partner-contact/partner_firstname
       ln -s ../addons-available/web/web_m2x_options

io.openems.backend.alerting/src/io/openems/backend/alerting/handler/OfflineEdgeHandler.java

@@ -194,7 +194,8 @@ protected void tryRemoveEdge(Edge edge) {
 	protected void tryAddEdge(Edge edge) {
 		if (this.isValidEdge(edge)) {
 			var msg = this.getEdgeMessage(edge);
-			if (msg != null) {
+			var msgScheduler = this.msgScheduler;
+			if (msg != null && msgScheduler != null) {
 				this.msgScheduler.schedule(msg);
 			}
 		}

io.openems.backend.b2bwebsocket/src/io/openems/backend/b2bwebsocket/Backend2BackendWebsocket.java

@@ -24,7 +24,6 @@
 import io.openems.backend.common.metadata.Metadata;
 import io.openems.backend.common.timedata.TimedataManager;
 import io.openems.common.utils.ThreadPoolUtils;
-import io.openems.common.websocket.AbstractWebsocketServer.DebugMode;
 
 @Designate(ocd = Config.class, factory = true)
 @Component(//
@@ -37,6 +36,8 @@
 })
 public class Backend2BackendWebsocket extends AbstractOpenemsBackendComponent implements EventHandler {
 
+	private static final String COMPONENT_ID = "b2bwebsocket0";
+
 	public static final int DEFAULT_PORT = 8076;
 
 	protected final ScheduledExecutorService executor = Executors.newScheduledThreadPool(10,
@@ -64,6 +65,10 @@ public Backend2BackendWebsocket() {
 	@Activate
 	private void activate(Config config) {
 		this.config = config;
+
+		if (this.metadata.isInitialized()) {
+			this.startServer();
+		}
 	}
 
 	@Deactivate
@@ -74,14 +79,13 @@ private void deactivate() {
 
 	/**
 	 * Create and start new server.
-	 *
-	 * @param port      the port
-	 * @param poolSize  number of threads dedicated to handle the tasks
-	 * @param debugMode activate a regular debug log about the state of the tasks
 	 */
-	private synchronized void startServer(int port, int poolSize, DebugMode debugMode) {
-		this.server = new WebsocketServer(this, this.getName(), port, poolSize, debugMode);
-		this.server.start();
+	private synchronized void startServer() {
+		if (this.server == null) {
+			this.server = new WebsocketServer(this, this.getName(), this.config.port(), this.config.poolSize(),
+					this.config.debugMode());
+			this.server.start();
+		}
 	}
 
 	/**
@@ -112,8 +116,13 @@ protected void logError(Logger log, String message) {
 	public void handleEvent(Event event) {
 		switch (event.getTopic()) {
 		case Metadata.Events.AFTER_IS_INITIALIZED:
-			this.startServer(this.config.port(), this.config.poolSize(), this.config.debugMode());
+			this.startServer();
 			break;
 		}
 	}
+
+	public String getId() {
+		return COMPONENT_ID;
+	}
+
 }

io.openems.backend.b2bwebsocket/src/io/openems/backend/b2bwebsocket/OnOpen.java

@@ -27,7 +27,7 @@ public void run(WebSocket ws, JsonObject handshake) throws OpenemsNamedException
 		try {
 			// Read "Authorization" header for Simple HTTP authentication. Source:
 			// https://stackoverflow.com/questions/16000517/how-to-get-password-from-http-basic-authentication
-			final var authorization = JsonUtils.getAsString(handshake, "Authorization");
+			final var authorization = JsonUtils.getAsString(handshake, "authorization");
 			if (authorization == null || !authorization.toLowerCase().startsWith("basic")) {
 				throw OpenemsError.COMMON_AUTHENTICATION_FAILED.exception();
 			}

io.openems.backend.b2bwebsocket/src/io/openems/backend/b2bwebsocket/WebsocketServer.java

@@ -27,7 +27,7 @@ public WebsocketServer(Backend2BackendWebsocket parent, String name, int port, i
 			var data = TreeBasedTable.<Long, String, JsonElement>create();
 			var now = Instant.now().toEpochMilli();
 			ThreadPoolUtils.debugMetrics(executor).forEach((key, value) -> {
-				data.put(now, "b2bwebsocket/" + key, new JsonPrimitive(value));
+				data.put(now, parent.getId() + "/" + key, new JsonPrimitive(value));
 			});
 			parent.timedataManager.write("backend0", new TimestampedDataNotification(data));
 		});

io.openems.backend.common/src/io/openems/backend/common/metadata/Edge.java

@@ -1,9 +1,10 @@
 package io.openems.backend.common.metadata;
 
-import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
+import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
 
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -23,9 +24,9 @@ public class Edge {
 
 	private final String id;
 	private String comment;
-	private SemanticVersion version;
-	private String producttype;
-	private ZonedDateTime lastmessage;
+	private final AtomicReference<SemanticVersion> version = new AtomicReference<>(SemanticVersion.ZERO);
+	private final AtomicReference<String> producttype = new AtomicReference<>("");
+	private final AtomicReference<ZonedDateTime> lastmessage = new AtomicReference<>(null);
 	private boolean isOnline = false;
 
 	private final List<EdgeUser> user;
@@ -34,12 +35,14 @@ public Edge(Metadata parent, String id, String comment, String version, String p
 			ZonedDateTime lastmessage) {
 		this.id = id;
 		this.comment = comment;
-		this.version = SemanticVersion.fromStringOrZero(version);
-		this.producttype = producttype;
-		this.lastmessage = lastmessage;
 
 		this.parent = parent;
 		this.user = new ArrayList<>();
+
+		// Avoid initial events
+		this.setProducttype(producttype, false);
+		this.setVersion(SemanticVersion.fromStringOrZero(version), false);
+		this.setLastmessage(lastmessage, false);
 	}
 
 	public String getId() {
@@ -66,10 +69,10 @@ public JsonObject toJsonObject() {
 		return JsonUtils.buildJsonObject() //
 				.addProperty("id", this.id) //
 				.addProperty("comment", this.comment) //
-				.addProperty("version", this.version.toString()) //
-				.addProperty("producttype", this.producttype) //
+				.addProperty("version", this.version.get().toString()) //
+				.addProperty("producttype", this.producttype.get()) //
 				.addProperty("online", this.isOnline) //
-				.addPropertyIfNotNull("lastmessage", this.lastmessage) //
+				.addPropertyIfNotNull("lastmessage", this.lastmessage.get()) //
 				.build();
 	}
 
@@ -110,36 +113,34 @@ public synchronized void setOnline(boolean isOnline) {
 	}
 
 	/**
-	 * Sets the Last-Message-Timestamp to now() and calls the
-	 * setLastmessage-Listeners.
+	 * Sets the Last-Message-Timestamp to now() (truncated to Minutes) and emits a
+	 * ON_SET_LASTMESSAGE event; but only max one event per Minute.
 	 */
-	public synchronized void setLastmessage() {
-		this.setLastmessage(ZonedDateTime.now(ZoneOffset.UTC));
+	public void setLastmessage() {
+		this.setLastmessage(ZonedDateTime.now());
 	}
 
 	/**
-	 * Sets the Last-Message-Timestamp and calls the setLastmessage-Listeners.
+	 * Sets the Last-Message-Timestamp (truncated to Minutes) and emits a
+	 * ON_SET_LASTMESSAGE event; but only max one event per Minute.
 	 * 
 	 * @param timestamp the Last-Message-Timestamp
 	 */
-	public synchronized void setLastmessage(ZonedDateTime timestamp) {
+	public void setLastmessage(ZonedDateTime timestamp) {
 		this.setLastmessage(timestamp, true);
 	}
 
-	/**
-	 * Sets the Last-Message-Timestamp.
-	 *
-	 * @param timestamp     the Last-Message-Timestamp
-	 * @param callListeners whether to call the setLastmessage-Listeners
-	 */
-	public synchronized void setLastmessage(ZonedDateTime timestamp, boolean callListeners) {
-		if (callListeners) {
+	private void setLastmessage(ZonedDateTime timestamp, boolean emitEvent) {
+		if (timestamp == null) {
+			return;
+		}
+		timestamp = timestamp.truncatedTo(ChronoUnit.MINUTES);
+		var previousTimestamp = this.lastmessage.getAndSet(timestamp);
+		if (emitEvent && (previousTimestamp == null || previousTimestamp.isBefore(timestamp))) {
 			EventBuilder.from(this.parent.getEventAdmin(), Events.ON_SET_LASTMESSAGE) //
 					.addArg(Events.OnSetLastmessage.EDGE, this) //
-					.send(); //
+					.send();
 		}
-		var now = timestamp;
-		this.lastmessage = now;
 	}
 
 	/**
@@ -148,80 +149,69 @@ public synchronized void setLastmessage(ZonedDateTime timestamp, boolean callLis
 	 * @return Last-Message-Timestamp in UTC Timezone
 	 */
 	public ZonedDateTime getLastmessage() {
-		return this.lastmessage;
+		return this.lastmessage.get();
 	}
 
 	/*
 	 * Version
 	 */
 	public SemanticVersion getVersion() {
-		return this.version;
+		return this.version.get();
 	}
 
 	/**
-	 * Sets the version and calls the SetVersion-Listeners.
+	 * Sets the version and emits a ON_SET_VERSION event.
 	 *
 	 * @param version the version
 	 */
-	public synchronized void setVersion(SemanticVersion version) {
+	public void setVersion(SemanticVersion version) {
 		this.setVersion(version, true);
 	}
 
-	/**
-	 * Sets the version.
-	 *
-	 * @param version       the version
-	 * @param callListeners whether to call the SetVersion-Listeners
-	 */
-	public synchronized void setVersion(SemanticVersion version, boolean callListeners) {
-		if (!Objects.equal(this.version, version)) { // on change
-			if (callListeners) {
-				this.log.info(
-						"Edge [" + this.getId() + "]: Update version from [" + this.version + "] to [" + version + "]");
-
-				EventBuilder.from(this.parent.getEventAdmin(), Events.ON_SET_VERSION) //
-						.addArg(Events.OnSetVersion.EDGE, this) //
-						.addArg(Events.OnSetVersion.VERSION, version) //
-						.send(); //
-			}
-			this.version = version;
+	private void setVersion(SemanticVersion version, boolean emitEvent) {
+		if (version == null) {
+			version = SemanticVersion.ZERO;
+		}
+		var oldVersion = this.version.getAndSet(version);
+		if (emitEvent && !Objects.equal(oldVersion, version)) { // on change
+			this.log.info("Edge [" + this.getId() + "]: Update version from [" + oldVersion + "] to [" + version + "]");
+
+			EventBuilder.from(this.parent.getEventAdmin(), Events.ON_SET_VERSION) //
+					.addArg(Events.OnSetVersion.EDGE, this) //
+					.addArg(Events.OnSetVersion.VERSION, version) //
+					.send();
 		}
 	}
 
 	/*
 	 * Producttype
 	 */
 	public String getProducttype() {
-		return this.producttype;
+		return this.producttype.get();
 	}
 
 	/**
-	 * Sets the Producttype and calls the SetProducttype-Listeners.
+	 * Sets the Producttype and emits a ON_SET_PRODUCTTYPE event.
 	 *
 	 * @param producttype the Producttype
 	 */
-	public synchronized void setProducttype(String producttype) {
+	public void setProducttype(String producttype) {
 		this.setProducttype(producttype, true);
 	}
 
-	/**
-	 * Sets the Producttype.
-	 *
-	 * @param producttype   the Producttype
-	 * @param callListeners whether to call the SetProducttype-Listeners
-	 */
-	public synchronized void setProducttype(String producttype, boolean callListeners) {
-		if (!Objects.equal(this.producttype, producttype)) { // on change
-			if (callListeners) {
-				this.log.info("Edge [" + this.getId() + "]: Update Product-Type to [" + producttype + "]. It was ["
-						+ this.producttype + "]");
-
-				EventBuilder.from(this.parent.getEventAdmin(), Events.ON_SET_PRODUCTTYPE) //
-						.addArg(Events.OnSetProducttype.EDGE, this) //
-						.addArg(Events.OnSetProducttype.PRODUCTTYPE, producttype) //
-						.send(); //
-			}
-			this.producttype = producttype;
+	private void setProducttype(String producttype, boolean emitEvent) {
+		if (producttype == null) {
+			producttype = "";
+		}
+		var oldProducttype = this.producttype.getAndSet(producttype);
+		if (emitEvent && !Objects.equal(oldProducttype, producttype)) { // on change
+			this.log.info("Edge [" + this.getId() + "]: Update Product-Type from [" + oldProducttype + "] to ["
+					+ producttype + "]");
+
+			EventBuilder.from(this.parent.getEventAdmin(), Events.ON_SET_PRODUCTTYPE) //
+					.addArg(Events.OnSetProducttype.EDGE, this) //
+					.addArg(Events.OnSetProducttype.PRODUCTTYPE, producttype) //
+					.send();
 		}
 	}
 

io.openems.backend.common/src/io/openems/backend/common/metadata/Metadata.java

@@ -365,4 +365,42 @@ public List<EdgeMetadata> getPageDevice(User user, PaginationOptions paginationO
 	 */
 	public EdgeMetadata getEdgeMetadataForUser(User user, String edgeId) throws OpenemsNamedException;
 
+	public interface GenericSystemLog {
+
+		/**
+		 * Gets the edgeId of the target log.
+		 * 
+		 * @return the edgeId
+		 */
+		public String edgeId();
+
+		/**
+		 * Gets the user which triggered the log.
+		 * 
+		 * @return the user
+		 */
+		public User user();
+
+		/**
+		 * Gets a short string which represents the whole log.
+		 * 
+		 * @return the teaser string
+		 */
+		public String teaser();
+
+		/**
+		 * Gets a map of values of the log.
+		 * 
+		 * @return the map
+		 */
+		public Map<String, String> getValues();
+	}
+
+	/**
+	 * Handles a Systemlog-Message.
+	 * 
+	 * @param systemLog the log
+	 */
+	public void logGenericSystemLog(GenericSystemLog systemLog);
+
 }