ConversionTester.java
/**
* Copyright (c) 2017-2018, RTE (http://www.rte-france.com)
* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/.
* SPDX-License-Identifier: MPL-2.0
*/
package com.powsybl.cgmes.conversion.test;
import com.google.common.io.ByteStreams;
import com.google.common.jimfs.Configuration;
import com.google.common.jimfs.Jimfs;
import com.powsybl.cgmes.conversion.CgmesExport;
import com.powsybl.cgmes.conversion.CgmesImport;
import com.powsybl.cgmes.conversion.CgmesModelExtension;
import com.powsybl.cgmes.conversion.Conversion;
import com.powsybl.cgmes.conversion.test.network.compare.Comparison;
import com.powsybl.cgmes.conversion.test.network.compare.ComparisonConfig;
import com.powsybl.cgmes.model.*;
import com.powsybl.commons.datasource.DataSource;
import com.powsybl.commons.datasource.DirectoryDataSource;
import com.powsybl.commons.datasource.ReadOnlyDataSource;
import com.powsybl.commons.datasource.ZipArchiveDataSource;
import com.powsybl.iidm.network.Network;
import com.powsybl.iidm.network.impl.NetworkFactoryImpl;
import com.powsybl.iidm.serde.XMLExporter;
import com.powsybl.loadflow.LoadFlowParameters;
import com.powsybl.loadflow.resultscompletion.LoadFlowResultsCompletion;
import com.powsybl.loadflow.resultscompletion.LoadFlowResultsCompletionParameters;
import com.powsybl.loadflow.validation.ValidationConfig;
import com.powsybl.loadflow.validation.ValidationType;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.FileSystem;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Properties;
import java.util.function.Consumer;
import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
/**
* @author Luma Zamarre��o {@literal <zamarrenolm at aia.es>}
*/
public class ConversionTester {
public ConversionTester(Properties importParams, Properties exportParams, List<String> tripleStoreImplementations,
ComparisonConfig networkComparison) {
this.importParams = importParams;
this.exportParams = exportParams;
this.tripleStoreImplementations = tripleStoreImplementations;
this.networkComparison = networkComparison;
this.onlyReport = false;
this.strictTopologyTest = true;
this.exportXiidm = false;
this.exportCgmes = false;
this.testExportImportCgmes = false;
}
public ConversionTester(Properties importParams, List<String> tripleStoreImplementations,
ComparisonConfig networkComparison) {
this(importParams, null, tripleStoreImplementations, networkComparison);
}
public void setOnlyReport(boolean onlyReport) {
this.onlyReport = onlyReport;
}
public void setReportConsumer(Consumer<String> reportConsumer) {
this.reportConsumer = reportConsumer;
}
public void setTestExportImportCgmes(boolean testExportImportCgmes) {
this.testExportImportCgmes = testExportImportCgmes;
}
public void setValidateBusBalances(boolean b) {
this.validateBusBalances = b;
}
public void setValidateBusBalancesUsingThreshold(double threshold) {
this.validateBusBalances = true;
this.validateBusBalancesThreshold = threshold;
}
public void testConversion(Network expected, GridModelReference gm) throws IOException {
testConversion(expected, gm, this.networkComparison);
}
public void testConversion(Network expected, GridModelReference gm, ComparisonConfig config)
throws IOException {
if (onlyReport) {
testConversionOnlyReport(gm);
} else {
for (String impl : tripleStoreImplementations) {
LOG.info("testConversion. TS implementation {}, grid model {}", impl, gm.name());
testConversion(expected, gm, config, impl);
}
}
}
public Network lastConvertedNetwork() {
return lastConvertedNetwork;
}
private void testConversion(Network expected, GridModelReference gm, ComparisonConfig config, String impl)
throws IOException {
Properties iparams = importParams == null ? new Properties() : importParams;
iparams.put("storeCgmesModelAsNetworkExtension", "true");
iparams.put("powsyblTripleStore", impl);
// This is to be able to easily compare the topology computed
// by powsybl against the topology present in the CGMES model
iparams.put("createBusbarSectionForEveryConnectivityNode", "true");
try (FileSystem fs = Jimfs.newFileSystem(Configuration.unix())) {
CgmesImport i = new CgmesImport();
ReadOnlyDataSource ds = gm.dataSource();
Network network = i.importData(ds, new NetworkFactoryImpl(), iparams);
if (network.getSubstationCount() == 0) {
fail("Model is empty");
}
CgmesModel cgmes = network.getExtension(CgmesModelExtension.class).getCgmesModel();
if (!new TopologyTester(cgmes, network).test(strictTopologyTest)) {
fail("Topology test failed");
}
if (expected != null) {
new Comparison(expected, network, config).compare();
}
if (exportXiidm) {
exportXiidm(gm.name(), impl, expected, network);
}
if (exportCgmes) {
exportCgmes(gm.name(), impl, network);
}
if (testExportImportCgmes) {
testExportImportCgmes(network, ds, fs, i, iparams, config);
}
if (validateBusBalances) {
validateBusBalances(network);
}
lastConvertedNetwork = network;
}
}
private void testConversionOnlyReport(GridModelReference gm) {
CgmesImport i = new CgmesImport();
ReadOnlyDataSource ds = gm.dataSource();
LOG.info("Importer.exists() == {}", i.exists(ds));
Network n = i.importData(ds, new NetworkFactoryImpl(), importParams);
CgmesModel m = n.getExtension(CgmesModelExtension.class).getCgmesModel();
new Conversion(m).report(reportConsumer);
}
private static void exportXiidm(String name, String impl, Network expected, Network actual) throws IOException {
String name1 = name.replace('/', '-');
Path path = Files.createTempDirectory("temp-conversion-" + name1 + "-" + impl + "-");
XMLExporter xmlExporter = new XMLExporter();
// Last component of the path is the name for the exported XML
if (expected != null) {
xmlExporter.export(expected, null, new DirectoryDataSource(path, "expected"));
}
if (actual != null) {
xmlExporter.export(actual, null, new DirectoryDataSource(path, "actual"));
}
}
private static void exportCgmes(String name, String impl, Network network) throws IOException {
String name1 = name.replace('/', '-');
Path path = Files.createTempDirectory("temp-export-cgmes-" + name1 + "-" + impl + "-");
new CgmesExport().export(network, null, new DirectoryDataSource(path, "foo"));
}
private static String subsetFromName(String name) {
return Stream.of(CgmesSubset.values())
.filter(s -> s.isValidName(name))
.map(CgmesSubset::getIdentifier)
.findFirst()
.orElse("unknown");
}
private void testExportImportCgmes(Network network, ReadOnlyDataSource originalDs, FileSystem fs, CgmesImport i, Properties iparams,
ComparisonConfig config) throws IOException {
// We copy everything from the original data source to the temporary destination with a normalized name
// And then export the requested instance files to the same temporary destination
// We will overwrite some of the files, with the expected content
// Create a temporary directory to store the exported files
Path path = fs.getPath("temp-export-cgmes");
Files.createDirectories(path);
String baseName = originalDs.getBaseName();
DataSource ds = new ZipArchiveDataSource(path, baseName);
// Copy the original files to the temporary destination, ensuring a normalized name
for (String name : new CgmesOnDataSource(originalDs).names()) {
String normalizedName = baseName + "_" + subsetFromName(name) + ".xml";
try (OutputStream out = new BufferedOutputStream(ds.newOutputStream(normalizedName, false));
InputStream in = originalDs.newInputStream(name)) {
ByteStreams.copy(in, out);
}
}
// Export the requested instance files to the temporary destination, overwriting some of the original files
new CgmesExport().export(network, exportParams, ds);
// Import the exported files and compare with the original network
Network actual = i.importData(ds, new NetworkFactoryImpl(), iparams);
new Comparison(network, actual, config).compare();
}
public void validateBusBalances(Network network) throws IOException {
try (FileSystem fs = Jimfs.newFileSystem(Configuration.unix())) {
ValidationConfig config = loadFlowValidationConfig(validateBusBalancesThreshold);
Path working = Files.createDirectories(fs.getPath("lf-validation"));
computeMissingFlows(network, config.getLoadFlowParameters());
assertTrue(ValidationType.BUSES.check(network, config, working));
}
}
private static ValidationConfig loadFlowValidationConfig(double threshold) {
ValidationConfig config = ValidationConfig.load();
config.setVerbose(true);
config.setThreshold(threshold);
config.setOkMissingValues(false);
config.setLoadFlowParameters(new LoadFlowParameters());
LOG.info("twtSplitShuntAdmittance is {}", config.getLoadFlowParameters().isTwtSplitShuntAdmittance());
return config;
}
public static void computeMissingFlows(Network network, LoadFlowParameters lfparams) {
LoadFlowResultsCompletionParameters p = new LoadFlowResultsCompletionParameters(
LoadFlowResultsCompletionParameters.EPSILON_X_DEFAULT,
LoadFlowResultsCompletionParameters.APPLY_REACTANCE_CORRECTION_DEFAULT,
LoadFlowResultsCompletionParameters.Z0_THRESHOLD_DIFF_VOLTAGE_ANGLE);
LoadFlowResultsCompletion lf = new LoadFlowResultsCompletion(p, lfparams);
try {
lf.run(network, null);
} catch (Exception e) {
LOG.error("computeFlows, error {}", e.getMessage());
}
}
private final Properties importParams;
private final Properties exportParams;
private final List<String> tripleStoreImplementations;
private final ComparisonConfig networkComparison;
private boolean onlyReport;
private boolean exportXiidm;
private boolean exportCgmes;
private boolean testExportImportCgmes;
private boolean validateBusBalances;
private double validateBusBalancesThreshold = 0.01;
private Consumer<String> reportConsumer;
private boolean strictTopologyTest;
private Network lastConvertedNetwork;
private static final Logger LOG = LoggerFactory.getLogger(ConversionTester.class);
}