Fix that reftran should propagate msr Source records (issue #317)

.gitignore

@@ -14,10 +14,10 @@ build_gcc/
 build/
 bin/
 doc/
-*/vcpkg/
-*/vcpkg_installed/
-*/Release/
-*/Debug/
+vcpkg/
+vcpkg_installed/
+Release/
+Debug/
 
 *.sln
 *.vcxproj

dynadjust/CMakeLists.txt

@@ -471,6 +471,8 @@ else()
                     ${VCPKG_INSTALLED_DIR}/${VCPKG_TARGET_TRIPLET}/include/openblas
                     C:/vcpkg/installed/x64-windows/include
                     C:/vcpkg/installed/x64-windows/include/openblas
+                    C:/data/vcpkg/installed/x64-windows/include
+                    C:/data/vcpkg/installed/x64-windows/include/openblas
                 )
             else()
                 list(APPEND BLAS_INCLUDE_SEARCH_PATHS
@@ -865,7 +867,15 @@ if (BUILD_TESTING)
     add_test (NAME test-urban-phased-network COMMAND $<TARGET_FILE:${DNADIFF_TARGET}> urban.phased.adj urban.phased.adj.expected --skip-headers 62 -t 0.001)
     add_test (NAME test-urban-thread-network COMMAND $<TARGET_FILE:${DNADIFF_TARGET}> urban_mt.phased-mt.adj urban_mt.phased-mt.adj.expected -t 0.01 -v)
 
-    # 8. gnss reference frame transformations
+    # 8. Source tag preservation in --export-xml (issue #317)
+    # Import XML data with <Source> tags and verify they are preserved in export
+    add_test (NAME import-source-test COMMAND $<TARGET_FILE:${DNAIMPORT_TARGET}> -n srctest source-test-stn.xml source-test-msr.xml --export-xml -r GDA2020)
+    add_test (NAME check-source-import COMMAND bash check_source_tags.sh srctestmsr.xml CAMPAIGN_2023A CAMPAIGN_2023B EMPTY)
+    # Transform to a different frame and verify <Source> tags survive reftran
+    add_test (NAME reftran-source-test COMMAND $<TARGET_FILE:${DNAREFTRAN_TARGET}> srctest -r itrf2014 -e 01.01.2020 --export-xml)
+    add_test (NAME check-source-reftran COMMAND bash check_source_tags.sh srctest.ITRF2014.01.01.2020msr.xml CAMPAIGN_2023A CAMPAIGN_2023B EMPTY)
+
+    # 9. gnss reference frame transformations
     add_test (NAME ref-gnss01-network COMMAND $<TARGET_FILE:${DNAREFTRAN_TARGET}> gnss -r itrf2014 -e 01.01.2020)
     add_test (NAME ref-gnss02-network COMMAND $<TARGET_FILE:${DNAREFTRAN_TARGET}> gnss -r itrf1988 -e 03.12.1995)
     add_test (NAME ref-gnss03-network COMMAND $<TARGET_FILE:${DNAREFTRAN_TARGET}> gnss -r itrf1989)
@@ -1699,6 +1709,10 @@ if (BUILD_TESTING)
     set_tests_properties(ref-itrf-pmm-06 PROPERTIES DEPENDS ref-itrf-pmm-05)
     #set_tests_properties(ref-itrf-pmm-07 PROPERTIES DEPENDS ref-itrf-pmm-06)
 
+    set_tests_properties(check-source-import PROPERTIES DEPENDS import-source-test)
+    set_tests_properties(reftran-source-test PROPERTIES DEPENDS check-source-import)
+    set_tests_properties(check-source-reftran PROPERTIES DEPENDS reftran-source-test)
+
     # Force all tests to run serially (prevent parallel execution)
     get_property(all_tests DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY TESTS)
     if(all_tests)

dynadjust/dynadjust/dnaimport/dnainterop.cpp

@@ -3949,6 +3949,7 @@ void dna_import::ImportStnsMsrsFromNetwork(vdnaStnPtr* vStations, vdnaMsrPtr* vM
 			}
 
 			msrPtr->SetMeasurementRec(binaryStn, it_msr, it_dbid);
+			msrPtr->ResolveSourceFile(bms_meta_.sourceFileMeta, bms_meta_.sourceFileCount);
 			vMeasurements->push_back(msrPtr);
 		}
 
@@ -4107,8 +4108,9 @@ void dna_import::ImportStnsMsrsFromBlock(vdnaStnPtr* vStations, vdnaMsrPtr* vMea
 		}
 
 		msrPtr->SetMeasurementRec(binaryStn, it_msr, it_dbid);
+		msrPtr->ResolveSourceFile(bms_meta_.sourceFileMeta, bms_meta_.sourceFileCount);
 		vMeasurements->push_back(msrPtr);
-	}	
+	}
 }
 
 

dynadjust/dynadjust/dnareftran/dnareftran.cpp

@@ -1938,7 +1938,8 @@ void dna_reftran::SerialiseDNA(const std::string& stnfilename, const std::string
 			comment.append(", epoch ").append(datumTo_.GetEpoch_s());
 		comment.append(".  Exported by reftran.");
 		dna.set_dbid_ptr(&v_msr_db_map_);
-		dna.write_dna_files(&bstBinaryRecords_, &bmsBinaryRecords_, 
+		dna.set_source_file_meta_ptr(bms_meta_.sourceFileMeta, bms_meta_.sourceFileCount);
+		dna.write_dna_files(&bstBinaryRecords_, &bmsBinaryRecords_,
 			stnfilename, msrfilename, projectSettings_.g.network_name,
 			datumTo_, projection, flagUnused, "Station coordinates" + comment, "GNSS measurements" + comment);
 	}
@@ -2124,6 +2125,7 @@ void dna_reftran::SerialiseDynaMLMsr(std::ofstream* xml_file)
 		}
 
 		msrPtr->SetMeasurementRec(bstBinaryRecords_, _it_msr, _it_dbid);
+		msrPtr->ResolveSourceFile(bms_meta_.sourceFileMeta, bms_meta_.sourceFileCount);
 		msrPtr->WriteDynaMLMsr(xml_file, comment);
 	}
 }

dynadjust/include/config/dnatypes-structs.hpp

@@ -335,17 +335,67 @@ typedef struct input_file_meta {
 	UINT16	datatype;						// Input data type (station, measurement, both)
 } input_file_meta_t;
 
+// Source file metadata for measurement provenance
+typedef struct source_file_meta {
+	char	filename[FILE_NAME_WIDTH+1];
+} source_file_meta_t;
+
 // Binary file metadata
 typedef struct binary_file_meta {
 	binary_file_meta ()
-		: binCount(0), reduced(false), reftran(false), geoid(false), inputFileCount(0), inputFileMeta(NULL) {}
+		: binCount(0), reduced(false), reftran(false), geoid(false)
+		, inputFileCount(0), inputFileMeta(NULL)
+		, sourceFileCount(0), sourceFileMeta(nullptr) {}
 	binary_file_meta (const std::string& app_name)
-		: binCount(0), reduced(false), reftran(false), geoid(false), inputFileCount(0), inputFileMeta(NULL) {
+		: binCount(0), reduced(false), reftran(false), geoid(false)
+		, inputFileCount(0), inputFileMeta(NULL)
+		, sourceFileCount(0), sourceFileMeta(nullptr) {
             snprintf(modifiedBy, sizeof(modifiedBy), "%s", app_name.c_str());
 	}
 	~binary_file_meta() {
 		if (inputFileMeta != NULL)
 			delete []inputFileMeta;
+		if (sourceFileMeta != nullptr)
+			delete []sourceFileMeta;
+	}
+
+	binary_file_meta(const binary_file_meta&) = delete;
+	binary_file_meta& operator=(const binary_file_meta&) = delete;
+
+	binary_file_meta(binary_file_meta&& rhs) noexcept
+		: binCount(rhs.binCount), reduced(rhs.reduced)
+		, reftran(rhs.reftran), geoid(rhs.geoid)
+		, inputFileCount(rhs.inputFileCount), inputFileMeta(rhs.inputFileMeta)
+		, sourceFileCount(rhs.sourceFileCount), sourceFileMeta(rhs.sourceFileMeta)
+	{
+		memcpy(modifiedBy, rhs.modifiedBy, sizeof(modifiedBy));
+		memcpy(epsgCode, rhs.epsgCode, sizeof(epsgCode));
+		memcpy(epoch, rhs.epoch, sizeof(epoch));
+		rhs.inputFileMeta = nullptr;
+		rhs.sourceFileMeta = nullptr;
+	}
+
+	binary_file_meta& operator=(binary_file_meta&& rhs) noexcept
+	{
+		if (this != &rhs)
+		{
+			delete []inputFileMeta;
+			delete []sourceFileMeta;
+			binCount = rhs.binCount;
+			reduced = rhs.reduced;
+			reftran = rhs.reftran;
+			geoid = rhs.geoid;
+			memcpy(modifiedBy, rhs.modifiedBy, sizeof(modifiedBy));
+			memcpy(epsgCode, rhs.epsgCode, sizeof(epsgCode));
+			memcpy(epoch, rhs.epoch, sizeof(epoch));
+			inputFileCount = rhs.inputFileCount;
+			inputFileMeta = rhs.inputFileMeta;
+			sourceFileCount = rhs.sourceFileCount;
+			sourceFileMeta = rhs.sourceFileMeta;
+			rhs.inputFileMeta = nullptr;
+			rhs.sourceFileMeta = nullptr;
+		}
+		return *this;
 	}
     std::uint64_t		binCount;						// number of records in the binary file
 	bool				reduced;						// indicates whether the data is reduced(true) or raw(false)
@@ -356,10 +406,13 @@ typedef struct binary_file_meta {
 	bool				geoid;							// geoid separation values have been obtained
     std::uint64_t		inputFileCount;					// Number of source file metadata elements
 	input_file_meta_t*	inputFileMeta;					// Source file metadata
+	std::uint64_t		sourceFileCount;				// Number of unique source files for measurements
+	source_file_meta_t*	sourceFileMeta;					// Source file metadata for measurement provenance
 } binary_file_meta_t;
 
 typedef std::vector<input_file_meta_t> vifm_t;
 typedef vifm_t::iterator it_vifm_t;
+typedef std::vector<source_file_meta_t> vsfm_t;
 typedef std::vector<binary_file_meta_t> vbfm_t;
 typedef vbfm_t::iterator it_vbfm_t;
 

dynadjust/include/io/bms_file.cpp

@@ -25,6 +25,7 @@
 #include <fstream>
 #include <ios>
 #include <iostream>
+#include <map>
 #include <sstream>
 
 #include <include/functions/dnaiostreamfuncs.hpp>
@@ -246,6 +247,29 @@ void BmsFile::WriteFile(const std::string& bms_filename,
   UINT32 msrIndex(0);
 
   try {
+    // Build unique source file index map
+    std::map<std::string, UINT32> sourceFileMap;
+    std::vector<std::string> sourceFileList;
+    for (it_msr = vMeasurements->begin(); it_msr != vMeasurements->end(); ++it_msr) {
+      std::string src = it_msr->get()->GetSource();
+      if (sourceFileMap.find(src) == sourceFileMap.end()) {
+        sourceFileMap[src] = static_cast<UINT32>(sourceFileList.size());
+        sourceFileList.push_back(src);
+      }
+      it_msr->get()->SetSourceFileIndex(sourceFileMap[src]);
+    }
+
+    // Populate source file metadata
+    bms_meta.sourceFileCount = sourceFileList.size();
+    if (bms_meta.sourceFileMeta != nullptr)
+      delete[] bms_meta.sourceFileMeta;
+    bms_meta.sourceFileMeta = new source_file_meta_t[bms_meta.sourceFileCount];
+    for (std::uint64_t i(0); i < bms_meta.sourceFileCount; ++i) {
+      memset(bms_meta.sourceFileMeta[i].filename, '\0', sizeof(bms_meta.sourceFileMeta[i].filename));
+      snprintf(bms_meta.sourceFileMeta[i].filename, sizeof(bms_meta.sourceFileMeta[i].filename),
+        "%s", sourceFileList[i].c_str());
+    }
+
     WriteFileInfo(bms_file);
     WriteFileMetadata(bms_file, bms_meta);
 

dynadjust/include/io/bms_file_loader.cpp

@@ -24,6 +24,7 @@
 #include <fstream>
 #include <ios>
 #include <iostream>
+#include <map>
 #include <sstream>
 
 #include <include/functions/dnaiostreamfuncs.hpp>
@@ -101,8 +102,8 @@ void BmsFileLoader::LoadFileMeta(const std::string& bms_filename,
      << bms_filename << "." << std::endl;
 
   try {
-    readFileInfo(bms_file);
-    readFileMetadata(bms_file, bms_meta);
+    ReadFileInfo(bms_file);
+    ReadFileMetadata(bms_file, bms_meta);
   } catch (const std::ios_base::failure& f) {
     ss << f.what();
     throw std::runtime_error(ss.str());
@@ -142,8 +143,8 @@ std::uint64_t BmsFileLoader::LoadFile(const std::string& bms_filename,
      << bms_filename << "." << std::endl;
 
   try {
-    readFileInfo(bms_file);
-    readFileMetadata(bms_file, bms_meta);
+    ReadFileInfo(bms_file);
+    ReadFileMetadata(bms_file, bms_meta);
 
     vbinary_msr->reserve(bms_meta.binCount);
     for (msr = 0; msr < bms_meta.binCount; msr++) {
@@ -198,8 +199,8 @@ void BmsFileLoader::WriteFile(const std::string& bms_filename,
      << "." << std::endl;
 
   try {
-    writeFileInfo(bms_file);
-    writeFileMetadata(bms_file, bms_meta);
+    WriteFileInfo(bms_file);
+    WriteFileMetadata(bms_file, bms_meta);
 
     measurements::it_vmsr_t it_msr(vbinary_msr->begin());
     for (it_msr = vbinary_msr->begin(); it_msr != vbinary_msr->end(); ++it_msr) {
@@ -245,8 +246,31 @@ void BmsFileLoader::WriteFile(const std::string& bms_filename,
   UINT32 msrIndex(0);
 
   try {
-    writeFileInfo(bms_file);
-    writeFileMetadata(bms_file, bms_meta);
+    // Build unique source file index map
+    std::map<std::string, UINT32> sourceFileMap;
+    std::vector<std::string> sourceFileList;
+    for (it_msr = vMeasurements->begin(); it_msr != vMeasurements->end(); ++it_msr) {
+      std::string src = it_msr->get()->GetSource();
+      if (sourceFileMap.find(src) == sourceFileMap.end()) {
+        sourceFileMap[src] = static_cast<UINT32>(sourceFileList.size());
+        sourceFileList.push_back(src);
+      }
+      it_msr->get()->SetSourceFileIndex(sourceFileMap[src]);
+    }
+
+    // Populate source file metadata
+    bms_meta.sourceFileCount = sourceFileList.size();
+    if (bms_meta.sourceFileMeta != nullptr)
+      delete[] bms_meta.sourceFileMeta;
+    bms_meta.sourceFileMeta = new source_file_meta_t[bms_meta.sourceFileCount];
+    for (std::uint64_t i(0); i < bms_meta.sourceFileCount; ++i) {
+      memset(bms_meta.sourceFileMeta[i].filename, '\0', sizeof(bms_meta.sourceFileMeta[i].filename));
+      snprintf(bms_meta.sourceFileMeta[i].filename, sizeof(bms_meta.sourceFileMeta[i].filename),
+        "%s", sourceFileList[i].c_str());
+    }
+
+    WriteFileInfo(bms_file);
+    WriteFileMetadata(bms_file, bms_meta);
 
     for (it_msr = vMeasurements->begin(); it_msr != vMeasurements->end();
          ++it_msr) {

dynadjust/include/io/dnaiodna.cpp

@@ -590,6 +590,7 @@ void dna_io_dna::write_msr_file(const vstn_t& vbinary_stn, pvmsr_t vbinary_msr,
 			}
 
 			msrPtr->SetMeasurementRec(vbinary_stn, _it_msr, _it_dbid);
+			msrPtr->ResolveSourceFile(sourceFileMeta_, sourceFileCount_);
 			msrPtr->WriteDNAMsr(&dna_msr_file, dmw_, dml_);
 		}
 

dynadjust/include/io/dnaiodna.hpp

@@ -103,10 +103,14 @@ class dna_io_dna : public DynadjustFile
 	inline const dna_msr_fields	dna_msr_widths() { return dmw_; }
 
 	void set_dbid_ptr(pv_msr_database_id_map pv_msr_db_map);
+	void set_source_file_meta_ptr(const source_file_meta_t* sfm, std::uint64_t count) {
+		sourceFileMeta_ = sfm;
+		sourceFileCount_ = count;
+	}
 
 	inline bool filespecifiedReferenceFrame() { return m_filespecifiedReferenceFrame_; }
 	inline bool filespecifiedEpoch() { return m_filespecifiedEpoch_; }
-	
+
 protected:
 
 private:
@@ -143,6 +147,9 @@ class dna_io_dna : public DynadjustFile
 	bool					m_databaseIDsSet_;
 	bool                    m_filespecifiedReferenceFrame_;
 	bool					m_filespecifiedEpoch_;
+
+	const source_file_meta_t*	sourceFileMeta_ = nullptr;
+	std::uint64_t				sourceFileCount_ = 0;
 };
 
 }	// namespace iostreams

dynadjust/include/io/dynadjust_file.cpp

@@ -101,9 +101,14 @@ void DynadjustFile::WriteFileMetadata(std::ofstream& file_stream, binary_file_me
 		file_stream.write(reinterpret_cast<char *>(file_meta.inputFileMeta[i].filename), FILE_NAME_WIDTH); 
 		file_stream.write(reinterpret_cast<char *>(file_meta.inputFileMeta[i].epsgCode), STN_EPSG_WIDTH);
 		file_stream.write(reinterpret_cast<char *>(file_meta.inputFileMeta[i].epoch), STN_EPOCH_WIDTH);
-		file_stream.write(reinterpret_cast<char *>(&file_meta.inputFileMeta[i].filetype), sizeof(UINT16)); 
-		file_stream.write(reinterpret_cast<char *>(&file_meta.inputFileMeta[i].datatype), sizeof(UINT16)); 
+		file_stream.write(reinterpret_cast<char *>(&file_meta.inputFileMeta[i].filetype), sizeof(UINT16));
+		file_stream.write(reinterpret_cast<char *>(&file_meta.inputFileMeta[i].datatype), sizeof(UINT16));
 	}
+
+	// Write source file count and source file meta (v1.1+)
+	file_stream.write(reinterpret_cast<char *>(&file_meta.sourceFileCount), sizeof(std::uint64_t));
+	for (std::uint64_t i(0); i<file_meta.sourceFileCount; ++i)
+		file_stream.write(reinterpret_cast<char *>(file_meta.sourceFileMeta[i].filename), FILE_NAME_WIDTH);
 }
 
 
@@ -133,11 +138,36 @@ void DynadjustFile::ReadFileMetadata(std::ifstream& file_stream, binary_file_met
 		file_stream.read(reinterpret_cast<char *>(file_meta.inputFileMeta[i].filename), FILE_NAME_WIDTH); 
 		file_stream.read(reinterpret_cast<char *>(file_meta.inputFileMeta[i].epsgCode), STN_EPSG_WIDTH);
 		file_stream.read(reinterpret_cast<char *>(file_meta.inputFileMeta[i].epoch), STN_EPOCH_WIDTH);
-		file_stream.read(reinterpret_cast<char *>(&file_meta.inputFileMeta[i].filetype), sizeof(UINT16)); 
-		file_stream.read(reinterpret_cast<char *>(&file_meta.inputFileMeta[i].datatype), sizeof(UINT16)); 
+		file_stream.read(reinterpret_cast<char *>(&file_meta.inputFileMeta[i].filetype), sizeof(UINT16));
+		file_stream.read(reinterpret_cast<char *>(&file_meta.inputFileMeta[i].datatype), sizeof(UINT16));
+	}
+
+	// Read source file meta (v1.1+)
+	if (versionAtLeast(1, 1))
+	{
+		file_stream.read(reinterpret_cast<char *>(&file_meta.sourceFileCount), sizeof(std::uint64_t));
+		if (file_meta.sourceFileMeta != nullptr)
+			delete []file_meta.sourceFileMeta;
+
+		file_meta.sourceFileMeta = new source_file_meta_t[file_meta.sourceFileCount];
+
+		for (std::uint64_t i(0); i<file_meta.sourceFileCount; ++i)
+			file_stream.read(reinterpret_cast<char *>(file_meta.sourceFileMeta[i].filename), FILE_NAME_WIDTH);
+	}
+	else
+	{
+		file_meta.sourceFileCount = 0;
+		file_meta.sourceFileMeta = nullptr;
 	}
 }
-
+
+
+bool DynadjustFile::versionAtLeast(int major, int minor) const
+{
+	int vmajor = 0, vminor = 0;
+	sscanf(version_.c_str(), "%d.%d", &vmajor, &vminor);
+	return (vmajor > major) || (vmajor == major && vminor >= minor);
+}
 
 void DynadjustFile::WriteVersion(std::ofstream& file_stream)
 {

dynadjust/include/io/dynadjust_file.hpp

@@ -37,7 +37,7 @@
 #include <include/config/dnatypes-fwd.hpp>
 #include <include/config/dnatypes-structs.hpp>
 
-#define __FILE_VERSION__ "1.0"
+#define __FILE_VERSION__ "1.1"
 
 namespace dynadjust {
 namespace iostreams {
@@ -80,6 +80,8 @@ class DNATYPE_API DynadjustFile {
 
 protected:
 
+	bool versionAtLeast(int major, int minor) const;
+
 	void WriteVersion(std::ofstream& file_stream);
 	void ReadVersion(std::ifstream& file_stream);
 	void WriteDate(std::ofstream& file_stream);