From c2b585fbd23e1e7de8f7dba457ca632bb07708da Mon Sep 17 00:00:00 2001 From: Sasha Rahlin Date: Mon, 23 Dec 2024 12:47:50 +1300 Subject: [PATCH] Update netCDF datasource plugin to work with modern netCDF4 C++ bindings --- cmake/modules/FindNetcdf.cmake | 10 +- src/datasources/netcdf/netcdfplugin.cpp | 6 +- src/datasources/netcdf/netcdfsource.cpp | 282 +++++++++++------------- src/datasources/netcdf/netcdfsource.h | 8 +- 4 files changed, 132 insertions(+), 174 deletions(-) diff --git a/cmake/modules/FindNetcdf.cmake b/cmake/modules/FindNetcdf.cmake index c87f40074..88236abc5 100644 --- a/cmake/modules/FindNetcdf.cmake +++ b/cmake/modules/FindNetcdf.cmake @@ -17,11 +17,11 @@ if(NOT NETCDF_INCLUDEDIR) if(NOT kst_cross) include(FindPkgConfig) - pkg_check_modules(NETCDF QUIET netcdf) + pkg_check_modules(NETCDF netcdf netcdf-cxx4) endif() if(NETCDF_INCLUDEDIR AND NETCDF_LIBRARIES) - FIND_LIBRARY(NETCDF_LIBRARY_CPP netcdf_c++ + FIND_LIBRARY(NETCDF_LIBRARY_CPP netcdf_c++4 HINTS ${NETCDF_LIBRARY_DIRS}) set(NETCDF_LIBRARY_C -L${NETCDF_LIBRARY_DIRS} ${NETCDF_LIBRARIES} CACHE STRING "" FORCE) else() @@ -46,9 +46,9 @@ else() find_netcdf_lib(netcdf_c netcdf) find_netcdf_lib(netcdf_c_debug netcdfd) - find_netcdf_lib(netcdf_cpp netcdf_c++) - find_netcdf_lib(netcdf_cpp_debug netcdf_c++d) - + find_netcdf_lib(netcdf_cpp netcdf_c++4) + find_netcdf_lib(netcdf_cpp_debug netcdf_c++4d) + if(netcdf_c AND netcdf_c_debug) set(NETCDF_LIBRARY_C optimized ${netcdf_c} debug ${netcdf_c_debug} CACHE STRING "" FORCE) endif() diff --git a/src/datasources/netcdf/netcdfplugin.cpp b/src/datasources/netcdf/netcdfplugin.cpp index 01cdb0f6b..74bf46736 100644 --- a/src/datasources/netcdf/netcdfplugin.cpp +++ b/src/datasources/netcdf/netcdfplugin.cpp @@ -126,13 +126,11 @@ int NetCdfPlugin::understands(QSettings *cfg, const QString& filename) const return 0; } - NcFile *ncfile = new NcFile(filename.toUtf8().data()); - if (ncfile->is_valid()) { + netCDF::NcFile ncfile(filename.toUtf8().data(), netCDF::NcFile::read); + if (!ncfile.isNull()) { KST_DBG qDebug() << filename << " looks like netCDF !" << endl; - delete ncfile; return 80; } else { - delete ncfile; return 0; } } diff --git a/src/datasources/netcdf/netcdfsource.cpp b/src/datasources/netcdf/netcdfsource.cpp index 4a5a466b2..65ccbc124 100644 --- a/src/datasources/netcdf/netcdfsource.cpp +++ b/src/datasources/netcdf/netcdfsource.cpp @@ -38,6 +38,7 @@ using namespace Kst; +using namespace netCDF; static const QString netCdfTypeString = "netCDF Files"; @@ -192,50 +193,72 @@ bool DataInterfaceNetCdfVector::isValid(const QString& field) const QMap DataInterfaceNetCdfVector::metaScalars(const QString& field) { - NcVar *var = 0; + NcVar var; if (field != "INDEX") { - var = netcdf._ncfile->get_var((NcToken) field.toLatin1().constData()); + var = netcdf._ncfile->getVar(field.toLatin1().constData()); } - if (!var) { + if (var.isNull()) { NETCDF_DBG qDebug() << "Queried field " << field << " which can't be read" << endl; return QMap(); } QMap fieldScalars; - fieldScalars["NbAttributes"] = var->num_atts(); - for (int i=0; inum_atts(); ++i) { - NcAtt *att = var->get_att(i); + fieldScalars["NbAttributes"] = var.getAttCount(); + for (auto att: var.getAtts()) { + size_t n = att.second.getAttLength(); + std::vector data(n); // Only handle char attributes as fieldStrings, the others as fieldScalars - if (att->type() == ncByte || att->type() == ncShort || att->type() == ncInt - || att->type() == ncLong || att->type() == ncFloat || att->type() == ncDouble) { - // Some attributes may have multiple values => load the first as is, and for the others - // add a -2, -3, etc... suffix as obviously we can have only one value per scalar. - // Do it in two steps to avoid a test in the loop while keeping a "clean" name for the first one - fieldScalars[QString(att->name())] = att->values()->as_double(0); - for (int j=1; jvalues()->num(); ++j) { - fieldScalars[QString(att->name()) + QString("-") + QString::number(j+1)] = att->values()->as_double(j); - } + switch (att.second.getType().getTypeClass()) { + case NC_BYTE: + case NC_UBYTE: + case NC_SHORT: + case NC_USHORT: + case NC_INT: + case NC_UINT: + case NC_INT64: + case NC_UINT64: + case NC_FLOAT: + case NC_DOUBLE: + att.second.getValues((double *)&data[0]); + break; + default: + break; } + // Some attributes may have multiple values => load the first as is, and for the others + // add a -2, -3, etc... suffix as obviously we can have only one value per scalar. + // Do it in two steps to avoid a test in the loop while keeping a "clean" name for the first one + QString name(att.first.c_str()); + fieldScalars[name] = data[0]; + for (size_t j=1; j DataInterfaceNetCdfVector::metaStrings(const QString& field) { - NcVar *var = 0; + NcVar var; if (field != "INDEX") { - var = netcdf._ncfile->get_var(field.toLatin1().constData()); + var = netcdf._ncfile->getVar(field.toLatin1().constData()); } - if (!var) { + if (var.isNull()) { NETCDF_DBG qDebug() << "Queried field " << field << " which can't be read" << endl; return QMap(); } QMap fieldStrings; QString tmpString; - for (int i=0; inum_atts(); ++i) { - NcAtt *att = var->get_att(i); + for (auto att: var.getAtts()) { // Only handle char/unspecified attributes as fieldStrings, the others as fieldScalars - if (att->type() == ncChar || att->type() == ncNoType) { - fieldStrings[att->name()] = QString(att->values()->as_string(0)); + switch (att.second.getType().getTypeClass()) { + case NC_CHAR: + case NC_STRING: + { + std::string s; + att.second.getValues(s); + fieldStrings[QString(att.first.c_str())] = QString(s.c_str()); + } + break; + default: + break; } // qDebug() << att->name() << ": " << att->values()->num() << endl; } @@ -282,20 +305,19 @@ const DataMatrix::DataInfo DataInterfaceNetCdfMatrix::dataInfo(const QString& ma return DataMatrix::DataInfo(); } - QByteArray bytes = matrix.toLatin1(); - NcVar *var = netcdf._ncfile->get_var(bytes.constData()); // var is owned by _ncfile - if (!var) { + NcVar var = netcdf._ncfile->getVar(matrix.toLatin1().constData()); // var is owned by _ncfile + if (var.isNull()) { return DataMatrix::DataInfo(); } - if (var->num_dims() != 2) { + if (var.getDimCount() != 2) { return DataMatrix::DataInfo(); } DataMatrix::DataInfo info; // TODO is this right? - info.xSize = var->get_dim(0)->size(); - info.ySize = var->get_dim(1)->size(); + info.xSize = var.getDim(0).getSize(); + info.ySize = var.getDim(1).getSize(); return info; } @@ -326,7 +348,6 @@ bool DataInterfaceNetCdfMatrix::isValid(const QString& field) const { NetcdfSource::NetcdfSource(Kst::ObjectStore *store, QSettings *cfg, const QString& filename, const QString& type, const QDomElement &element) : Kst::DataSource(store, cfg, filename, type), _ncfile(0L), - _ncErr(NcError::silent_nonfatal), is(new DataInterfaceNetCdfScalar(*this)), it(new DataInterfaceNetCdfString(*this)), iv(new DataInterfaceNetCdfVector(*this)), @@ -367,8 +388,8 @@ void NetcdfSource::reset() { bool NetcdfSource::initFile() { - _ncfile = new NcFile(_filename.toUtf8().data(), NcFile::ReadOnly); - if (!_ncfile->is_valid()) { + _ncfile = new NcFile(_filename.toUtf8().data(), NcFile::read); + if (_ncfile->isNull()) { qDebug() << _filename << ": failed to open in initFile()" << endl; return false; } @@ -377,43 +398,38 @@ bool NetcdfSource::initFile() { _fieldList.clear(); _fieldList += "INDEX"; - int nb_vars = _ncfile->num_vars(); - NETCDF_DBG qDebug() << nb_vars << " vars found in total" << endl; + NETCDF_DBG qDebug() << _ncfile->getVarCount() << " vars found in total" << endl; _maxFrameCount = 0; - for (int i = 0; i < nb_vars; i++) { - NcVar *var = _ncfile->get_var(i); - if (!var) { + for (auto var: _ncfile->getVars()) { + if (var.second.isNull()) { continue; } - if (var->num_dims() == 0) { - _scalarList += var->name(); - } else if (var->num_dims() == 1) { - _fieldList += var->name(); - int fc = var->num_vals() / var->rec_size(); + if (var.second.getDimCount() == 0) { + _scalarList += var.first.c_str(); + } else if (var.second.getDimCount() == 1) { + _fieldList += var.first.c_str(); + int fc = var.second.getDim(0).getSize(); _maxFrameCount = qMax(_maxFrameCount, fc); - _frameCounts[var->name()] = fc; - } else if (var->num_dims() == 2) { - _matrixList += var->name(); + _frameCounts[QString(var.first.c_str())] = fc; + } else if (var.second.getDimCount() == 2) { + _matrixList += var.first.c_str(); } } // Get strings - int globalAttributesNb = _ncfile->num_atts(); - for (int i = 0; i < globalAttributesNb; ++i) { + for (auto att: _ncfile->getAtts()) { // Get only first value, should be enough for a start especially as strings are complete - NcAtt *att = _ncfile->get_att(i); - if (att) { - QString attrName = QString(att->name()); - char *attString = att->as_string(0); - QString attrValue = QString(att->as_string(0)); - delete[] attString; + if (!att.second.isNull()) { + QString attrName = QString(att.first.c_str()); + std::string s; + att.second.getValues(s); + QString attrValue = QString(s.c_str()); //TODO port //KstString *ms = new KstString(KstObjectTag(attrName, tag()), this, attrValue); _strings[attrName] = attrValue; } - delete att; } setUpdateType(Timer); @@ -437,16 +453,17 @@ Kst::Object::UpdateType NetcdfSource::internalDataSourceUpdate() { bool updated = false; /* Update member variables _ncfile, _maxFrameCount, and _frameCounts and indicate that an update is needed */ - int nb_vars = _ncfile->num_vars(); - for (int j = 0; j < nb_vars; j++) { - NcVar *var = _ncfile->get_var(j); - if (!var) { + for (auto var: _ncfile->getVars()) { + if (var.second.isNull()) { continue; } - int fc = var->num_vals() / var->rec_size(); - _maxFrameCount = qMax(_maxFrameCount, fc); - updated = updated || (_frameCounts[var->name()] != fc); - _frameCounts[var->name()] = fc; + if (var.second.getDimCount() == 1) { + int fc = var.second.getDim(0).getSize(); + _maxFrameCount = qMax(_maxFrameCount, fc); + QString name(var.first.c_str()); + updated = updated || (_frameCounts[name] != fc); + _frameCounts[name] = fc; + } } return updated ? Object::Updated : Object::NoChange; } @@ -455,10 +472,9 @@ Kst::Object::UpdateType NetcdfSource::internalDataSourceUpdate() { int NetcdfSource::readScalar(double *v, const QString& field) { // TODO error handling - QByteArray bytes = field.toLatin1(); - NcVar *var = _ncfile->get_var(bytes.constData()); // var is owned by _ncfile - if (var) { - var->get(v); + NcVar var = _ncfile->getVar(field.toLatin1().constData()); // var is owned by _ncfile + if (!var.isNull()) { + var.getVar(v); return 1; } return 0; @@ -467,20 +483,17 @@ int NetcdfSource::readScalar(double *v, const QString& field) int NetcdfSource::readString(QString *stringValue, const QString& stringName) { // TODO more error handling? - NcAtt *att = _ncfile->get_att((NcToken) stringName.toLatin1().data()); - if (att) { - *stringValue = QString(att->as_string(0)); - delete att; + NcGroupAtt att = _ncfile->getAtt(stringName.toLatin1().constData()); + if (!att.isNull()) { + std::string s; + att.getValues(s); + *stringValue = QString(s.c_str()); return 1; } return 0; } int NetcdfSource::readField(double *v, const QString& field, int s, int n) { - NcType dataType = ncNoType; /* netCDF data type */ - /* Values for one record */ - NcValues *record = 0;// = new NcValues(dataType,numFrameVals); - NETCDF_DBG qDebug() << "Entering NetcdfSource::readField with params: " << field << ", from " << s << " for " << n << " frames" << endl; /* For INDEX field */ @@ -496,24 +509,24 @@ int NetcdfSource::readField(double *v, const QString& field, int s, int n) { } /* For a variable from the netCDF file */ - QByteArray bytes = field.toLatin1(); - NcVar *var = _ncfile->get_var(bytes.constData()); // var is owned by _ncfile - if (!var) { + NcVar var = _ncfile->getVar(field.toLatin1().constData()); // var is owned by _ncfile + if (var.isNull()) { NETCDF_DBG qDebug() << "Queried field " << field << " which can't be read" << endl; return -1; } - dataType = var->type(); - - if (s >= var->num_vals() / var->rec_size()) { + if (s >= (int) var.getDim(0).getSize()) { return 0; } bool oneSample = n < 0; - int recSize = var->rec_size(); double add_offset = 1.0, scale_factor = 1.0; - switch (dataType) { - case ncShort: + std::vector sv(1); + sv[0] = s; + std::vector nv(1); + nv[0] = n; + switch (var.getType().getTypeClass()) { + case NC_SHORT: { // Check for special attributes add_offset and scale_factor indicating the use of the convention described in // @@ -524,83 +537,37 @@ int NetcdfSource::readField(double *v, const QString& field, int s, int n) { scale_factor = iv->metaScalars(field)["scale_factor"]; } if (oneSample) { - record = var->get_rec(s); - v[0] = packed ? record->as_short(0)*scale_factor+add_offset : record->as_short(0); - delete record; + short record; + var.getVar(sv, &record); + v[0] = packed ? record*scale_factor+add_offset : record; } else { - for (int i = 0; i < n; i++) { - record = var->get_rec(i+s); - if (packed) { - for (int j = 0; j < recSize; j++) { - v[i*recSize + j] = record->as_short(j)*scale_factor+add_offset; - } - } - else { - for (int j = 0; j < recSize; j++) { - v[i*recSize + j] = record->as_short(j); - } - } - delete record; + std::vector record(n); + var.getVar(sv, nv, (short *)&record[0]); + for (int i = 0; i < n; i++) { + if (packed) { + v[i] = record[i]*scale_factor+add_offset; + } else { + v[i] = record[i]; + } } } } break; - case ncInt: + case NC_INT: + case NC_UINT: + case NC_INT64: + case NC_UINT64: + case NC_FLOAT: + case NC_DOUBLE: { if (oneSample) { - record = var->get_rec(s); - v[0] = record->as_int(0); - delete record; + var.getVar(sv, v); } else { - for (int i = 0; i < n; i++) { - record = var->get_rec(i+s); - NETCDF_DBG qDebug() << "Read record " << i+s << endl; - for (int j = 0; j < recSize; j++) { - v[i*recSize + j] = record->as_int(j); - } - delete record; - } + var.getVar(sv, nv, v); } } break; - - case ncFloat: - { - if (oneSample) { - record = var->get_rec(s); - v[0] = record->as_float(0); - delete record; - } else { - for (int i = 0; i < n; i++) { - record = var->get_rec(i+s); - for (int j = 0; j < recSize; j++) { - v[i*recSize + j] = record->as_float(j); - } - delete record; - } - } - } - break; - - case ncDouble: - { - if (oneSample) { - record = var->get_rec(s); - v[0] = record->as_double(0); - delete record; - } else { - for (int i = 0; i < n; i++) { - record = var->get_rec(i+s); - for (int j = 0; j < recSize; j++) { - v[i*recSize + j] = record->as_double(j); - } - delete record; - } - } - } - break; - default: NETCDF_DBG qDebug() << field << ": wrong datatype for kst, no values read" << endl; return -1; @@ -610,7 +577,7 @@ int NetcdfSource::readField(double *v, const QString& field, int s, int n) { NETCDF_DBG qDebug() << "Finished reading " << field << endl; - return oneSample ? 1 : n * recSize; + return oneSample ? 1 : n; } @@ -620,19 +587,17 @@ int NetcdfSource::readField(double *v, const QString& field, int s, int n) { int NetcdfSource::readMatrix(double *v, const QString& field) { /* For a variable from the netCDF file */ - QByteArray bytes = field.toLatin1(); - NcVar *var = _ncfile->get_var(bytes.constData()); // var is owned by _ncfile - if (!var) { + NcVar var = _ncfile->getVar(field.toLatin1().constData()); // var is owned by _ncfile + if (var.isNull()) { NETCDF_DBG qDebug() << "Queried field " << field << " which can't be read" << endl; return -1; } - int xSize = var->get_dim(0)->size(); - int ySize = var->get_dim(1)->size(); + int xSize = var.getDim(0).getSize(); + int ySize = var.getDim(1).getSize(); - var->get(v, xSize, ySize); + var.getVar(v); - return xSize * ySize; } @@ -647,13 +612,12 @@ int NetcdfSource::samplesPerFrame(const QString& field) { if (field.toLower() == "index") { return 1; } - QByteArray bytes = field.toLatin1(); - NcVar *var = _ncfile->get_var(bytes.constData()); - if (!var) { + NcVar var = _ncfile->getVar(field.toLatin1().constData()); + if (var.isNull()) { NETCDF_DBG qDebug() << "Queried field " << field << " which can't be read" << endl; return 0; } - return var->rec_size(); + return 1; } diff --git a/src/datasources/netcdf/netcdfsource.h b/src/datasources/netcdf/netcdfsource.h index 7bd7dca77..da77a7454 100644 --- a/src/datasources/netcdf/netcdfsource.h +++ b/src/datasources/netcdf/netcdfsource.h @@ -22,8 +22,7 @@ #include "datasource.h" #include "dataplugin.h" -#include -#include +#include class DataInterfaceNetCdfScalar; @@ -71,10 +70,7 @@ class NetcdfSource : public Kst::DataSource { QMap _frameCounts; int _maxFrameCount; - NcFile *_ncfile; - - // we must hold an NcError to overwrite the exit-on-error behaviour of netCDF - NcError _ncErr; + netCDF::NcFile *_ncfile; QMap _strings; -- GitLab