-
Carl Philipp Klemm authoredCarl Philipp Klemm authored
model.cpp 21.85 KiB
//SPDX-License-Identifier: LGPL-3.0-or-later
//
// eisgenerator - a shared libary and application to generate EIS spectra
// Copyright (C) 2022-2024 Carl Philipp Klemm <carl@uvos.xyz>
//
// This file is part of eisgenerator.
//
// eisgenerator is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// eisgenerator is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with eisgenerator. If not, see <http://www.gnu.org/licenses/>.
//
#include <cstddef>
#include <model.h>
#include <iostream>
#include <assert.h>
#include <sstream>
#include <string>
#include <vector>
#include <thread>
#include <algorithm>
#include <execution>
#include <dlfcn.h>
#include <functional>
#include "componant/componant.h"
#include "componant/resistor.h"
#include "strops.h"
#include "componant/paralellseriel.h"
#include "log.h"
#include "normalize.h"
#include "basicmath.h"
#include "compile.h"
#include "compcache.h"
using namespace eis;
Componant *Model::processBrackets(std::string& str, size_t& bracketCounter, size_t paramSweepCount, bool defaultToRange)
{
size_t bracketStart = deepestBraket(str);
Log(Log::DEBUG)<<str<<" bracket start "<<(bracketStart == std::string::npos ? std::string("npos") : std::to_string(bracketStart));
if(bracketStart == std::string::npos)
{
Componant* componant = processBracket(str, paramSweepCount, defaultToRange);
if(!componant)
throw parse_errror("can not create componant type B for " + str);
return componant;
}
size_t bracketEnd = opposingBraket(str, bracketStart);
if(bracketEnd == std::string::npos)
{
return nullptr;
}
std::string bracket = str.substr(bracketStart+1, bracketEnd-1-bracketStart);
Componant* componant = processBracket(bracket, paramSweepCount, defaultToRange);
if(!componant)
throw parse_errror("can not create componant type B for " + str);
_bracketComponants.push_back(componant);
str.erase(str.begin()+bracketStart, str.begin()+bracketEnd+1);
str.insert(str.begin()+bracketStart, bracketCounter+48);
++bracketCounter;
return processBrackets(str, bracketCounter, paramSweepCount, defaultToRange);
}
Componant *Model::processBracket(std::string& str, size_t paramSweepCount, bool defaultToRange)
{
Log(Log::DEBUG)<<__func__<<'('<<str<<')';
std::vector<std::string> tokens = tokenize(str, '-', '{', '}');
std::vector<Componant*> nodes;
for(const std::string& nodeStr : tokens)
{
Log(Log::DEBUG)<<__func__<<" full node str: "<<nodeStr;
std::vector<Componant*> componants;
for(size_t i = 0; i < nodeStr.size(); ++i)
{
Log(Log::DEBUG)<<__func__<<" arg: "<<nodeStr[i];
if(Componant::isValidComponantChar(nodeStr[i]))
{
componants.push_back(Componant::createNewComponant(nodeStr[i], getParamStr(nodeStr, i), paramSweepCount, defaultToRange));
i = paramSkipIndex(nodeStr, i);
}
else
{
switch(nodeStr[i])
{
case '{':
i = opposingBraket(nodeStr, i, '}');
case '}':
Log(Log::WARN)<<getModelStr()<<" stray "<<nodeStr[i]<<" in model string";
break;
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
{
size_t j = nodeStr[i]-48;
if(_bracketComponants.size() > j)
componants.push_back(_bracketComponants[j]);
break;
}
default:
Log(Log::DEBUG)<<"Invalid char "<<nodeStr[i]<<" in model string";
return nullptr;
}
}
}
if(componants.size() > 1)
nodes.push_back(new Parallel(componants));
else if(componants.size() == 1)
nodes.push_back(componants[0]);
else
Log(Log::WARN)<<"empty node for "<<nodeStr;
}
if(nodes.size() > 1)
return new Serial(nodes);
else if(nodes.size() == 1)
return nodes[0];
else
return nullptr;
}
std::string Model::getParamStr(const std::string& str, size_t index)
{
if(static_cast<int64_t>(str.size())-index < 3 || str[index+1] != '{')
{
Log(Log::WARN)<<"missing parameter string for "<<str[index];
return "";
}
size_t end = opposingBraket(str, index, '}');
std::string parameterStr = str.substr(index+2, end-index-2);
Log(Log::DEBUG)<<"param for "<<str[index]<<' '<<parameterStr;
return parameterStr;
}
size_t Model::paramSkipIndex(const std::string& str, size_t index)
{
if(index+2 > str.size() || str[index+1] != '{')
return index;
size_t opposing = opposingBraket(str, index, '}');
if(opposing != std::string::npos)
return opposing;
return index;
}
Model::Model(const std::string& str, size_t paramSweepCount, bool defaultToRange): _modelStr(str)
{
size_t bracketCounter = 0;
std::string strCpy(str);
_model = processBrackets(strCpy, bracketCounter, paramSweepCount, defaultToRange);
}
Model::Model(const Model& in)
{
operator=(in);
}
Model& Model::operator=(const Model& in)
{
delete _model;
_modelStr = in._modelStr;
_bracketComponants.clear();
_flatComponants.clear();
_model = Componant::copy(in._model);
_compiledModel = in._compiledModel;
return *this;
}
Model::~Model()
{
delete _model;
}
std::vector<Range> Model::getFlatParameterRanges()
{
std::vector<Componant*> flatComponants = getFlatComponants();
std::vector<Range> out;
out.reserve(getParameterCount());
for(Componant* componant : flatComponants)
{
const std::vector<Range> ranges = componant->getParamRanges();
for(const Range& range : ranges)
out.push_back(range);
}
return out;
}
std::vector<fvalue> Model::getFlatParameters()
{
std::vector<Range> flatRanges = getFlatParameterRanges();
std::vector<fvalue> out;
out.reserve(flatRanges.size());
for(const Range& range : flatRanges)
out.push_back(range.stepValue());
return out;
}
std::vector<Range> Model::getDefaultParameters()
{
std::vector<Componant*> flatComponants = getFlatComponants();
std::vector<Range> out;
out.reserve(getParameterCount());
for(Componant* componant : flatComponants)
{
const std::vector<Range> ranges = componant->getDefaultParameters(true);
for(const Range& range : ranges)
out.push_back(range);
}
return out;
}
std::vector<std::string> Model::getParameterNames()
{
std::vector<Componant*> flatComponants = getFlatComponants();
std::vector<std::string> out;
out.reserve(getParameterCount());
for(Componant* componant : flatComponants)
{
for(size_t i = 0; i < componant->paramCount(); ++i)
out.push_back(componant->getUniqueName() + "_" + std::to_string(i));
}
return out;
}
DataPoint Model::execute(fvalue omega, size_t index)
{
if(_model)
{
resolveSteps(index);
DataPoint dataPoint;
dataPoint.omega = omega;
dataPoint.im = _model->execute(omega);
return dataPoint;
}
else
{
Log(Log::WARN)<<"model not ready";
}
return DataPoint({std::complex<fvalue>(0,0), 0});
}
void Model::addComponantToFlat(Componant* componant, std::vector<Componant*>* flatComponants)
{
Parallel* paralell = dynamic_cast<Parallel*>(componant);
if(paralell)
{
for(Componant* element : paralell->componants)
addComponantToFlat(element, flatComponants);
return;
}
Serial* serial = dynamic_cast<Serial*>(componant);
if(serial)
{
for(Componant* element : serial->componants)
addComponantToFlat(element, flatComponants);
return;
}
flatComponants->push_back(componant);
}
std::vector<Componant*> Model::getFlatComponants(Componant *model)
{
if(model == nullptr || model == _model)
{
if(!_flatComponants.empty())
return _flatComponants;
addComponantToFlat(_model, &_flatComponants);
return getFlatComponants();
}
else
{
std::vector<Componant*> flatComponants;
addComponantToFlat(model, &flatComponants);
return flatComponants;
}
}
size_t Model::setParamSweepCountClosestTotal(size_t totalCount)
{
size_t activeParams = getActiveParameterCount();
if(activeParams < 1)
{
Log(Log::WARN)<<getModelStr()<<" requested "<<totalCount<<
" param sweep steps from this model, but this model has no active parameters, ignoreing request";
return 1;
}
size_t countPerParam = std::pow(totalCount, 1.0/activeParams);
for(Componant* componant : getFlatComponants())
{
for(eis::Range& range : componant->getParamRanges())
{
if(range.count > 1)
range.count = countPerParam;
}
}
return std::pow(countPerParam, activeParams);
}
size_t Model::getParameterCount()
{
size_t count = 0;
for(Componant* componant : getFlatComponants())
count += componant->paramCount();
return count;
}
size_t Model::getActiveParameterCount()
{
size_t count = 0;
for(Componant* componant : getFlatComponants())
{
for(const eis::Range& range : componant->getParamRanges())
{
if(range.count > 1)
++count;
}
}
return count;
}
std::vector<DataPoint> Model::executeSweep(const Range& omega, size_t index)
{
return executeSweep(omega.getRangeVector(), index);
}
std::vector<DataPoint> Model::executeSweep(const std::vector<fvalue>& omega, size_t index)
{
std::vector<DataPoint> results;
results.reserve(omega.size());
if(_compiledModel)
{
resolveSteps(index);
std::vector<fvalue> parameters = getFlatParameters();
std::vector<std::complex<fvalue>> values = _compiledModel->symbol(parameters, omega);
for(size_t i = 0; i < omega.size(); ++i)
{
DataPoint dataPoint;
dataPoint.omega = omega[i];
dataPoint.im = values[i];
results.push_back(dataPoint);
}
}
else
{
for(size_t i = 0; i < omega.size(); ++i)
{
fvalue omegaStep = omega[i];
results.push_back(execute(omegaStep, index));
}
}
return results;
}
std::vector<std::vector<DataPoint>> Model::executeSweeps(const Range& omega, const std::vector<size_t>& indecies, bool parallel)
{
return executeSweeps(omega.getRangeVector(), indecies, parallel);
}
std::vector<std::vector<DataPoint>> Model::executeSweeps(const std::vector<fvalue>& omega, const std::vector<size_t>& indecies, bool parallel)
{
unsigned int threadsCount = parallel ? std::thread::hardware_concurrency() : 1;
if(indecies.size() < threadsCount*10)
threadsCount = 1;
size_t countPerThread = indecies.size()/threadsCount;
std::vector<std::thread> threads(threadsCount);
std::vector<Model> models(threadsCount, *this);
std::vector<std::vector<DataPoint>> data(indecies.size());
for(size_t i = 0; i < threadsCount; ++i)
{
size_t start = i*countPerThread;
size_t stop = i < threadsCount-1 ? (i+1)*countPerThread : indecies.size();
threads[i] = std::thread(sweepThreadFn, &data, &models[i], start, stop, omega);
}
for(size_t i = 0; i < threadsCount; ++i)
threads[i].join();
return data;
}
void Model::sweepThreadFn(std::vector<std::vector<DataPoint>>* data, Model* model, size_t start, size_t stop, const std::vector<fvalue>& omega)
{
for(size_t i = start; i < stop; ++i)
{
data->at(i) = model->executeSweep(omega, i);
}
}
std::vector<std::vector<DataPoint>> Model::executeAllSweeps(const Range& omega)
{
size_t count = getRequiredStepsForSweeps();
unsigned int threadsCount = std::thread::hardware_concurrency();
if(count < threadsCount*10)
threadsCount = 1;
size_t countPerThread = count/threadsCount;
std::vector<std::thread> threads(threadsCount);
std::vector<Model> models(threadsCount, *this);
std::vector<std::vector<DataPoint>> data(count);
for(size_t i = 0; i < threadsCount; ++i)
{
size_t start = i*countPerThread;
size_t stop = i < threadsCount-1 ? (i+1)*countPerThread : count;
threads[i] = std::thread(sweepThreadFn, &data, &models[i], start, stop, omega.getRangeVector());
}
for(size_t i = 0; i < threadsCount; ++i)
threads[i].join();
return data;
}
void Model::resolveSteps(int64_t index)
{
std::vector<Componant*> componants = getFlatComponants();
if(index == 0)
{
for(Componant* componant : componants)
{
for(Range& range : componant->getParamRanges())
range.step = 0;
}
return;
}
std::vector<Range*> flatRanges;
for(Componant* componant : componants)
{
for(Range& range : componant->getParamRanges())
flatRanges.push_back(&range);
}
std::vector<size_t> placeMagnitude;
placeMagnitude.reserve(flatRanges.size());
//Log(Log::DEBUG)<<"Magnitudes:";
for(size_t i = 0; i < flatRanges.size(); ++i)
{
size_t magnitude = 1;
for(int64_t j = static_cast<int64_t>(i)-1; j >= 0; --j)
magnitude = magnitude*flatRanges[j]->count;
placeMagnitude.push_back(magnitude);
//Log(Log::DEBUG)<<placeMagnitude.back();
}
//Log(Log::DEBUG)<<"Steps for index "<<index<<" ranges "<<flatRanges.size()<<" Ranges:";
for(int64_t i = flatRanges.size()-1; i >= 0; --i)
{
flatRanges[i]->step = index/placeMagnitude[i];
index = index % placeMagnitude[i];
//Log(Log::DEBUG)<<placeMagnitude[i]<<'('<<flatRanges[i]->step<<')'<<(i == 0 ? "" : " + ");
}
}
size_t Model::getRequiredStepsForSweeps()
{
size_t stepsRequired = 1;
std::vector<Componant*> componants = getFlatComponants();
for(Componant* componant : componants)
{
std::vector<Range> ranges = componant->getParamRanges();
for(const Range& range : ranges)
stepsRequired *= range.count;
}
return stepsRequired;
}
std::string Model::getModelStr() const
{
std::string output;
output.reserve(_modelStr.size());
int bracket = 0;
for(const char c : _modelStr)
{
if(c == '{')
++bracket;
else if(bracket == 0)
output.push_back(c);
if(c == '}')
{
--bracket;
if(bracket < 0)
return _modelStr;
}
}
return output;
}
std::string Model::getModelStrWithParam(size_t index)
{
if(!_model)
return "";
resolveSteps(index);
std::string out = _model->getComponantString();
eisRemoveUnneededBrackets(out);
return out;
}
std::string Model::getModelStrWithParam() const
{
if(!_model)
return "";
std::string out = _model->getComponantString(false);
eisRemoveUnneededBrackets(out);
return out;
}
bool Model::isReady()
{
return _model;
}
bool Model::isParamSweep()
{
return getRequiredStepsForSweeps() > 1;
}
std::vector<size_t> Model::getRecommendedParamIndices(eis::Range omegaRange, double distance, bool threaded)
{
std::vector<std::vector<eis::DataPoint>> sweeps;
size_t count = getRequiredStepsForSweeps();
eis::Log(eis::Log::INFO)<<"Executeing "<<count<<" steps";
std::vector<std::vector<eis::DataPoint>> allSweeps;
std::vector<size_t> indices;
if(threaded)
allSweeps = executeAllSweeps(omegaRange);
for(size_t i = 0; i < count; ++i)
{
std::vector<eis::DataPoint> data;
if(threaded)
data = allSweeps[i];
else
data = executeSweep(omegaRange, i);
normalize(data);
fvalue maxJump = maximumNyquistJump(data);
if(maxJump > 0.30)
{
eis::Log(eis::Log::DEBUG)<<"skipping output for step "<<i
<<" is not well centered: "<<maxJump;
continue;
}
fvalue correlation = std::abs(pearsonCorrelation(data));
if(correlation > 0.8)
{
eis::Log(eis::Log::DEBUG)<<"skipping output for step "<<i
<<" as data is too linear: "<<correlation;
continue;
}
std::vector<std::vector<eis::DataPoint>>::iterator search;
if(threaded)
{
search = std::find_if(std::execution::par, sweeps.begin(), sweeps.end(),
[distance, &data](std::vector<eis::DataPoint>& a){return distance > eisDistance(data, a);});
}
else
{
search = std::find_if(std::execution::seq, sweeps.begin(), sweeps.end(),
[distance, &data](std::vector<eis::DataPoint>& a){return distance > eisDistance(data, a);});
}
if(search == sweeps.end())
{
indices.push_back(i);
sweeps.push_back(data);
if(threaded)
resolveSteps(i);
}
if(i % 200 == 0)
{
eis::Log(eis::Log::INFO, false)<<'.';
std::cout<<std::flush;
}
}
std::vector<size_t> out;
out.reserve(indices.size());
for(size_t candidate : indices)
{
resolveSteps(candidate);
if(!allElementsContribute(omegaRange))
eis::Log(eis::Log::DEBUG)<<"skipping "<<candidate<<" as not all elements contribute";
else if(!hasSeriesDifference(omegaRange))
eis::Log(eis::Log::DEBUG)<<"skipping "<<candidate<<" as not all elements in series are different";
else
out.push_back(candidate);
}
eis::Log(eis::Log::INFO, false)<<'\n';
return out;
}
size_t Model::getUuid() const
{
return std::hash<std::string>{}(getModelStr());
}
bool Model::compile()
{
if(!_model->compileable())
{
Log(Log::WARN)<<"This model can not be compiled, because it contains "
<<"componants that lack a compiled reprisentation, expect performance degredation";
return false;
}
CompCache* cache = CompCache::getInstance();
_compiledModel = cache->getObject(getUuid());
if(!_compiledModel)
{
std::filesystem::path tmp = getTempdir();
size_t uuid = getUuid();
std::filesystem::path path = tmp/(std::to_string(getUuid())+".so");
int ret = compile_code(getCode(), path.string());
if(ret != 0)
{
Log(Log::WARN)<<"Unable to compile model!! expect performance degredation";
return false;
}
CompiledObject object;
object.objectCode = dlopen(path.string().c_str(), RTLD_NOW);
if(!object.objectCode)
throw std::runtime_error("Unable to dlopen compiled model " + std::string(dlerror()));
std::string symbolName = getCompiledFunctionName();
object.symbol =
reinterpret_cast<std::vector<std::complex<fvalue>>(*)(const std::vector<fvalue>&, const std::vector<fvalue>&)>
(dlsym(object.objectCode, symbolName.c_str()));
if(!object.symbol)
throw std::runtime_error(path.string() + " dosent have a symbol " + symbolName);
cache->addObject(uuid, object);
_compiledModel = cache->getObject(uuid);
}
return true;
}
std::string Model::getCode()
{
if(!_model || !_model->compileable())
return "";
std::vector<std::string> parameters;
std::string formular = _model->getCode(parameters);
std::string out =
"#include <cmath>\n"
"#include <cassert>\n"
"#include <vector>\n"
"#include <complex>\n\n"
"typedef float fvalue;\n\n"
"extern \"C\"\n{\n\n"
"std::vector<std::complex<fvalue>> ";
out.append(getCompiledFunctionName());
out.append("(const std::vector<fvalue>& parameters, const std::vector<fvalue> omegas)\n{\n\tassert(parameters.size() == ");
out.append(std::to_string(parameters.size()));
out.append(");\n\n");
out.append("\tstd::vector<std::complex<fvalue>> out(omegas.size());\n");
for(size_t i = 0; i < parameters.size(); ++i)
out.append("\tfvalue " + parameters[i] + " = parameters[" + std::to_string(i) + "];\n");
out.append("\tfor(size_t i = 0; i < omegas.size(); ++i)\n\t{\n");
out.append("\t\tconst fvalue& omega = omegas[i];\n");
out.append("\t\tout[i] = ");
out.append(formular);
out.append(";\n\t}\n\treturn out;\n}\n\n}\n");
return out;
}
std::string Model::getTorchScript()
{
if(!_model || !_model->compileable())
return "";
std::vector<std::string> parameters;
std::string formular = _model->getTorchScript(parameters);
std::stringstream out;
out<<"def "<<getCompiledFunctionName()<<"(parameters: torch.Tensor, omegas: torch.Tensor) -> torch.Tensor:\n";
out<<" assert parameters.size(0) == "<<parameters.size()<<"\n\n";
for(size_t i = 0; i < parameters.size(); ++i)
out<<" "<<parameters[i]<<" = parameters["<<i<<"]\n";
out<<"\n return "<<formular<<"+0*omegas\n";
return out.str();
}
std::string Model::getCompiledFunctionName() const
{
return "model_"+std::to_string(getUuid());
}
void Model::removeSeriesResitance(std::string& model)
{
for(size_t i = 0; i < model.size(); ++i)
{
if(model[i] == Resistor::staticGetComponantChar())
{
if((i == 0 || model[i-1] == '-') && (i == model.size()-1 || model[i+1] == '-' || model[i+1] == '{'))
{
if(i != model.size()-1 && model[i+1] == '{')
{
size_t close = opposingBraket(model, i+1, getOpposingBracketChar(model[i+1]));
if(close == model.size()-1 || model[close+1] == '-')
{
eraseModelElement(model, i);
i = 0;
}
}
else
{
eraseModelElement(model, i);
i = 0;
}
}
}
else if(model[i] == '(')
{
i = opposingBraket(model, i, getOpposingBracketChar(model[i]));
}
}
}
bool Model::allElementsContribute(eis::Range omegaRange, fvalue threashold)
{
std::vector<Componant*> componants = getFlatComponants();
componants.push_back(_model);
std::vector<ParallelSerial*> combiners;
for(Componant* componant : componants)
{
ParallelSerial* candidate = dynamic_cast<ParallelSerial*>(componant);
if(candidate)
combiners.push_back(candidate);
}
std::vector<bool> contributesGlobal;
for(fvalue omega : omegaRange.getRangeVector())
{
std::vector<bool> contributes;
for(ParallelSerial* combiner : combiners)
{
std::vector<bool> contributesLocal = combiner->contributes(omega);
contributes.insert(contributes.end(), contributesLocal.begin(), contributesLocal.end());
}
if(contributesGlobal.empty())
{
contributesGlobal = contributes;
}
else
{
for(size_t i = 0; i < contributesGlobal.size(); ++i)
contributesGlobal[i] = contributes[i] || contributesGlobal[i];
}
if(std::find(contributesGlobal.begin(), contributesGlobal.end(), false) == contributesGlobal.end())
break;
}
return std::find(contributesGlobal.begin(), contributesGlobal.end(), false) == contributesGlobal.end();
}
bool Model::hasSeriesDifference(eis::Range omegaRange, fvalue threashold)
{
std::vector<Componant*> componants = getFlatComponants();
componants.push_back(_model);
std::vector<Serial*> serials;
for(Componant* componant : componants)
{
Serial* candidate = dynamic_cast<Serial*>(componant);
if(candidate)
serials.push_back(candidate);
}
std::vector<bool> isDifferentGlobal;
for(fvalue omega : omegaRange.getRangeVector())
{
bool allContribute = true;
for(Serial* serial : serials)
{
std::vector<std::complex<fvalue>> impedances;
for(Componant* componant : serial->componants)
impedances.push_back(componant->execute(omega));
for(size_t i = 0; i < impedances.size() && allContribute; ++i)
{
for(size_t j = i+1; j < impedances.size(); ++j)
{
if(std::abs(impedances[i] - impedances[j])/std::abs(impedances[i]) < threashold)
{
allContribute = false;
break;
}
}
}
if(!allContribute)
break;
}
if(allContribute)
return true;
}
return false;
}