From 797b8c6f4c6635acc703c7752233136019f423d2 Mon Sep 17 00:00:00 2001 From: Mikhalkovich Stanislav Date: Sat, 21 Feb 2026 13:34:18 +0300 Subject: [PATCH] ML - GradientBoostingRegressor --- Configuration/GlobalAssemblyInfo.cs | 2 +- Configuration/Version.defs | 4 +- Release/pabcversion.txt | 2 +- ReleaseGenerators/PascalABCNET_version.nsh | 2 +- ReleaseGenerators/RebuildStandartModules.pas | 2 +- .../RebuildStandartModulesMono.pas | 2 +- ReleaseGenerators/sect_Core.nsh | 5 +- bin/Lib/InspectionML.pas | 76 ++++++ bin/Lib/MLABC.pas | 3 + bin/Lib/MLModelsABC.pas | 227 +++++++++++++++++- bin/Lib/ValidationML.pas | 2 +- 11 files changed, 313 insertions(+), 14 deletions(-) create mode 100644 bin/Lib/InspectionML.pas diff --git a/Configuration/GlobalAssemblyInfo.cs b/Configuration/GlobalAssemblyInfo.cs index c6b50b099..4af0c85a6 100644 --- a/Configuration/GlobalAssemblyInfo.cs +++ b/Configuration/GlobalAssemblyInfo.cs @@ -15,7 +15,7 @@ internal static class RevisionClass public const string Major = "3"; public const string Minor = "11"; public const string Build = "1"; - public const string Revision = "3761"; + public const string Revision = "3762"; public const string MainVersion = Major + "." + Minor; public const string FullVersion = Major + "." + Minor + "." + Build + "." + Revision; diff --git a/Configuration/Version.defs b/Configuration/Version.defs index 30ae61d5c..656a3358e 100644 --- a/Configuration/Version.defs +++ b/Configuration/Version.defs @@ -1,4 +1,4 @@ -%MINOR%=11 -%REVISION%=3761 %COREVERSION%=1 +%REVISION%=3762 +%MINOR%=11 %MAJOR%=3 diff --git a/Release/pabcversion.txt b/Release/pabcversion.txt index a01528214..521c7661c 100644 --- a/Release/pabcversion.txt +++ b/Release/pabcversion.txt @@ -1 +1 @@ -3.11.1.3761 +3.11.1.3762 diff --git a/ReleaseGenerators/PascalABCNET_version.nsh b/ReleaseGenerators/PascalABCNET_version.nsh index 72b6bb0b2..f7eebd55a 100644 --- a/ReleaseGenerators/PascalABCNET_version.nsh +++ b/ReleaseGenerators/PascalABCNET_version.nsh @@ -1 +1 @@ -!define VERSION '3.11.1.3761' +!define VERSION '3.11.1.3762' diff --git a/ReleaseGenerators/RebuildStandartModules.pas b/ReleaseGenerators/RebuildStandartModules.pas index 9e3754d37..36c7fdf93 100644 --- a/ReleaseGenerators/RebuildStandartModules.pas +++ b/ReleaseGenerators/RebuildStandartModules.pas @@ -21,7 +21,7 @@ uses TasksArr, TasksMatr, TasksStr, Tasks1Begin, Tasks1BoolIfCase, Tasks1Loops, Tasks1Arr, WPF, DataFrameABC, DataFrameABCCore, LinearAlgebraML, PreprocessorABC, - MetricsABC, MLABC, MLCoreABC, MLModelsABC, ValidationML, MLExceptions + MetricsABC, MLABC, MLCoreABC, MLModelsABC, ValidationML, MLExceptions, InspectionML ; begin diff --git a/ReleaseGenerators/RebuildStandartModulesMono.pas b/ReleaseGenerators/RebuildStandartModulesMono.pas index 8b73c440e..91fb09cef 100644 --- a/ReleaseGenerators/RebuildStandartModulesMono.pas +++ b/ReleaseGenerators/RebuildStandartModulesMono.pas @@ -11,7 +11,7 @@ ABCDatabases, School, SF, TurtleABC, DataFrameABC, DataFrameABCCore, LinearAlgebraML, PreprocessorABC, - MetricsABC, MLABC, MLCoreABC, MLModelsABC, ValidationML, MLExceptions + MetricsABC, MLABC, MLCoreABC, MLModelsABC, ValidationML, MLExceptions, InspectionML ; begin diff --git a/ReleaseGenerators/sect_Core.nsh b/ReleaseGenerators/sect_Core.nsh index 3ebfd7e04..a51d7c4c1 100644 --- a/ReleaseGenerators/sect_Core.nsh +++ b/ReleaseGenerators/sect_Core.nsh @@ -189,7 +189,7 @@ File ..\bin\Lib\MLModelsABC.pcu File ..\bin\Lib\ValidationML.pcu File ..\bin\Lib\MLExceptions.pcu - + File ..\bin\Lib\InspectionML.pcu File ..\bin\Lib\PABCRtl.dll File ..\bin\Lib\HelixToolkit.Wpf.dll @@ -293,6 +293,7 @@ ${AddFile} "MLModelsABC.pcu" ${AddFile} "ValidationML.pcu" ${AddFile} "MLExceptions.pcu" + ${AddFile} "InspectionML.pcu" ${AddFile} "turtle.png" @@ -433,6 +434,7 @@ File ..\bin\Lib\MLModelsABC.pas File ..\bin\Lib\ValidationML.pas File ..\bin\Lib\MLExceptions.pas + File ..\bin\Lib\InspectionML.pas File ..\bin\Lib\__RedirectIOMode.vb File ..\bin\Lib\VBSystem.vb @@ -523,6 +525,7 @@ ${AddFile} "MLModelsABC.pas" ${AddFile} "ValidationML.pas" ${AddFile} "MLExceptions.pas" + ${AddFile} "InspectionML.pas" ${AddFile} "__RedirectIOMode.vb" ${AddFile} "VBSystem.vb" diff --git a/bin/Lib/InspectionML.pas b/bin/Lib/InspectionML.pas new file mode 100644 index 000000000..23e7d4a8d --- /dev/null +++ b/bin/Lib/InspectionML.pas @@ -0,0 +1,76 @@ +/// InspectionML — инструменты анализа поведения обученных моделей. +/// +/// Модуль предназначен для исследования и интерпретации уже обученных +/// моделей машинного обучения. +/// +/// Содержит алгоритмы: +/// • оценки важности признаков +/// • анализа чувствительности модели +/// • построения частичных зависимостей +/// • диагностических процедур +/// +/// Область ответственности: +/// • работает только с обученными моделями (IModel) +/// • не участвует в обучении +/// • не вычисляет метрики напрямую +/// • не изменяет состояние модели +/// +/// Архитектурный принцип: +/// • модуль зависит от абстракции IModel +/// • модели не зависят от данного модуля +unit InspectionML; + +interface + +uses MLCoreABC, LinearAlgebraML; + +type + Inspection = static class + public + /// PermutationImportance — оценка важности признаков методом перестановок. + /// Для каждого признака случайно перемешивает его столбец и измеряет + /// падение выбранной метрики качества модели. + /// Работает с любой реализацией IModel. + static function PermutationImportance(model: IModel; X: Matrix; y: Vector; + scoreFunc: (Vector, Vector) -> real): Vector; + end; + +implementation + +static function Inspection.PermutationImportance(model: IModel; X: Matrix; y: Vector; + scoreFunc: (Vector, Vector) -> real): Vector; +begin + if X.Rows <> y.Length then + raise new Exception('PermutationImportance: dimension mismatch'); + + var baselinePred := model.Predict(X); + var baselineScore := scoreFunc(y, baselinePred); + + var n := X.Rows; + var p := X.Cols; + + var resultVec := new Vector(p); + + for var j := 0 to p-1 do + begin + var Xperm := X.Clone; + + // Fisher–Yates shuffle столбца j + for var i := n-1 downto 1 do + begin + var k := Random(i+1); + var tmp := Xperm[i,j]; + Xperm[i,j] := Xperm[k,j]; + Xperm[k,j] := tmp; + end; + + var permPred := model.Predict(Xperm); + var permScore := scoreFunc(y, permPred); + + resultVec[j] := baselineScore - permScore; + end; + + Result := resultVec; +end; + +end. \ No newline at end of file diff --git a/bin/Lib/MLABC.pas b/bin/Lib/MLABC.pas index b8d90db37..c88e16d8d 100644 --- a/bin/Lib/MLABC.pas +++ b/bin/Lib/MLABC.pas @@ -11,6 +11,7 @@ uses MetricsABC; uses PreprocessorABC; uses DataFrameABC; uses MLExceptions; +uses InspectionML; type Vector = LinearAlgebraML.Vector; @@ -51,6 +52,8 @@ type MLNotFittedException = MLExceptions.MLNotFittedException; MLDimensionException = MLExceptions.MLDimensionException; + Inspection = InspectionML.Inspection; + implementation const diff --git a/bin/Lib/MLModelsABC.pas b/bin/Lib/MLModelsABC.pas index 962236bfe..ee474c974 100644 --- a/bin/Lib/MLModelsABC.pas +++ b/bin/Lib/MLModelsABC.pas @@ -277,6 +277,7 @@ type fFeatureImportances: Vector; fRandomSeed: integer; fMaxFeatures := 0; + fRowIndices: array of integer := nil; function BuildTree(X: Matrix; y: Vector; indices: array of integer; depth: integer): DecisionTreeNode; @@ -292,7 +293,8 @@ type function LeafNode(value: real): DecisionTreeNode; procedure CopyBaseState(dest: DecisionTreeBase); function GetFeatureSubset(nFeatures: integer): array of integer; virtual; - + + procedure SetRowIndices(rows: array of integer); public constructor Create(maxDepth: integer := 10; minSamplesSplit: integer := 2; minSamplesLeaf: integer := 1); @@ -432,6 +434,36 @@ type function FeatureImportances: Vector; override; end; + + GradientBoostingRegressor = class(IRegressor) + private + fNEstimators: integer; + fLearningRate: real; + fMaxDepth: integer; + fMinSamplesSplit: integer; + fMinSamplesLeaf: integer; + fSubsample: real; + fRandomSeed: integer; + + fEstimators: List; + fInitValue: real; + fFitted: boolean; + fFeatureCount: integer; + + public + constructor Create( + nEstimators: integer := 100; + learningRate: real := 0.1; + maxDepth: integer := 3; + minSamplesSplit: integer := 2; + minSamplesLeaf: integer := 1; + subsample: real := 1.0; + randomSeed: integer := 42); + + function Fit(X: Matrix; y: Vector): IModel; + function Predict(X: Matrix): Vector; + function Clone: IModel; + end; {$endregion Models} @@ -766,6 +798,23 @@ const 'Неизвестный тип FeatureScore!!Unknown FeatureScore type'; ER_SELECTKBEST_FIT_INVALID = 'Для SelectKBest необходимо вызывать Fit(X, y)!!SelectKBest requires Fit(X, y)'; + ER_FIT_NOT_CALLED = + 'Необходимо вызвать Fit перед Predict!!Fit must be called before Predict'; + ER_X_NULL = + 'X не может быть nil!!X cannot be nil'; + ER_Y_NULL = + 'y не может быть nil!!y cannot be nil'; + ER_XY_SIZE_MISMATCH = + 'Размерности X и y не согласованы!!X and y size mismatch'; + ER_FEATURE_COUNT_MISMATCH = + 'Число признаков не совпадает!!Feature count mismatch'; + ER_N_ESTIMATORS_NOT_POSITIVE = + 'Параметр nEstimators должен быть > 0!!nEstimators must be > 0'; + ER_LEARNING_RATE_NOT_POSITIVE = + 'Параметр learningRate должен быть > 0!!learningRate must be > 0'; + ER_SUBSAMPLE_OUT_OF_RANGE = + 'Параметр subsample должен быть в диапазоне (0, 1]!!subsample must be in (0, 1]'; + {$endregion ErrConstants} //----------------------------- @@ -1473,6 +1522,14 @@ begin Result := subset; end; +procedure DecisionTreeBase.SetRowIndices(rows: array of integer); +begin + if Length(rows) = 0 then + ArgumentError('Row subset cannot be empty!!Row subset cannot be empty'); + + fRowIndices := Copy(rows); +end; + function DecisionTreeBase.FeatureImportances: Vector; begin Result := fFeatureImportances.Clone; @@ -1979,6 +2036,8 @@ begin Result := MajorityClass(y, indices); end; +// DecisionTreeRegressor + constructor DecisionTreeRegressor.Create(maxDepth: integer; minSamplesSplit: integer; minSamplesLeaf: integer); begin inherited Create(maxDepth, minSamplesSplit, minSamplesLeaf); @@ -2021,18 +2080,28 @@ begin fFeatureImportances := new Vector(X.Cols); - var indices := new integer[X.Rows]; - for var i := 0 to X.Rows - 1 do - indices[i] := i; + var indices: array of integer; + + // 🔹 Ключевое изменение + if fRowIndices = nil then + begin + SetLength(indices, X.Rows); + for var i := 0 to X.Rows - 1 do + indices[i] := i; + end + else + indices := fRowIndices; fRoot := BuildTree(X, y, indices, 0); var s := fFeatureImportances.Sum; if s > 0 then - for var i := 0 to fFeatureImportances.Length-1 do + for var i := 0 to fFeatureImportances.Length - 1 do fFeatureImportances[i] /= s; fFitted := true; + + fRowIndices := nil; Result := Self; end; @@ -2077,6 +2146,7 @@ begin Result := m; end; + //----------------------------- // RandomForestBase //----------------------------- @@ -2354,6 +2424,153 @@ begin Result := resultVec; end; +//----------------------------- +// GradientBoostingRegressor +//----------------------------- +constructor GradientBoostingRegressor.Create( + nEstimators: integer; + learningRate: real; + maxDepth: integer; + minSamplesSplit: integer; + minSamplesLeaf: integer; + subsample: real; + randomSeed: integer); +begin + if nEstimators <= 0 then + ArgumentOutOfRangeError(ER_N_ESTIMATORS_NOT_POSITIVE); + + if learningRate <= 0 then + ArgumentOutOfRangeError(ER_LEARNING_RATE_NOT_POSITIVE); + + if (subsample <= 0) or (subsample > 1) then + ArgumentOutOfRangeError(ER_SUBSAMPLE_OUT_OF_RANGE); + + fNEstimators := nEstimators; + fLearningRate := learningRate; + fMaxDepth := maxDepth; + fMinSamplesSplit := minSamplesSplit; + fMinSamplesLeaf := minSamplesLeaf; + fSubsample := subsample; + fRandomSeed := randomSeed; + + fEstimators := new List; + fFitted := false; +end; + +function GradientBoostingRegressor.Fit(X: Matrix; y: Vector): IModel; +begin + if X = nil then + ArgumentNullError(ER_X_NULL); + + if y = nil then + ArgumentNullError(ER_Y_NULL); + + if X.Rows <> y.Length then + DimensionError(ER_XY_SIZE_MISMATCH); + + if X.Rows = 0 then + ArgumentError(ER_EMPTY_DATASET); + + fEstimators.Clear; + fFeatureCount := X.Cols; + + var n := y.Length; + + // F0 = mean(y) + var sum := 0.0; + for var i := 0 to n - 1 do + sum += y[i]; + fInitValue := sum / n; + + var yPred := new Vector(n); + for var i := 0 to n - 1 do + yPred[i] := fInitValue; + + Randomize(fRandomSeed); + + for var m := 0 to fNEstimators - 1 do + begin + // residuals + var r := new Vector(n); + for var i := 0 to n - 1 do + r[i] := y[i] - yPred[i]; + + var tree := new DecisionTreeRegressor( + fMaxDepth, + fMinSamplesSplit, + fMinSamplesLeaf); + + // subsample (без копирования X) + if fSubsample < 1.0 then + begin + var k := Round(n * fSubsample); + var indices := new integer[k]; + for var i := 0 to k - 1 do + indices[i] := Random(n); + + tree.SetRowIndices(indices); + end; + + tree.Fit(X, r); + fEstimators.Add(tree); + + // update prediction + var delta := tree.Predict(X); + for var i := 0 to n - 1 do + yPred[i] += fLearningRate * delta[i]; + end; + + fFitted := true; + Result := Self; +end; + +function GradientBoostingRegressor.Predict(X: Matrix): Vector; +begin + if not fFitted then + NotFittedError(ER_FIT_NOT_CALLED); + + if X = nil then + ArgumentNullError(ER_X_NULL); + + if X.Cols <> fFeatureCount then + DimensionError(ER_FEATURE_COUNT_MISMATCH); + + var n := X.Rows; + var yPred := new Vector(n); + + for var i := 0 to n - 1 do + yPred[i] := fInitValue; + + foreach var tree in fEstimators do + begin + var delta := tree.Predict(X); + for var i := 0 to n - 1 do + yPred[i] += fLearningRate * delta[i]; + end; + + Result := yPred; +end; + +function GradientBoostingRegressor.Clone: IModel; +begin + var copy := new GradientBoostingRegressor( + fNEstimators, + fLearningRate, + fMaxDepth, + fMinSamplesSplit, + fMinSamplesLeaf, + fSubsample, + fRandomSeed); + + copy.fInitValue := fInitValue; + copy.fFeatureCount := fFeatureCount; + copy.fFitted := fFitted; + + foreach var tree in fEstimators do + copy.fEstimators.Add(tree.Clone as DecisionTreeRegressor); + + Result := copy; +end; //----------------------------- diff --git a/bin/Lib/ValidationML.pas b/bin/Lib/ValidationML.pas index 5e5b10f83..d6980fadd 100644 --- a/bin/Lib/ValidationML.pas +++ b/bin/Lib/ValidationML.pas @@ -2,7 +2,7 @@ interface -uses LinearAlgebraML, MLModelsABC; +uses LinearAlgebraML, MLCoreABC; type Validation = static class