diff --git a/InstallerSamples/MachineLearning/05_Validation/05_GridSearch_LogisticRegression.exe b/InstallerSamples/MachineLearning/05_Validation/05_GridSearch_LogisticRegression.exe deleted file mode 100644 index 6f6ef6337..000000000 Binary files a/InstallerSamples/MachineLearning/05_Validation/05_GridSearch_LogisticRegression.exe and /dev/null differ diff --git a/InstallerSamples/MachineLearning/06_Pipelines/02_DataPipeline_Regression_Categorical.exe b/InstallerSamples/MachineLearning/06_Pipelines/02_DataPipeline_Regression_Categorical.exe deleted file mode 100644 index cc1320094..000000000 Binary files a/InstallerSamples/MachineLearning/06_Pipelines/02_DataPipeline_Regression_Categorical.exe and /dev/null differ diff --git a/InstallerSamples/MachineLearning/06_Pipelines/05_MatrixPipeline_Classification.exe b/InstallerSamples/MachineLearning/06_Pipelines/05_MatrixPipeline_Classification.exe deleted file mode 100644 index bbfcd2fb1..000000000 Binary files a/InstallerSamples/MachineLearning/06_Pipelines/05_MatrixPipeline_Classification.exe and /dev/null differ diff --git a/InstallerSamples/MachineLearning/06_Pipelines/06_MatrixPipeline_Regression.exe b/InstallerSamples/MachineLearning/06_Pipelines/06_MatrixPipeline_Regression.exe deleted file mode 100644 index 73b6cd50c..000000000 Binary files a/InstallerSamples/MachineLearning/06_Pipelines/06_MatrixPipeline_Regression.exe and /dev/null differ diff --git a/InstallerSamples/MachineLearning/07_Clustering/05_ClusteringMetrics.exe b/InstallerSamples/MachineLearning/07_Clustering/05_ClusteringMetrics.exe deleted file mode 100644 index cbbb00830..000000000 Binary files a/InstallerSamples/MachineLearning/07_Clustering/05_ClusteringMetrics.exe and /dev/null differ diff --git a/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/06_RandomForest.pas b/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/06_RandomForest.pas index c992573bb..7777f0be0 100644 --- a/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/06_RandomForest.pas +++ b/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/06_RandomForest.pas @@ -7,7 +7,7 @@ begin var features := ['rooms','area','kitchen_area','floor','floors_total','metro_minutes']; var target := 'price'; - var (trainDf, testDf) := df.TrainTestSplit(0.2, 42); + var (trainDf, testDf) := df.TrainTestSplit(0.2, seed := 42); var Xtrain := trainDf.ToMatrix(features); var ytrain := trainDf.ToVector(target); diff --git a/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/07_GradientBoosting.pas b/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/07_GradientBoosting.pas index 5dae0b51c..b9dee2d90 100644 --- a/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/07_GradientBoosting.pas +++ b/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/07_GradientBoosting.pas @@ -7,7 +7,7 @@ begin var features := ['rooms','area','kitchen_area','floor','floors_total','metro_minutes']; var target := 'price'; - var (trainDf, testDf) := df.TrainTestSplit(0.2, 42); + var (trainDf, testDf) := df.TrainTestSplit(0.2, seed := 42); var Xtrain := trainDf.ToMatrix(features); var ytrain := trainDf.ToVector(target); diff --git a/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/10_FeatureImportance1.pas b/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/10_FeatureImportance1.pas index efb674254..d766ef969 100644 --- a/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/10_FeatureImportance1.pas +++ b/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/10_FeatureImportance1.pas @@ -11,9 +11,10 @@ begin var pipe := DataPipeline.Build( + TaskKind.tkRegression, target, features, - new LabelEncoder('renovation'), + new OrdinalEncoder('renovation'), model ); diff --git a/InstallerSamples/MachineLearning/08_Datasets/TitanicRu/06_ModelComparison.pas b/InstallerSamples/MachineLearning/08_Datasets/TitanicRu/06_ModelComparison.pas index f0e024d27..3c434dddd 100644 --- a/InstallerSamples/MachineLearning/08_Datasets/TitanicRu/06_ModelComparison.pas +++ b/InstallerSamples/MachineLearning/08_Datasets/TitanicRu/06_ModelComparison.pas @@ -12,10 +12,10 @@ begin df := portImputer.FitTransform(df); // Кодируем категориальные признаки числами. - var sexEncoder := new LabelEncoder('Пол'); + var sexEncoder := new OrdinalEncoder('Пол'); df := sexEncoder.FitTransform(df); - var portEncoder := new LabelEncoder('ПортПосадки'); + var portEncoder := new OrdinalEncoder('ПортПосадки'); df := portEncoder.FitTransform(df); var features := ['Класс', 'Пол', 'Возраст', 'БратьяИСупруги', 'РодителиИДети', 'ЦенаБилета', 'ПортПосадки']; diff --git a/InstallerSamples/MachineLearning/08_Datasets/TitanicRu/09_FeatureImportance.pas b/InstallerSamples/MachineLearning/08_Datasets/TitanicRu/09_FeatureImportance.pas index 3f39e4992..f9a04e8db 100644 --- a/InstallerSamples/MachineLearning/08_Datasets/TitanicRu/09_FeatureImportance.pas +++ b/InstallerSamples/MachineLearning/08_Datasets/TitanicRu/09_FeatureImportance.pas @@ -10,10 +10,10 @@ begin var portImputer := new Imputer('Саутгемптон', ['ПортПосадки']); df := portImputer.FitTransform(df); - var sexEncoder := new LabelEncoder('Пол'); + var sexEncoder := new OrdinalEncoder('Пол'); df := sexEncoder.FitTransform(df); - var portEncoder := new LabelEncoder('ПортПосадки'); + var portEncoder := new OrdinalEncoder('ПортПосадки'); df := portEncoder.FitTransform(df); var features := ['Класс', 'Пол', 'Возраст', 'БратьяИСупруги', 'РодителиИДети', 'ЦенаБилета', 'ПортПосадки']; diff --git a/InstallerSamples/MachineLearning/10_RealTasks/01_Iris_EndToEnd_Pipeline.exe b/InstallerSamples/MachineLearning/10_RealTasks/01_Iris_EndToEnd_Pipeline.exe deleted file mode 100644 index 2cd94563d..000000000 Binary files a/InstallerSamples/MachineLearning/10_RealTasks/01_Iris_EndToEnd_Pipeline.exe and /dev/null differ diff --git a/InstallerSamples/MachineLearning/10_RealTasks/02_MoscowHousing_EndToEnd_Pipeline.exe b/InstallerSamples/MachineLearning/10_RealTasks/02_MoscowHousing_EndToEnd_Pipeline.exe deleted file mode 100644 index 47c74c137..000000000 Binary files a/InstallerSamples/MachineLearning/10_RealTasks/02_MoscowHousing_EndToEnd_Pipeline.exe and /dev/null differ diff --git a/InstallerSamples/MachineLearning/13_Seminars/seminar1.pas b/InstallerSamples/MachineLearning/13_Seminars/seminar1.pas index 5ea75164c..c15c1906e 100644 --- a/InstallerSamples/MachineLearning/13_Seminars/seminar1.pas +++ b/InstallerSamples/MachineLearning/13_Seminars/seminar1.pas @@ -11,10 +11,10 @@ begin df := imputer.FitTransform(df); // Кодируем категориальные признаки - var encoder := new LabelEncoder('region_name'); + var encoder := new OrdinalEncoder('region_name'); df := encoder.FitTransform(df); - var encoder2 := new LabelEncoder('federal_district'); + var encoder2 := new OrdinalEncoder('federal_district'); df := encoder2.FitTransform(df); df.Print; diff --git a/InstallerSamples/MachineLearning/13_Seminars/seminar2.pas b/InstallerSamples/MachineLearning/13_Seminars/seminar2.pas index 3ca94a083..d0fc3e119 100644 --- a/InstallerSamples/MachineLearning/13_Seminars/seminar2.pas +++ b/InstallerSamples/MachineLearning/13_Seminars/seminar2.pas @@ -16,10 +16,10 @@ begin var imputer := new Imputer(['population', 'lat', 'lon']); df := imputer.FitTransform(df); - var le1 := new LabelEncoder('region_name'); + var le1 := new OrdinalEncoder('region_name'); df := le1.FitTransform(df); - var le2 := new LabelEncoder('federal_district'); + var le2 := new OrdinalEncoder('federal_district'); df := le2.FitTransform(df); // Явно задаём признаки diff --git a/InstallerSamples/MachineLearning/13_Seminars/seminar3.pas b/InstallerSamples/MachineLearning/13_Seminars/seminar3.pas index 94d701724..4ffd9e549 100644 --- a/InstallerSamples/MachineLearning/13_Seminars/seminar3.pas +++ b/InstallerSamples/MachineLearning/13_Seminars/seminar3.pas @@ -7,10 +7,10 @@ begin var imputer := new Imputer(['population', 'lat', 'lon']); df := imputer.FitTransform(df); - var le1 := new LabelEncoder('region_name'); + var le1 := new OrdinalEncoder('region_name'); df := le1.FitTransform(df); - var le2 := new LabelEncoder('federal_district'); + var le2 := new OrdinalEncoder('federal_district'); df := le2.FitTransform(df); var features := ['lat', 'lon', 'region_name', 'federal_district']; diff --git a/Release/pabcversion.txt b/Release/pabcversion.txt index da37ae549..c14d5d752 100644 --- a/Release/pabcversion.txt +++ b/Release/pabcversion.txt @@ -1 +1 @@ -3.11.1.3815 +3.11.1.3821 diff --git a/ReleaseGenerators/PascalABCNET_version.nsh b/ReleaseGenerators/PascalABCNET_version.nsh index 202d72ce1..d981f2abb 100644 --- a/ReleaseGenerators/PascalABCNET_version.nsh +++ b/ReleaseGenerators/PascalABCNET_version.nsh @@ -1 +1 @@ -!define VERSION '3.11.1.3815' +!define VERSION '3.11.1.3821' diff --git a/TestSuite/_MachineLearning/Core/030_labelencoder_first_appearance.pas b/TestSuite/_MachineLearning/Core/030_labelencoder_first_appearance.pas index ac12ca922..225e316e1 100644 --- a/TestSuite/_MachineLearning/Core/030_labelencoder_first_appearance.pas +++ b/TestSuite/_MachineLearning/Core/030_labelencoder_first_appearance.pas @@ -6,7 +6,7 @@ begin df.AddStrColumn('Region', Arr('South', 'North', 'South', 'East')); df := df.SetCategorical(['Region']); - var enc := new LabelEncoder('Region'); + var enc := new OrdinalEncoder('Region'); enc.Fit(df); var res := enc.Transform(df); diff --git a/TestSuite/_MachineLearning/Core/041_labelencoder_transform_type_drift_raises_contract_error.pas b/TestSuite/_MachineLearning/Core/041_labelencoder_transform_type_drift_raises_contract_error.pas index 56ee4226a..a20d068bb 100644 --- a/TestSuite/_MachineLearning/Core/041_labelencoder_transform_type_drift_raises_contract_error.pas +++ b/TestSuite/_MachineLearning/Core/041_labelencoder_transform_type_drift_raises_contract_error.pas @@ -10,7 +10,7 @@ begin testDf.AddIntColumn('City', Arr(1, 2, 3)); testDf := testDf.SetCategorical(['City']); - var enc := new LabelEncoder('City'); + var enc := new OrdinalEncoder('City'); enc.Fit(trainDf); CheckRaises(procedure -> begin diff --git a/bin/Lib/MLABC.pas b/bin/Lib/MLABC.pas index 60e254b3c..1dc0d8fb1 100644 --- a/bin/Lib/MLABC.pas +++ b/bin/Lib/MLABC.pas @@ -120,7 +120,7 @@ type Inspection = InspectionML.Inspection; IPreprocessor = PreprocessorABC.IPreprocessor; - LabelEncoder = PreprocessorABC.LabelEncoder; + OrdinalEncoder = PreprocessorABC.OrdinalEncoder; OneHotEncoder = PreprocessorABC.OneHotEncoder; ImputeStrategy = PreprocessorABC.ImputeStrategy; Imputer = PreprocessorABC.Imputer; diff --git a/bin/Lib/MLDatasets.pas b/bin/Lib/MLDatasets.pas index 52518e8f8..da313ae89 100644 --- a/bin/Lib/MLDatasets.pas +++ b/bin/Lib/MLDatasets.pas @@ -1157,6 +1157,7 @@ begin ArgumentNullError(ER_ARG_NULL, 'name'); var baseDir := PascalABCDirectory + 'Files\Datasets\'; + //var baseDir := 'C:\Program Files (x86)\PascalABC.NET\Files\Datasets\'; var metaPath := baseDir + name + '.meta'; var csvPath := baseDir + name + '.csv'; diff --git a/bin/Lib/MLModelsABC.pas b/bin/Lib/MLModelsABC.pas index 1e67ab066..d3ebc28e4 100644 --- a/bin/Lib/MLModelsABC.pas +++ b/bin/Lib/MLModelsABC.pas @@ -25,6 +25,14 @@ unit MLModelsABC; // См. статистическую политику в модуле MLABC. // ============================================================= +{ + Производительность + + DecisionTreeRegressor.Fit - 340 мс против 156 мс в Питоне при той же точности + GradientBoostingRegressor.Fit - 5500 мс против 5500 мс в Питоне + RandomForestRegressor.Fit - 1280 мс против 480 мс в Питоне +} + interface uses MLCoreABC; @@ -432,6 +440,8 @@ type Found: boolean; Feature: integer; Threshold: real; + LeftCount: integer; + LeftOrderSize: integer; WeightedScore: real; static function Invalid: RegSplitResult; @@ -439,6 +449,8 @@ type Result.Found := false; Result.Feature := -1; Result.Threshold := 0.0; + Result.LeftCount := 0; + Result.LeftOrderSize := 0; Result.WeightedScore := real.PositiveInfinity; end; end; @@ -708,8 +720,9 @@ type private fLeafL2: real; fSortedOrders: array of array of integer; - fSortedValues: array of array of real; - + fUseSortedOrdersAsRoot: boolean; + fRowWeights: array of integer; + fVisitMarks: array of integer; fVisitId: integer; @@ -726,12 +739,14 @@ type function FindBestSplitReg(X: Matrix; y: Vector; nodeOrders: array of array of integer): RegSplitResult; procedure BuildSortedOrders(X: Matrix; indices: array of integer); function BuildInitialNodeOrders(indices: array of integer): array of array of integer; - procedure SplitNodeOrders(X: Matrix; nodeOrders: array of array of integer; feature: integer; threshold: real; + procedure SplitNodeOrders(nodeOrders: array of array of integer; feature: integer; leftCount, leftOrderSize: integer; var leftOrders, rightOrders: array of array of integer); function BuildMembershipMask(rowCount: integer; indices: array of integer): array of boolean; procedure ComputeNodeStats(yData: array of real; indices: array of integer; var sumAll, sumSqAll: real); function WeightedVariance(n, leftCount: integer; leftSum, leftSumSq, sumAll, sumSqAll: real): real; function GetFeatureSubset(p: integer): array of integer; + function SampleWeight(rowIndex: integer): integer; + function TotalWeight(indices: array of integer): integer; /// Проверяет, является ли узел "чистым". /// Для регрессии это означает, что все значения y одинаковы @@ -758,6 +773,16 @@ type /// Выполняет предсказание для всех объектов X. /// Возвращает вектор вещественных значений. function Predict(X: Matrix): Vector; override; + +/// Внутренний hook для ансамблей: позволяет переиспользовать +/// уже отсортированные порядки строк по признакам. +/// Обычному пользовательскому коду не нужен. + procedure SetPreSortedOrders(sortedOrders: array of array of integer); + +/// Внутренний hook для bootstrap-подвыборок с повторами. +/// Передаются уже готовые сортированные порядки именно для корневого узла. + procedure SetPreSortedRootOrders(sortedOrders: array of array of integer); + procedure SetBootstrapRootOrders(sortedOrders: array of array of integer; rowWeights: array of integer); /// Копирует только конфигурацию модели (без обученного состояния). /// Используется для создания независимых экземпляров модели. @@ -4022,7 +4047,7 @@ function DecisionTreeRegressor.BuildTreeNode(X: Matrix; y: Vector; nodeOrders: array of array of integer; depth: integer): DecisionTreeNode; begin var indices := nodeOrders[0]; - var n := indices.Length; + var n := TotalWeight(indices); if (fMaxDepth >= 0) and (depth >= fMaxDepth) then exit(LeafNode(LeafValue(y, indices))); @@ -4048,12 +4073,13 @@ begin exit(LeafNode(LeafValue(y, indices))); var leftOrders, rightOrders: array of array of integer; - SplitNodeOrders(X, nodeOrders, split.Feature, split.Threshold, leftOrders, rightOrders); + SplitNodeOrders(nodeOrders, split.Feature, split.LeftCount, split.LeftOrderSize, leftOrders, rightOrders); var leftArr := leftOrders[0]; var rightArr := rightOrders[0]; + var rightCount := n - split.LeftCount; - if (leftArr.Length < fMinSamplesLeaf) or - (rightArr.Length < fMinSamplesLeaf) then + if (split.LeftCount < fMinSamplesLeaf) or + (rightCount < fMinSamplesLeaf) then exit(LeafNode(LeafValue(y, indices))); var delta := parentVar - split.WeightedScore; @@ -4095,7 +4121,7 @@ begin Result := RegSplitResult.Invalid; var indices := nodeOrders[0]; - var n := indices.Length; + var n := TotalWeight(indices); if n < 2 then exit; @@ -4108,6 +4134,11 @@ begin var bestScore := real.PositiveInfinity; var bestFeature := -1; var bestThreshold := 0.0; + var bestLeftCount := 0; + var bestLeftOrderSize := 0; + var invN := 1.0 / n; + var minLeaf := fMinSamplesLeaf; + var maxLeftCount := n - minLeaf; var features := GetFeatureSubset(X.ColCount); @@ -4116,64 +4147,62 @@ begin var j := features[fj]; var order := nodeOrders[j]; var orderLen := order.Length; + + if orderLen < 2 then + continue; - var leftCount := 0; - var leftSum := 0.0; - var leftSumSq := 0.0; + var firstIdx := order[0]; + var firstWeight := SampleWeight(firstIdx); + var leftCount := firstWeight; + var leftSum := firstWeight * yData[firstIdx]; + var leftSumSq := firstWeight * yData[firstIdx] * yData[firstIdx]; + var prevValue := xData[firstIdx, j]; - var prevValue := 0.0; - var firstIncluded := true; - - for var i := 0 to orderLen - 1 do + for var i := 1 to orderLen - 1 do begin + if leftCount > maxLeftCount then + break; + var idx := order[i]; var xCur := xData[idx, j]; - var yCur := yData[idx]; - - if not firstIncluded then + + if (leftCount >= minLeaf) and (prevValue <> xCur) then begin var rightCount := n - leftCount; + var rightSum := sumAll - leftSum; + var rightSumSq := sumSqAll - leftSumSq; + + var leftScore := leftSumSq - (leftSum * leftSum) / leftCount; + var rightScore := rightSumSq - (rightSum * rightSum) / rightCount; + var weighted := (leftScore + rightScore) * invN; + + if weighted < 0 then + weighted := 0.0; - if (leftCount >= fMinSamplesLeaf) and - (rightCount >= fMinSamplesLeaf) and - (prevValue <> xCur) then + if weighted < bestScore then begin - var rightCountReal := rightCount; - var leftMean := leftSum / leftCount; - var leftVar := leftSumSq / leftCount - leftMean * leftMean; - - var rightSum := sumAll - leftSum; - var rightSumSq := sumSqAll - leftSumSq; - var rightMean := rightSum / rightCount; - var rightVar := rightSumSq / rightCount - rightMean * rightMean; - - if leftVar < 0 then - leftVar := 0.0; - if rightVar < 0 then - rightVar := 0.0; - - var weighted := (leftCount * leftVar + rightCountReal * rightVar) / n; - - if weighted < bestScore then - begin - bestScore := weighted; - bestFeature := j; - bestThreshold := (prevValue + xCur) * 0.5; - end; + bestScore := weighted; + bestFeature := j; + bestThreshold := (prevValue + xCur) * 0.5; + bestLeftCount := leftCount; + bestLeftOrderSize := i; end; end; - leftCount += 1; - leftSum += yCur; - leftSumSq += yCur * yCur; + var wCur := SampleWeight(idx); + var yCur := yData[idx]; + leftCount += wCur; + leftSum += wCur * yCur; + leftSumSq += wCur * yCur * yCur; prevValue := xCur; - firstIncluded := false; end; end; Result.Found := bestFeature <> -1; Result.Feature := bestFeature; Result.Threshold := bestThreshold; + Result.LeftCount := bestLeftCount; + Result.LeftOrderSize := bestLeftOrderSize; Result.WeightedScore := bestScore; end; @@ -4190,14 +4219,103 @@ type end; end; +function BuildPreSortedOrders(X: Matrix): array of array of integer; +begin + var p := X.ColCount; + var n := X.RowCount; + var xData := X.Data; + + SetLength(Result, p); + + for var j := 0 to p - 1 do + begin + var pairs: array of SortPair; + SetLength(pairs, n); + + for var i := 0 to n - 1 do + begin + pairs[i].Value := xData[i, j]; + pairs[i].Index := i; + end; + + System.Array.Sort(pairs, new SortPairComparer); + + Result[j] := new integer[n]; + for var i := 0 to n - 1 do + Result[j][i] := pairs[i].Index; + end; +end; + +function BuildRowCounts(rows: array of integer; rowCount: integer): array of integer; +begin + Result := new integer[rowCount]; + + for var i := 0 to rows.Length - 1 do + Result[rows[i]] += 1; +end; + +function BuildSortedOrdersFromCounts( + fullSortedOrders: array of array of integer; + rowCounts: array of integer +): array of array of integer; +begin + var p := Length(fullSortedOrders); + SetLength(Result, p); + + var total := 0; + for var i := 0 to rowCounts.Length - 1 do + total += rowCounts[i]; + + for var j := 0 to p - 1 do + begin + Result[j] := new integer[total]; + var k := 0; + + foreach var idx in fullSortedOrders[j] do + for var rep := 1 to rowCounts[idx] do + begin + Result[j][k] := idx; + k += 1; + end; + end; +end; + +function BuildUniqueOrdersFromCounts( + fullSortedOrders: array of array of integer; + rowCounts: array of integer +): array of array of integer; +begin + var p := Length(fullSortedOrders); + SetLength(Result, p); + + var total := 0; + for var i := 0 to rowCounts.Length - 1 do + if rowCounts[i] > 0 then + total += 1; + + for var j := 0 to p - 1 do + begin + Result[j] := new integer[total]; + var k := 0; + + foreach var idx in fullSortedOrders[j] do + if rowCounts[idx] > 0 then + begin + Result[j][k] := idx; + k += 1; + end; + end; +end; + procedure DecisionTreeRegressor.BuildSortedOrders(X: Matrix; indices: array of integer); begin + // Обычный путь для отдельного дерева: + // построить сортировку только по реально используемым строкам. var p := X.ColCount; var n := indices.Length; var xData := X.Data; SetLength(fSortedOrders, p); - SetLength(fSortedValues, p); for var j := 0 to p - 1 do begin @@ -4214,13 +4332,9 @@ begin System.Array.Sort(pairs,new SortPairComparer); fSortedOrders[j] := new integer[n]; - fSortedValues[j] := new real[n]; for var i := 0 to n - 1 do - begin - fSortedValues[j][i] := pairs[i].Value; fSortedOrders[j][i] := pairs[i].Index; - end; end; end; @@ -4252,13 +4366,13 @@ begin end; end; -procedure DecisionTreeRegressor.SplitNodeOrders(X: Matrix; +procedure DecisionTreeRegressor.SplitNodeOrders( nodeOrders: array of array of integer; feature: integer; - threshold: real; + leftCount: integer; + leftOrderSize: integer; var leftOrders, rightOrders: array of array of integer); begin - var xData := X.Data; var p := Length(nodeOrders); fVisitId += 1; @@ -4266,31 +4380,33 @@ begin var splitArr := nodeOrders[feature]; var splitLen := splitArr.Length; + var rightCount := splitLen - leftOrderSize; + var markLeft := leftOrderSize <= rightCount; - var leftCount := 0; - - for var i := 0 to splitLen - 1 do - begin - var idx := splitArr[i]; - - if xData[idx, feature] <= threshold then - begin - fVisitMarks[idx] := mark; - leftCount += 1; - end; - end; - - var rightCount := splitLen - leftCount; + if markLeft then + for var i := 0 to leftOrderSize - 1 do + fVisitMarks[splitArr[i]] := mark + else + for var i := leftOrderSize to splitLen - 1 do + fVisitMarks[splitArr[i]] := mark; SetLength(leftOrders, p); SetLength(rightOrders, p); + leftOrders[feature] := new integer[leftOrderSize]; + rightOrders[feature] := new integer[rightCount]; + System.Array.Copy(splitArr, 0, leftOrders[feature], 0, leftOrderSize); + System.Array.Copy(splitArr, leftOrderSize, rightOrders[feature], 0, rightCount); + for var j := 0 to p - 1 do begin + if j = feature then + continue; + var src := nodeOrders[j]; var n := src.Length; - var left := new integer[leftCount]; + var left := new integer[leftOrderSize]; var right := new integer[rightCount]; var li := 0; @@ -4300,15 +4416,31 @@ begin begin var idx := src[k]; - if fVisitMarks[idx] = mark then + if markLeft then begin - left[li] := idx; - li += 1; + if fVisitMarks[idx] = mark then + begin + left[li] := idx; + li += 1; + end + else + begin + right[ri] := idx; + ri += 1; + end; end else begin - right[ri] := idx; - ri += 1; + if fVisitMarks[idx] = mark then + begin + right[ri] := idx; + ri += 1; + end + else + begin + left[li] := idx; + li += 1; + end; end; end; @@ -4331,9 +4463,11 @@ begin for var i := 0 to indices.Length - 1 do begin - var v := yData[indices[i]]; - sumAll += v; - sumSqAll += v * v; + var idx := indices[i]; + var w := SampleWeight(idx); + var v := yData[idx]; + sumAll += w * v; + sumSqAll += w * v * v; end; end; @@ -4380,6 +4514,48 @@ begin end; end; +procedure DecisionTreeRegressor.SetPreSortedOrders(sortedOrders: array of array of integer); +begin + // Внутренний fast-path для ансамблей: + // используем готовую полную сортировку X и затем фильтруем её по fRowIndices. + fSortedOrders := sortedOrders; + fRowWeights := nil; + fUseSortedOrdersAsRoot := false; +end; + +procedure DecisionTreeRegressor.SetPreSortedRootOrders(sortedOrders: array of array of integer); +begin + // Внутренний fast-path для bootstrap-выборок с повторами: + // сортировка уже соответствует корневому узлу текущего дерева. + fSortedOrders := sortedOrders; + fRowWeights := nil; + fUseSortedOrdersAsRoot := true; +end; + +procedure DecisionTreeRegressor.SetBootstrapRootOrders( + sortedOrders: array of array of integer; + rowWeights: array of integer); +begin + fSortedOrders := sortedOrders; + fRowWeights := rowWeights; + fUseSortedOrdersAsRoot := true; +end; + +function DecisionTreeRegressor.SampleWeight(rowIndex: integer): integer; +begin + if fRowWeights = nil then + Result := 1 + else + Result := fRowWeights[rowIndex]; +end; + +function DecisionTreeRegressor.TotalWeight(indices: array of integer): integer; +begin + Result := 0; + for var i := 0 to indices.Length - 1 do + Result += SampleWeight(indices[i]); +end; + //============================== // DecisionTreeClassifier //============================== @@ -4571,7 +4747,7 @@ end; function DecisionTreeRegressor.LeafValue(y: Vector; indices: array of integer): real; begin - var n := indices.Length; + var n := TotalWeight(indices); if n = 0 then exit(0.0); // безопасный fallback, не должен происходить @@ -4582,7 +4758,7 @@ begin var sum := 0.0; foreach var idx in indices do - sum += y[idx]; + sum += SampleWeight(idx) * y[idx]; var denom: real; @@ -4637,12 +4813,16 @@ begin if indices = nil then indices := Arr(0..X.RowCount - 1); - BuildSortedOrders(X, indices); + if (fSortedOrders = nil) or (Length(fSortedOrders) = 0) then + BuildSortedOrders(X, indices); SetLength(fVisitMarks, X.RowCount); fVisitId := 0; - fRoot := BuildTreeNew(X, y, indices, 0); + if fUseSortedOrdersAsRoot then + fRoot := BuildTreeNode(X, y, fSortedOrders, 0) + else + fRoot := BuildTreeNew(X, y, indices, 0); var s := fFeatureImportances.Sum; if s > 0 then @@ -4653,7 +4833,8 @@ begin fRowIndices := nil; fSortedOrders := nil; - fSortedValues := nil; + fUseSortedOrdersAsRoot := false; + fRowWeights := nil; fVisitMarks := nil; @@ -4830,6 +5011,7 @@ begin fFeatureCount := p; SetLength(fTrees, fNTrees); + var fullSortedOrders := BuildPreSortedOrders(X); // --- OOB buffers (regression) --- var oobSum: Vector := nil; @@ -4858,8 +5040,11 @@ begin var rows: array of integer; BootstrapRowIndices(n, rows); - - tree.SetRowIndices(rows); + + var rowCounts := BuildRowCounts(rows, n); + var bootSortedOrders := BuildUniqueOrdersFromCounts(fullSortedOrders, rowCounts); + tree.SetBootstrapRootOrders(bootSortedOrders, rowCounts); + tree.Fit(X, y); // --- OOB accumulate --- @@ -5729,6 +5914,9 @@ begin var yPredTrain := new Vector(nTrain); for var i := 0 to nTrain - 1 do yPredTrain[i] := fInitValue; + + var residuals := new Vector(nTrain); + var preSortedOrders := BuildPreSortedOrders(XTrain); // --- OOB logic --- var useSubsample := fSubsample < 1.0; @@ -5754,8 +5942,7 @@ begin for var m := 0 to fNEstimators - 1 do begin // 1. residuals - var r := new Vector(nTrain); - ComputePseudoResiduals(yTrain, yPredTrain, r); + ComputePseudoResiduals(yTrain, yPredTrain, residuals); var stageSeed := fRng.Next(integer.MaxValue); @@ -5766,6 +5953,7 @@ begin fLeafL2, stageSeed ); + tree.SetPreSortedOrders(preSortedOrders); // --- subsample --- var rows: array of integer := nil; @@ -5784,7 +5972,7 @@ begin end; end; - tree.Fit(XTrain, r); + tree.Fit(XTrain, residuals); fEstimators.Add(tree); var deltaTrain := tree.Predict(XTrain); diff --git a/bin/Lib/PreprocessorABC.pas b/bin/Lib/PreprocessorABC.pas index b9e624cc1..8c081f061 100644 --- a/bin/Lib/PreprocessorABC.pas +++ b/bin/Lib/PreprocessorABC.pas @@ -48,14 +48,14 @@ type /// Пропущенные значения (NA) игнорируются при обучении /// и сохраняются как пропуски при преобразовании. /// Работает только со строковыми столбцами и предназначен для признаков. -/// Не должен применяться к целевому столбцу (target). - LabelEncoder = class(IPreprocessor, IColumnBoundStep) +/// Не должен применяться к целевому столбцу (target) + OrdinalEncoder = class(IPreprocessor, IColumnBoundStep) private col: string; mapping: Dictionary; fitted: boolean; public - /// Создаёт LabelEncoder для указанного столбца + /// Создаёт OrdinalEncoder для указанного столбца constructor Create(column: string); /// Определяет множество категорий столбца и сохраняет их числовое кодирование. @@ -66,8 +66,10 @@ type /// • отображение категорий НЕ копируется методом Clone function Fit(df: DataFrame): IPreprocessor; - /// Заменяет категории их числовыми кодами - /// Возвращает новый DataFrame + /// Заменяет категории их числовыми кодами. + /// Неизвестные категории кодируются значением -1. + /// Пропущенные значения сохраняются как пропуски. + /// Возвращает новый DataFrame. function Transform(df: DataFrame): DataFrame; /// Выполняет Fit и Transform последовательно @@ -174,14 +176,14 @@ implementation uses MLExceptions; const - ER_LABELENCODER_NO_COLUMN = - 'LabelEncoder: столбец не указан!!LabelEncoder: column not specified'; - ER_LABELENCODER_NOT_STRING = - 'LabelEncoder: столбец "{0}" не является строковым!!' + - 'LabelEncoder: column "{0}" is not string'; - ER_LABELENCODER_UNSEEN_CATEGORY = - 'LabelEncoder: неизвестная категория "{0}"!!' + - 'LabelEncoder: unseen category "{0}"'; + ER_ORDINALENCODER_NO_COLUMN = + 'OrdinalEncoder: столбец не указан!!OrdinalEncoder: column not specified'; + ER_ORDINALENCODER_NOT_STRING = + 'OrdinalEncoder: столбец "{0}" не является строковым!!' + + 'OrdinalEncoder: column "{0}" is not string'; + ER_ORDINALENCODER_UNSEEN_CATEGORY = + 'OrdinalEncoder: неизвестная категория "{0}"!!' + + 'OrdinalEncoder: unseen category "{0}"'; ER_ONEHOT_NO_COLUMN = 'OneHotEncoder: столбец не указан!!OneHotEncoder: column not specified'; ER_ONEHOT_NOT_STRING = @@ -229,19 +231,19 @@ const //----------------------------- -// LabelEncoder +// OrdinalEncoder //----------------------------- -constructor LabelEncoder.Create(column: string); +constructor OrdinalEncoder.Create(column: string); begin if column = '' then - ArgumentError(ER_LABELENCODER_NO_COLUMN); + ArgumentError(ER_ORDINALENCODER_NO_COLUMN); col := column; fitted := false; end; -function LabelEncoder.Fit(df: DataFrame): IPreprocessor; +function OrdinalEncoder.Fit(df: DataFrame): IPreprocessor; begin if df = nil then ArgumentNullError(ER_ARG_NULL, 'df'); @@ -252,7 +254,7 @@ begin var idx := df.Schema.IndexOf(col); if df.Schema.ColumnTypeAt(idx) <> ColumnType.ctStr then - Error(ER_LABELENCODER_NOT_STRING, col); + Error(ER_ORDINALENCODER_NOT_STRING, col); mapping := new Dictionary; @@ -275,7 +277,7 @@ begin Result := Self; end; -function LabelEncoder.Transform(df: DataFrame): DataFrame; +function OrdinalEncoder.Transform(df: DataFrame): DataFrame; begin if not fitted then NotFittedError(ER_FIT_NOT_CALLED); @@ -299,10 +301,11 @@ begin begin var s := cur.Str(idx); - if not mapping.ContainsKey(s) then - Error(ER_LABELENCODER_UNSEEN_CATEGORY, s); - - data[row] := mapping[s]; + if s not in mapping then + data[row] := -1 + else + data[row] := mapping[s]; + valid[row] := True; end; @@ -328,20 +331,20 @@ begin Result := res.SetCategorical(catCols.ToArray); end; -function LabelEncoder.FitTransform(df: DataFrame): DataFrame; +function OrdinalEncoder.FitTransform(df: DataFrame): DataFrame; begin Fit(df); Result := Transform(df); end; -function LabelEncoder.ToString: string; +function OrdinalEncoder.ToString: string; begin - Result := 'LabelEncoder(' + col + ')'; + Result := 'OrdinalEncoder(' + col + ')'; end; -function LabelEncoder.Clone: IPreprocessor; +function OrdinalEncoder.Clone: IPreprocessor; begin - Result := new LabelEncoder(col); + Result := new OrdinalEncoder(col); end; procedure AppendAllColumnsExcept(