From f834ed4ea9b263e80201927033e731ce3dcea37b Mon Sep 17 00:00:00 2001 From: Mikhalkovich Stanislav Date: Wed, 25 Feb 2026 23:46:09 +0300 Subject: [PATCH] =?UTF-8?q?ML=20-=20Knn=20=D0=BC=D0=BE=D0=B4=D0=B5=D0=BB?= =?UTF-8?q?=D0=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bin/Lib/LinearAlgebraML.pas | 2 +- bin/Lib/MLABC.pas | 3 + bin/Lib/MLModelsABC.pas | 623 +++++++++++++++++++++++++++++++++++- 3 files changed, 624 insertions(+), 4 deletions(-) diff --git a/bin/Lib/LinearAlgebraML.pas b/bin/Lib/LinearAlgebraML.pas index 6cf6cc396..6f10c8200 100644 --- a/bin/Lib/LinearAlgebraML.pas +++ b/bin/Lib/LinearAlgebraML.pas @@ -77,7 +77,7 @@ type property Cols: integer read data.GetLength(1); property Item[i, j: integer]: real - read data[i, j] write SetData; default; + read data[i, j] write SetData; default; constructor Create(r, c: integer); constructor Create(values: array[,] of real); diff --git a/bin/Lib/MLABC.pas b/bin/Lib/MLABC.pas index 18a76962e..c9b99c034 100644 --- a/bin/Lib/MLABC.pas +++ b/bin/Lib/MLABC.pas @@ -48,7 +48,10 @@ type RandomForestClassifier = MLModelsABC.RandomForestClassifier; GradientBoostingRegressor = MLModelsABC.GradientBoostingRegressor; GradientBoostingClassifier = MLModelsABC.GradientBoostingClassifier; + KNNClassifier = MLModelsABC.KNNClassifier; + KNNRegressor = MLModelsABC.KNNRegressor; + KNNWeighting = MLModelsABC.KNNWeighting; TGBLoss = MLModelsABC.TGBLoss; TMaxFeaturesMode = MLModelsABC.TMaxFeaturesMode; diff --git a/bin/Lib/MLModelsABC.pas b/bin/Lib/MLModelsABC.pas index e76e17b99..7d1f9ec1c 100644 --- a/bin/Lib/MLModelsABC.pas +++ b/bin/Lib/MLModelsABC.pas @@ -940,6 +940,84 @@ type function ToString: string; override; end; +//----------------------------- +// KNN +//----------------------------- + + Neighbor = record + dist: double; + idx: integer; + end; + + KNNWeighting = (Uniform, Distance); + + KNNBase = abstract class(IModel) + protected + // ==== train state ==== + fXTrain: Matrix; + fK: integer; + fIsFitted: boolean; + + fWeighting: KNNWeighting; + + // ==== work buffers ==== + fNeighbors: array of Neighbor; + + // ==== common methods ==== + procedure CheckFitted; + procedure ValidatePredictInput(X: Matrix); + + function SquaredL2(trainRow: integer; XTest: Matrix; testRow: integer): double; + + procedure QuickSelect(k: integer); + function Partition(left, right: integer): integer; + + public + constructor Create(k: integer; weighting: KNNWeighting := KNNWeighting.Uniform); + + function Fit(X: Matrix; y: Vector): IModel; virtual; abstract; + function Predict(X: Matrix): Vector; virtual; abstract; + + function Clone: IModel; virtual; abstract; + end; + + KNNClassifier = class(KNNBase, IProbabilisticClassifier) + private + // ==== classification state ==== + fYEnc: array of integer; + fClasses: array of double; + fClassCount: integer; + + // ==== voting buffers ==== + fVotes: array of double; + fMark: array of integer; + fTouched: array of integer; + fEpoch: integer; + + procedure EncodeClasses(y: Vector); + + public + constructor Create(k: integer; weighting: KNNWeighting := KNNWeighting.Uniform); + + function Fit(X: Matrix; y: Vector): IModel; override; + function Predict(X: Matrix): Vector; override; + function PredictProba(X: Matrix): Matrix; + function GetClasses: array of double; + + function Clone: IModel; override; + end; + + KNNRegressor = class(KNNBase, IRegressor) + private + fYTrain: Vector; + + public + constructor Create(k: integer; weighting: KNNWeighting := KNNWeighting.Uniform); + + function Fit(X: Matrix; y: Vector): IModel; override; + function Predict(X: Matrix): Vector; override; + function Clone: IModel; override; + end; {$endregion Models} {$region Pipeline} @@ -1315,8 +1393,13 @@ const ER_MAX_DEPTH_INVALID = 'maxDepth должен быть > 0!!maxDepth must be > 0'; ER_SUBSAMPLE_INVALID = - 'subsample должен быть > 0!!subsample must be > 0'; - + 'subsample должен быть > 0!!subsample must be > 0'; + ER_NAN_IN_Y = + 'y содержит NaN!!y contains NaN'; + ER_NAN_IN_X = + 'x содержит NaN!!x contains NaN'; + ER_K_EXCEEDS_SAMPLES = + 'k превышает число обучающих объектов!!k exceeds number of training samples'; {$endregion ErrConstants} @@ -4478,13 +4561,547 @@ begin end; end; +//----------------------------- +// KNNBase +//----------------------------- +constructor KNNBase.Create(k: integer; weighting: KNNWeighting); +begin + if k < 1 then + ArgumentOutOfRangeError(ER_K_MUST_BE_POSITIVE); + + fK := k; + fWeighting := weighting; + fIsFitted := False; +end; + +procedure KNNBase.CheckFitted; +begin + if not fIsFitted then + NotFittedError(ER_FIT_NOT_CALLED); +end; + +procedure KNNBase.ValidatePredictInput(X: Matrix); +begin + if X.ColCount <> fXTrain.ColCount then + DimensionError(ER_FEATURE_COUNT_MISMATCH); +end; + +function KNNBase.SquaredL2(trainRow: integer; XTest: Matrix; testRow: integer): double; +begin + var sum := 0.0; + var d := fXTrain.ColCount; + + for var j := 0 to d - 1 do + begin + var diff := fXTrain[trainRow, j] - XTest[testRow, j]; + sum += diff * diff; + end; + + exit(sum); +end; + +// QuickSelect(fK - 1); - так вызываем + +procedure KNNBase.QuickSelect(k: integer); +begin + var left := 0; + var right := fNeighbors.Length - 1; + + while true do + begin + var pivotIndex := Partition(left, right); + + if pivotIndex = k then + exit + else if pivotIndex > k then + right := pivotIndex - 1 + else + left := pivotIndex + 1; + end; +end; + +function KNNBase.Partition(left, right: integer): integer; +begin + var pivot := fNeighbors[(left + right) div 2].dist; + + var i := left; + var j := right; + + while true do + begin + while fNeighbors[i].dist < pivot do i += 1; + while fNeighbors[j].dist > pivot do j -= 1; + + if i >= j then + exit(j); + + Swap(fNeighbors[i], fNeighbors[j]); + + i += 1; + j -= 1; + end; +end; + +//----------------------------- +// KNNClassifier +//----------------------------- + +constructor KNNClassifier.Create(k: integer; weighting: KNNWeighting); +begin + inherited Create(k, weighting); +end; + +procedure KNNClassifier.EncodeClasses(y: Vector); +begin + var n := y.Length; + + // собрать уникальные значения + var hs := new HashSet; + for var i := 0 to n - 1 do + begin + var v := y[i]; + if double.IsNaN(v) then + ArgumentError(ER_NAN_IN_Y); + hs.Add(v); + end; + + fClasses := hs.ToArray; + &Array.Sort(fClasses); + + fClassCount := fClasses.Length; + + // построить map label -> index + var dict := new Dictionary; + for var i := 0 to fClassCount - 1 do + dict[fClasses[i]] := i; + + SetLength(fYEnc, n); + + for var i := 0 to n - 1 do + fYEnc[i] := dict[y[i]]; +end; + +function KNNClassifier.Fit(X: Matrix; y: Vector): IModel; +begin + if X = nil then + ArgumentNullError(ER_X_NULL); + + if y = nil then + ArgumentNullError(ER_Y_NULL); + + if X.RowCount <> y.Length then + DimensionError(ER_XY_SIZE_MISMATCH); + + if fK > X.RowCount then + ArgumentOutOfRangeError(ER_K_EXCEEDS_SAMPLES); + + // проверить NaN в X + for var i := 0 to X.RowCount - 1 do + for var j := 0 to X.ColCount - 1 do + if double.IsNaN(X[i,j]) then + ArgumentError(ER_NAN_IN_X); + + // копия train data + fXTrain := X.Clone; // предполагаем, что Clone делает глубокую копию + + // кодирование классов + EncodeClasses(y); + + var n := fXTrain.RowCount; + var C := fClassCount; + + // выделение буферов + SetLength(fNeighbors, n); + + SetLength(fVotes, C); + SetLength(fMark, C); + SetLength(fTouched, C); + fEpoch := 0; + + fIsFitted := true; + + exit(self); +end; + +function KNNClassifier.GetClasses: array of double; +begin + Result := fClasses; +end; + +function KNNClassifier.Clone: IModel; +begin + var clone := new KNNClassifier(fK, fWeighting); + + if fIsFitted then + begin + clone.fXTrain := fXTrain.Clone; + clone.fClasses := fClasses.Clone as array of double; + clone.fYEnc := fYEnc.Clone as array of integer; + + clone.fClassCount := fClassCount; + + SetLength(clone.fNeighbors, fNeighbors.Length); + SetLength(clone.fVotes, fVotes.Length); + SetLength(clone.fMark, fMark.Length); + SetLength(clone.fTouched, fTouched.Length); + + clone.fEpoch := 0; + clone.fIsFitted := true; + end; + + Result := clone; +end; + +function KNNClassifier.Predict(X: Matrix): Vector; +begin + if not fIsFitted then + NotFittedError(ER_FIT_NOT_CALLED); + + if X = nil then + ArgumentNullError(ER_X_NULL); + + ValidatePredictInput(X); + + var m := X.RowCount; + var n := fXTrain.RowCount; + + Result := new Vector(m); + + for var i := 0 to m - 1 do + begin + // заполнить расстояния + for var t := 0 to n - 1 do + begin + fNeighbors[t].dist := SquaredL2(t, X, i); + fNeighbors[t].idx := t; + end; + + // выбрать k ближайших + QuickSelect(fK - 1); + + // exact match: если среди k ближайших есть dist=0, возвращаем его класс + var exactCls := -1; + for var t := 0 to fK - 1 do + if fNeighbors[t].dist = 0 then + begin + exactCls := fYEnc[fNeighbors[t].idx]; + break; + end; + + if exactCls <> -1 then + begin + Result[i] := fClasses[exactCls]; + continue; + end; + + // voting (stamping) + fEpoch += 1; + var touchCount := 0; + + if fWeighting = Uniform then + begin + for var t := 0 to fK - 1 do + begin + var trainIdx := fNeighbors[t].idx; + var cls := fYEnc[trainIdx]; + + if fMark[cls] <> fEpoch then + begin + fMark[cls] := fEpoch; + fVotes[cls] := 0.0; + fTouched[touchCount] := cls; + touchCount += 1; + end; + + fVotes[cls] += 1.0; + end; + end + else + begin + // weighted: веса 1 / dist (dist = squared distance) + for var t := 0 to fK - 1 do + begin + var trainIdx := fNeighbors[t].idx; + var cls := fYEnc[trainIdx]; + var dist := fNeighbors[t].dist; + + if fMark[cls] <> fEpoch then + begin + fMark[cls] := fEpoch; + fVotes[cls] := 0.0; + fTouched[touchCount] := cls; + touchCount += 1; + end; + + var w := 1.0 / dist; + fVotes[cls] += w; + end; + end; + + // argmax только по touched + var bestCls := fTouched[0]; + var bestVotes := fVotes[bestCls]; + + for var k2 := 1 to touchCount - 1 do + begin + var cls := fTouched[k2]; + var v := fVotes[cls]; + + if (v > bestVotes) or ((v = bestVotes) and (cls < bestCls)) then + begin + bestCls := cls; + bestVotes := v; + end; + end; + + Result[i] := fClasses[bestCls]; + end; +end; + +function KNNClassifier.PredictProba(X: Matrix): Matrix; +begin + if not fIsFitted then + NotFittedError(ER_FIT_NOT_CALLED); + + if X = nil then + ArgumentNullError(ER_X_NULL); + + ValidatePredictInput(X); + + var m := X.RowCount; + var n := fXTrain.RowCount; + + Result := new Matrix(m, fClassCount); // предполагаем нулевую инициализацию + + for var i := 0 to m - 1 do + begin + // заполнить расстояния + for var t := 0 to n - 1 do + begin + fNeighbors[t].dist := SquaredL2(t, X, i); + fNeighbors[t].idx := t; + end; + + // выбрать k ближайших + QuickSelect(fK - 1); + + // exact match: если среди k ближайших есть dist=0, вероятность 1 у его класса + var exactCls := -1; + for var t := 0 to fK - 1 do + if fNeighbors[t].dist = 0 then + begin + exactCls := fYEnc[fNeighbors[t].idx]; + break; + end; + + if exactCls <> -1 then + begin + Result[i, exactCls] := 1.0; + continue; + end; + + // voting (stamping) + fEpoch += 1; + var touchCount := 0; + + if fWeighting = Uniform then + begin + for var t := 0 to fK - 1 do + begin + var trainIdx := fNeighbors[t].idx; + var cls := fYEnc[trainIdx]; + + if fMark[cls] <> fEpoch then + begin + fMark[cls] := fEpoch; + fVotes[cls] := 0.0; + fTouched[touchCount] := cls; + touchCount += 1; + end; + + fVotes[cls] += 1.0; + end; + + // нормализация: сумма = k + for var k2 := 0 to touchCount - 1 do + begin + var cls := fTouched[k2]; + Result[i, cls] := fVotes[cls] / fK; + end; + end + else + begin + var sumW := 0.0; + + for var t := 0 to fK - 1 do + begin + var trainIdx := fNeighbors[t].idx; + var cls := fYEnc[trainIdx]; + var dist := fNeighbors[t].dist; + + if fMark[cls] <> fEpoch then + begin + fMark[cls] := fEpoch; + fVotes[cls] := 0.0; + fTouched[touchCount] := cls; + touchCount += 1; + end; + + var w := 1.0 / dist; + fVotes[cls] += w; + sumW += w; + end; + + // нормализация: сумма = sumW + for var k2 := 0 to touchCount - 1 do + begin + var cls := fTouched[k2]; + Result[i, cls] := fVotes[cls] / sumW; + end; + end; + end; +end; + +//----------------------------- +// KNNRegressor +//----------------------------- + +constructor KNNRegressor.Create(k: integer; weighting: KNNWeighting); +begin + inherited Create(k, weighting); +end; + +function KNNRegressor.Fit(X: Matrix; y: Vector): IModel; +begin + if X = nil then + ArgumentNullError(ER_X_NULL); + + if y = nil then + ArgumentNullError(ER_Y_NULL); + + if X.RowCount <> y.Length then + DimensionError(ER_XY_SIZE_MISMATCH); + + if fK > X.RowCount then + ArgumentOutOfRangeError(ER_K_EXCEEDS_SAMPLES); + + // Проверка NaN в X + for var i := 0 to X.RowCount - 1 do + for var j := 0 to X.ColCount - 1 do + if double.IsNaN(X[i,j]) then + ArgumentError(ER_NAN_IN_X); + + // Проверка NaN в y + for var i := 0 to y.Length - 1 do + if double.IsNaN(y[i]) then + ArgumentError(ER_NAN_IN_Y); + + fXTrain := X.Clone; + fYTrain := y.Clone; + + var n := fXTrain.RowCount; + SetLength(fNeighbors, n); + + fIsFitted := True; + + Result := Self; +end; + +function KNNRegressor.Predict(X: Matrix): Vector; +begin + if not fIsFitted then + NotFittedError(ER_FIT_NOT_CALLED); + + if X = nil then + ArgumentNullError(ER_X_NULL); + + ValidatePredictInput(X); + + var m := X.RowCount; + var n := fXTrain.RowCount; + + Result := new Vector(m); + + for var i := 0 to m - 1 do + begin + // заполнить расстояния + for var t := 0 to n - 1 do + begin + fNeighbors[t].dist := SquaredL2(t, X, i); + fNeighbors[t].idx := t; + end; + + // выбрать k ближайших (первые k элементов, порядок произвольный) + QuickSelect(fK - 1); + + // exact match: ищем dist=0 среди k ближайших + var exactIdx := -1; + for var t := 0 to fK - 1 do + if fNeighbors[t].dist = 0 then + begin + exactIdx := fNeighbors[t].idx; + break; + end; + + if exactIdx <> -1 then + begin + Result[i] := fYTrain[exactIdx]; + continue; + end; + + if fWeighting = KNNWeighting.Uniform then + begin + // среднее по k + var sum := 0.0; + for var t := 0 to fK - 1 do + sum += fYTrain[fNeighbors[t].idx]; + + Result[i] := sum / fK; + end + else + begin + // weighted: веса 1 / dist (dist = squared distance) + var sumW := 0.0; + var sumWY := 0.0; + + for var t := 0 to fK - 1 do + begin + var idx := fNeighbors[t].idx; + var dist := fNeighbors[t].dist; + + // dist=0 здесь уже не встречается из-за exact-match выше + var w := 1.0 / dist; + sumW += w; + sumWY += w * fYTrain[idx]; + end; + + Result[i] := sumWY / sumW; + end; + end; +end; + +function KNNRegressor.Clone: IModel; +begin + var clone := new KNNRegressor(fK, fWeighting); + + if fIsFitted then + begin + clone.fXTrain := fXTrain.Clone; + clone.fYTrain := fYTrain.Clone; + + SetLength(clone.fNeighbors, fNeighbors.Length); + + clone.fIsFitted := True; + end; + + Result := clone; +end; //----------------------------- // Pipeline //----------------------------- - constructor Pipeline.Create; begin fTransformers := new List;