diff --git a/Configuration/GlobalAssemblyInfo.cs b/Configuration/GlobalAssemblyInfo.cs index c215bb72c..2b8df16db 100644 --- a/Configuration/GlobalAssemblyInfo.cs +++ b/Configuration/GlobalAssemblyInfo.cs @@ -15,7 +15,7 @@ internal static class RevisionClass public const string Major = "3"; public const string Minor = "11"; public const string Build = "1"; - public const string Revision = "3788"; + public const string Revision = "3790"; public const string MainVersion = Major + "." + Minor; public const string FullVersion = Major + "." + Minor + "." + Build + "." + Revision; diff --git a/Configuration/Version.defs b/Configuration/Version.defs index 1f9ee1aba..7a435981c 100644 --- a/Configuration/Version.defs +++ b/Configuration/Version.defs @@ -1,4 +1,4 @@ %COREVERSION%=1 -%REVISION%=3788 +%REVISION%=3790 %MINOR%=11 %MAJOR%=3 diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df11.pas b/InstallerSamples/MachineLearning/02_DataFrame/df11.pas index c91fb23be..9ffaa9290 100644 --- a/InstallerSamples/MachineLearning/02_DataFrame/df11.pas +++ b/InstallerSamples/MachineLearning/02_DataFrame/df11.pas @@ -12,5 +12,8 @@ begin Kat,21,NA '''); - df.Describe.PrintLines; + var dfs := df.Describe('age'); + Println(dfs.Count,dfs.Max,dfs.Min,dfs.Mean,dfs.Std); + dfs := df.Describe('score'); + Println(dfs.Count,dfs.Max,dfs.Min,dfs.Mean,dfs.Std); end. \ No newline at end of file diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df14.pas b/InstallerSamples/MachineLearning/02_DataFrame/df14.pas index 87a780a8c..cb7146750 100644 --- a/InstallerSamples/MachineLearning/02_DataFrame/df14.pas +++ b/InstallerSamples/MachineLearning/02_DataFrame/df14.pas @@ -5,10 +5,12 @@ begin var df := DataFrame.FromCsvText(''' name,age,score Alice,20,85 + Santa,20,90 + Max,21,78 Bob,22,90 - Charlie,21,78 - Bob,22,90 + John,22,96 Clara,,78 + Sara,22,77 Kat,21,NA '''); diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df21.pas b/InstallerSamples/MachineLearning/02_DataFrame/df21.pas index 2a8fa4a88..bbcc4547a 100644 --- a/InstallerSamples/MachineLearning/02_DataFrame/df21.pas +++ b/InstallerSamples/MachineLearning/02_DataFrame/df21.pas @@ -5,9 +5,9 @@ begin var df := DataFrame.FromCsvText(''' name,age,score Alice,20,85 - Bob,22,90 + Bob,22,91 Charlie,21,78 - Bob,22,90 + Sam,22,90 Clara,,78 Kat,21,NA '''); diff --git a/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/02_PriceHistogram1.pas b/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/02_PriceHistogram1.pas index 443a1a23d..225dce7ea 100644 --- a/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/02_PriceHistogram1.pas +++ b/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/02_PriceHistogram1.pas @@ -9,11 +9,11 @@ begin var logPrice := price.Data.Select(x -> Ln(x)).ToArray; var fig := Plot.Grid(1,2); - Plot.Title('Распределение цен на квартиры в Москве'); + Plot.Title := 'Распределение цен на квартиры в Москве'; fig[0,0].Hist(price.Data, bins := 40); - fig[0,0].Title('Цена'); + fig[0,0].Title := 'Цена'; fig[0,1].Hist(logPrice, bins := 40); - fig[0,1].Title('log(Цена)'); + fig[0,1].Title := 'log(Цена)'; end. \ No newline at end of file diff --git a/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/03_AreaPriceScatter.pas b/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/03_AreaPriceScatter.pas index 9590ac008..9b6f8788d 100644 --- a/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/03_AreaPriceScatter.pas +++ b/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/03_AreaPriceScatter.pas @@ -11,5 +11,5 @@ begin Plot.XLabel('Площадь (м²)'); Plot.YLabel('Цена (руб)'); - Plot.Title('Цена квартиры vs площадь'); + Plot.Title := 'Цена квартиры vs площадь'; end. \ No newline at end of file diff --git a/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/04_MetroPriceScatter.pas b/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/04_MetroPriceScatter.pas index 336ac8e6e..8618eaa1c 100644 --- a/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/04_MetroPriceScatter.pas +++ b/InstallerSamples/MachineLearning/08_Datasets/MoscowHousing/04_MetroPriceScatter.pas @@ -11,5 +11,5 @@ begin Plot.XLabel('Минуты до метро'); Plot.YLabel('Цена (руб)'); - Plot.Title('Зависимость цены квартиры от расстояния до метро'); + Plot.Title := 'Зависимость цены квартиры от расстояния до метро'; end. \ No newline at end of file diff --git a/Release/pabcversion.txt b/Release/pabcversion.txt index 7c4c754a1..60296ca9a 100644 --- a/Release/pabcversion.txt +++ b/Release/pabcversion.txt @@ -1 +1 @@ -3.11.1.3788 +3.11.1.3790 diff --git a/ReleaseGenerators/PascalABCNET_version.nsh b/ReleaseGenerators/PascalABCNET_version.nsh index 6e04fcac2..b4cbf3d7f 100644 --- a/ReleaseGenerators/PascalABCNET_version.nsh +++ b/ReleaseGenerators/PascalABCNET_version.nsh @@ -1 +1 @@ -!define VERSION '3.11.1.3788' +!define VERSION '3.11.1.3790' diff --git a/bin/Lib/DataFrameABC.pas b/bin/Lib/DataFrameABC.pas index c98decd7f..7543fc76e 100644 --- a/bin/Lib/DataFrameABC.pas +++ b/bin/Lib/DataFrameABC.pas @@ -919,6 +919,55 @@ begin res.AddBoolColumn(name, data, valid); end; +procedure BuildColumnFromJoin(res: DataFrame; name: string; col: Column; idx: array of integer); +begin + case col.Info.ColType of + ctInt: + BuildIntColumnFromJoin(res, name, IntColumn(col), idx); + ctFloat: + BuildFloatColumnFromJoin(res, name, FloatColumn(col), idx); + ctStr: + BuildStrColumnFromJoin(res, name, StrColumn(col), idx); + ctBool: + BuildBoolColumnFromJoin(res, name, BoolColumn(col), idx); + else + Error(ER_UNSUPPORTED_COLUMN_TYPE, col.Info.ColType); + end; +end; + +function BuildLeftJoinResult( + Self, other: DataFrame; + leftIdx, rightIdx: List; + leftKey, rightKey: integer +): DataFrame; +begin + var leftArr := leftIdx.ToArray; + var rightArr := rightIdx.ToArray; + + var res := new DataFrame; + + // --- left columns (всегда берём из left) + for var ci := 0 to Self.ColumnCount - 1 do + BuildColumnFromJoin(res, Self.columns[ci].Info.Name, Self.columns[ci], leftArr); + + // --- right columns (учитывают -1) + for var ci := 0 to other.ColumnCount - 1 do + if ci <> rightKey then + BuildColumnFromJoin(res, 'right_' + other.columns[ci].Info.Name, other.columns[ci], rightArr); + + var schema := DataFrameSchema.Merge( + Self.fSchema, + other.fSchema, + [leftKey], + [rightKey], + 'right_' + ); + + res.SetSchema(schema); + + Result := res; +end; + //----------------------------- // DataFrame //----------------------------- @@ -1028,90 +1077,6 @@ begin end; end; -{procedure DataFrame.AppendJoinedRow(leftCur, rightCur: DataFrameCursor; leftKeyIdx, rightKeyIdx: array of integer); -begin - var col := 0; - - // 1. ключи (из left) - for var i := 0 to leftKeyIdx.Length - 1 do - begin - columns[col].AppendFromCursor(leftCur, leftKeyIdx[i]); - col += 1; - end; - - // 2. остальные столбцы слева - for var j := 0 to leftCur.ColumnCount - 1 do - if not leftKeyIdx.Contains(j) then - begin - columns[col].AppendFromCursor(leftCur, j); - col += 1; - end; - - // 3. остальные столбцы справа - for var j := 0 to rightCur.ColumnCount - 1 do - if not rightKeyIdx.Contains(j) then - begin - columns[col].AppendFromCursor(rightCur, j); - col += 1; - end; -end; - -procedure DataFrame.AppendLeftOnlyRow(leftCur: DataFrameCursor; leftKeyIdx, rightKeyIdx: array of integer); -begin - var col := 0; - - // ключи (из left) - for var i := 0 to leftKeyIdx.Length - 1 do - begin - columns[col].AppendFromCursor(leftCur, leftKeyIdx[i]); - col += 1; - end; - - // остальные столбцы слева - for var j := 0 to leftCur.ColumnCount - 1 do - if not leftKeyIdx.Contains(j) then - begin - columns[col].AppendFromCursor(leftCur, j); - col += 1; - end; - - while col < columns.Count do - begin - columns[col].AppendInvalid; - col += 1; - end; -end; - -procedure DataFrame.AppendRightOnlyRow(rightCur: DataFrameCursor; leftKeyIdx, rightKeyIdx: array of integer; - leftColumnCount: integer); -begin - var col := 0; - - // ключи (из right) - for var i := 0 to rightKeyIdx.Length - 1 do - begin - columns[col].AppendFromCursor(rightCur, rightKeyIdx[i]); - col += 1; - end; - - // остальные столбцы слева → NA - for var j := 0 to leftColumnCount - 1 do - if not leftKeyIdx.Contains(j) then - begin - columns[col].AppendInvalid; - col += 1; - end; - - // остальные столбцы справа - for var j := 0 to rightCur.ColumnCount - 1 do - if not rightKeyIdx.Contains(j) then - begin - columns[col].AppendFromCursor(rightCur, j); - col += 1; - end; -end;} - - function DataFrame.LeftJoinSingleKey(other: DataFrame; key: string): DataFrame; begin // 1. индексы ключей — через Schema @@ -1134,27 +1099,10 @@ begin end; end; -procedure BuildColumnFromJoin(res: DataFrame; name: string; col: Column; idx: array of integer); -begin - case col.Info.ColType of - ctInt: - BuildIntColumnFromJoin(res, name, IntColumn(col), idx); - ctFloat: - BuildFloatColumnFromJoin(res, name, FloatColumn(col), idx); - ctStr: - BuildStrColumnFromJoin(res, name, StrColumn(col), idx); - ctBool: - BuildBoolColumnFromJoin(res, name, BoolColumn(col), idx); - else - Error(ER_UNSUPPORTED_COLUMN_TYPE, col.Info.ColType); - end; -end; - function DataFrame.LeftJoinSingleKeyInt(other: DataFrame; leftKey, rightKey: integer): DataFrame; begin var index := new Dictionary>; - // --- build index (right) var rcur := other.GetCursor; while rcur.MoveNext do if rcur.IsValid(rightKey) then @@ -1171,7 +1119,6 @@ begin lst.Add(rcur.Position); end; - // --- collect index pairs var leftIdx := new List; var rightIdx := new List; @@ -1190,10 +1137,10 @@ begin var k := lcur.Int(leftKey); if index.ContainsKey(k) then - foreach var rpos in index[k] do + foreach var r in index[k] do begin leftIdx.Add(lpos); - rightIdx.Add(rpos); + rightIdx.Add(r); end else begin @@ -1202,38 +1149,13 @@ begin end; end; - var leftArr := leftIdx.ToArray; - var rightArr := rightIdx.ToArray; - - var res := new DataFrame; - - // --- left columns - for var ci := 0 to ColumnCount - 1 do - BuildColumnFromJoin(res, columns[ci].Info.Name, columns[ci], leftArr); - - // --- right columns (exclude key) - for var ci := 0 to other.ColumnCount - 1 do - if ci <> rightKey then - BuildColumnFromJoin(res, 'right_' + other.columns[ci].Info.Name, other.columns[ci], rightArr); - - // --- schema - var schema := DataFrameSchema.Merge( - fSchema, - other.fSchema, - [leftKey], - [rightKey], - 'right_' - ); - res.SetSchema(schema); - - Result := res; + Result := BuildLeftJoinResult(Self, other, leftIdx, rightIdx, leftKey, rightKey); end; function DataFrame.LeftJoinSingleKeyFloat(other: DataFrame; leftKey, rightKey: integer): DataFrame; begin var index := new Dictionary>; - // --- build index (right) var rcur := other.GetCursor; while rcur.MoveNext do if rcur.IsValid(rightKey) then @@ -1250,7 +1172,6 @@ begin lst.Add(rcur.Position); end; - // --- collect index pairs var leftIdx := new List; var rightIdx := new List; @@ -1269,10 +1190,10 @@ begin var k := lcur.Float(leftKey); if index.ContainsKey(k) then - foreach var rpos in index[k] do + foreach var r in index[k] do begin leftIdx.Add(lpos); - rightIdx.Add(rpos); + rightIdx.Add(r); end else begin @@ -1281,38 +1202,13 @@ begin end; end; - var leftArr := leftIdx.ToArray; - var rightArr := rightIdx.ToArray; - - var res := new DataFrame; - - // --- left columns - for var ci := 0 to ColumnCount - 1 do - BuildColumnFromJoin(res, columns[ci].Info.Name, columns[ci], leftArr); - - // --- right columns (exclude key) - for var ci := 0 to other.ColumnCount - 1 do - if ci <> rightKey then - BuildColumnFromJoin(res, 'right_' + other.columns[ci].Info.Name, other.columns[ci], rightArr); - - // --- schema - var schema := DataFrameSchema.Merge( - fSchema, - other.fSchema, - [leftKey], - [rightKey], - 'right_' - ); - res.SetSchema(schema); - - Result := res; + Result := BuildLeftJoinResult(Self, other, leftIdx, rightIdx, leftKey, rightKey); end; function DataFrame.LeftJoinSingleKeyStr(other: DataFrame; leftKey, rightKey: integer): DataFrame; begin var index := new Dictionary>; - // --- build index (right) var rcur := other.GetCursor; while rcur.MoveNext do if rcur.IsValid(rightKey) then @@ -1329,7 +1225,6 @@ begin lst.Add(rcur.Position); end; - // --- collect index pairs var leftIdx := new List; var rightIdx := new List; @@ -1348,10 +1243,10 @@ begin var k := lcur.Str(leftKey); if index.ContainsKey(k) then - foreach var rpos in index[k] do + foreach var r in index[k] do begin leftIdx.Add(lpos); - rightIdx.Add(rpos); + rightIdx.Add(r); end else begin @@ -1360,38 +1255,13 @@ begin end; end; - var leftArr := leftIdx.ToArray; - var rightArr := rightIdx.ToArray; - - var res := new DataFrame; - - // --- left columns - for var ci := 0 to ColumnCount - 1 do - BuildColumnFromJoin(res, columns[ci].Info.Name, columns[ci], leftArr); - - // --- right columns (exclude key) - for var ci := 0 to other.ColumnCount - 1 do - if ci <> rightKey then - BuildColumnFromJoin(res, 'right_' + other.columns[ci].Info.Name, other.columns[ci], rightArr); - - // --- schema - var schema := DataFrameSchema.Merge( - fSchema, - other.fSchema, - [leftKey], - [rightKey], - 'right_' - ); - res.SetSchema(schema); - - Result := res; + Result := BuildLeftJoinResult(Self, other, leftIdx, rightIdx, leftKey, rightKey); end; function DataFrame.LeftJoinSingleKeyBool(other: DataFrame; leftKey, rightKey: integer): DataFrame; begin var index := new Dictionary>; - // --- build index (right) var rcur := other.GetCursor; while rcur.MoveNext do if rcur.IsValid(rightKey) then @@ -1408,7 +1278,6 @@ begin lst.Add(rcur.Position); end; - // --- collect index pairs var leftIdx := new List; var rightIdx := new List; @@ -1427,10 +1296,10 @@ begin var k := lcur.Bool(leftKey); if index.ContainsKey(k) then - foreach var rpos in index[k] do + foreach var r in index[k] do begin leftIdx.Add(lpos); - rightIdx.Add(rpos); + rightIdx.Add(r); end else begin @@ -1439,31 +1308,7 @@ begin end; end; - var leftArr := leftIdx.ToArray; - var rightArr := rightIdx.ToArray; - - var res := new DataFrame; - - // --- left columns - for var ci := 0 to ColumnCount - 1 do - BuildColumnFromJoin(res, columns[ci].Info.Name, columns[ci], leftArr); - - // --- right columns (exclude key) - for var ci := 0 to other.ColumnCount - 1 do - if ci <> rightKey then - BuildColumnFromJoin(res, 'right_' + other.columns[ci].Info.Name, other.columns[ci], rightArr); - - // --- schema - var schema := DataFrameSchema.Merge( - fSchema, - other.fSchema, - [leftKey], - [rightKey], - 'right_' - ); - res.SetSchema(schema); - - Result := res; + Result := BuildLeftJoinResult(Self, other, leftIdx, rightIdx, leftKey, rightKey); end; function DataFrame.LeftJoinMultiKey(other: DataFrame; keys: array of string): DataFrame; @@ -1875,11 +1720,44 @@ begin end; end; +// Helper для следующих 4 функций +function BuildJoinResult( + Self, other: DataFrame; + leftIdx, rightIdx: List; + leftKey, rightKey: integer +): DataFrame; +begin + var leftArr := leftIdx.ToArray; + var rightArr := rightIdx.ToArray; + + var res := new DataFrame; + + // --- left columns + for var ci := 0 to Self.ColumnCount - 1 do + BuildColumnFromJoin(res, Self.columns[ci].Info.Name, Self.columns[ci], leftArr); + + // --- right columns (exclude key) + for var ci := 0 to other.ColumnCount - 1 do + if ci <> rightKey then + BuildColumnFromJoin(res, 'right_' + other.columns[ci].Info.Name, other.columns[ci], rightArr); + + var schema := DataFrameSchema.Merge( + Self.fSchema, + other.fSchema, + [leftKey], + [rightKey], + 'right_' + ); + + res.SetSchema(schema); + + Result := res; +end; + function DataFrame.JoinInnerSingleKeyInt(other: DataFrame; leftKey, rightKey: integer): DataFrame; begin var index := new Dictionary>; - // --- build index (right) var rcur := other.GetCursor; while rcur.MoveNext do if rcur.IsValid(rightKey) then @@ -1896,60 +1774,31 @@ begin lst.Add(rcur.Position); end; - // --- collect index pairs (только совпадения!) var leftIdx := new List; var rightIdx := new List; var lcur := GetCursor; while lcur.MoveNext do - begin - if not lcur.IsValid(leftKey) then - continue; + if lcur.IsValid(leftKey) then + begin + var k := lcur.Int(leftKey); - var lpos := lcur.Position; - var k := lcur.Int(leftKey); + var rows: List; + if index.TryGetValue(k, rows) then + foreach var r in rows do + begin + leftIdx.Add(lcur.Position); + rightIdx.Add(r); + end; + end; - var rows: List; - if index.TryGetValue(k, rows) then - foreach var rpos in rows do - begin - leftIdx.Add(lpos); - rightIdx.Add(rpos); - end; - end; - - var leftArr := leftIdx.ToArray; - var rightArr := rightIdx.ToArray; - - var res := new DataFrame; - - // --- left columns - for var ci := 0 to ColumnCount - 1 do - BuildColumnFromJoin(res, columns[ci].Info.Name, columns[ci], leftArr); - - // --- right columns (exclude key) - for var ci := 0 to other.ColumnCount - 1 do - if ci <> rightKey then - BuildColumnFromJoin(res, 'right_' + other.columns[ci].Info.Name, other.columns[ci], rightArr); - - // --- schema - var schema := DataFrameSchema.Merge( - fSchema, - other.fSchema, - [leftKey], - [rightKey], - 'right_' - ); - res.SetSchema(schema); - - Result := res; + Result := BuildJoinResult(Self, other, leftIdx, rightIdx, leftKey, rightKey); end; function DataFrame.JoinInnerSingleKeyFloat(other: DataFrame; leftKey, rightKey: integer): DataFrame; begin var index := new Dictionary>; - // --- build index (right) var rcur := other.GetCursor; while rcur.MoveNext do if rcur.IsValid(rightKey) then @@ -1966,60 +1815,31 @@ begin lst.Add(rcur.Position); end; - // --- collect index pairs (только совпадения) var leftIdx := new List; var rightIdx := new List; var lcur := GetCursor; while lcur.MoveNext do - begin - if not lcur.IsValid(leftKey) then - continue; + if lcur.IsValid(leftKey) then + begin + var k := lcur.Float(leftKey); - var lpos := lcur.Position; - var k := lcur.Float(leftKey); + var rows: List; + if index.TryGetValue(k, rows) then + foreach var r in rows do + begin + leftIdx.Add(lcur.Position); + rightIdx.Add(r); + end; + end; - var rows: List; - if index.TryGetValue(k, rows) then - foreach var rpos in rows do - begin - leftIdx.Add(lpos); - rightIdx.Add(rpos); - end; - end; - - var leftArr := leftIdx.ToArray; - var rightArr := rightIdx.ToArray; - - var res := new DataFrame; - - // --- left columns - for var ci := 0 to ColumnCount - 1 do - BuildColumnFromJoin(res, columns[ci].Info.Name, columns[ci], leftArr); - - // --- right columns (exclude key) - for var ci := 0 to other.ColumnCount - 1 do - if ci <> rightKey then - BuildColumnFromJoin(res, 'right_' + other.columns[ci].Info.Name, other.columns[ci], rightArr); - - // --- schema - var schema := DataFrameSchema.Merge( - fSchema, - other.fSchema, - [leftKey], - [rightKey], - 'right_' - ); - res.SetSchema(schema); - - Result := res; + Result := BuildJoinResult(Self, other, leftIdx, rightIdx, leftKey, rightKey); end; function DataFrame.JoinInnerSingleKeyStr(other: DataFrame; leftKey, rightKey: integer): DataFrame; begin var index := new Dictionary>; - // --- build index (right) var rcur := other.GetCursor; while rcur.MoveNext do if rcur.IsValid(rightKey) then @@ -2036,60 +1856,31 @@ begin lst.Add(rcur.Position); end; - // --- collect index pairs (only matches) var leftIdx := new List; var rightIdx := new List; var lcur := GetCursor; while lcur.MoveNext do - begin - if not lcur.IsValid(leftKey) then - continue; + if lcur.IsValid(leftKey) then + begin + var k := lcur.Str(leftKey); - var lpos := lcur.Position; - var k := lcur.Str(leftKey); + var rows: List; + if index.TryGetValue(k, rows) then + foreach var r in rows do + begin + leftIdx.Add(lcur.Position); + rightIdx.Add(r); + end; + end; - var rows: List; - if index.TryGetValue(k, rows) then - foreach var rpos in rows do - begin - leftIdx.Add(lpos); - rightIdx.Add(rpos); - end; - end; - - var leftArr := leftIdx.ToArray; - var rightArr := rightIdx.ToArray; - - var res := new DataFrame; - - // --- left columns - for var ci := 0 to ColumnCount - 1 do - BuildColumnFromJoin(res, columns[ci].Info.Name, columns[ci], leftArr); - - // --- right columns (exclude key) - for var ci := 0 to other.ColumnCount - 1 do - if ci <> rightKey then - BuildColumnFromJoin(res, 'right_' + other.columns[ci].Info.Name, other.columns[ci], rightArr); - - // --- schema - var schema := DataFrameSchema.Merge( - fSchema, - other.fSchema, - [leftKey], - [rightKey], - 'right_' - ); - res.SetSchema(schema); - - Result := res; + Result := BuildJoinResult(Self, other, leftIdx, rightIdx, leftKey, rightKey); end; function DataFrame.JoinInnerSingleKeyBool(other: DataFrame; leftKey, rightKey: integer): DataFrame; begin var index := new Dictionary>; - // --- build index (right) var rcur := other.GetCursor; while rcur.MoveNext do if rcur.IsValid(rightKey) then @@ -2106,53 +1897,25 @@ begin lst.Add(rcur.Position); end; - // --- collect index pairs (only matches) var leftIdx := new List; var rightIdx := new List; var lcur := GetCursor; while lcur.MoveNext do - begin - if not lcur.IsValid(leftKey) then - continue; + if lcur.IsValid(leftKey) then + begin + var k := lcur.Bool(leftKey); - var lpos := lcur.Position; - var k := lcur.Bool(leftKey); + var rows: List; + if index.TryGetValue(k, rows) then + foreach var r in rows do + begin + leftIdx.Add(lcur.Position); + rightIdx.Add(r); + end; + end; - var rows: List; - if index.TryGetValue(k, rows) then - foreach var rpos in rows do - begin - leftIdx.Add(lpos); - rightIdx.Add(rpos); - end; - end; - - var leftArr := leftIdx.ToArray; - var rightArr := rightIdx.ToArray; - - var res := new DataFrame; - - // --- left columns - for var ci := 0 to ColumnCount - 1 do - BuildColumnFromJoin(res, columns[ci].Info.Name, columns[ci], leftArr); - - // --- right columns (exclude key) - for var ci := 0 to other.ColumnCount - 1 do - if ci <> rightKey then - BuildColumnFromJoin(res, 'right_' + other.columns[ci].Info.Name, other.columns[ci], rightArr); - - // --- schema - var schema := DataFrameSchema.Merge( - fSchema, - other.fSchema, - [leftKey], - [rightKey], - 'right_' - ); - res.SetSchema(schema); - - Result := res; + Result := BuildJoinResult(Self, other, leftIdx, rightIdx, leftKey, rightKey); end; function DataFrame.JoinInnerMultiKey(other: DataFrame; keys: array of string): DataFrame; diff --git a/bin/Lib/MLPipelineABC.pas b/bin/Lib/MLPipelineABC.pas index bed5f7399..167fca92c 100644 --- a/bin/Lib/MLPipelineABC.pas +++ b/bin/Lib/MLPipelineABC.pas @@ -366,12 +366,20 @@ end; function DataPipeline.Add(step: IPipelineStep): DataPipeline; begin + // --- global invariants if step = nil then ArgumentError(ER_PIPELINE_STEP_NULL); if fFitted then Error(ER_PIPELINE_MODIFY_AFTER_FIT); + // --- target protection + // NOTE: Only built-in LabelEncoder is restricted for target. + // Custom preprocessors are not checked. + if step is LabelEncoder(var enc) then + if enc.ColumnName = fTarget then + ArgumentError(ER_LABELENCODER_TARGET_NOT_ALLOWED, fTarget); + // --- DataFrame step if step is IPreprocessor then begin @@ -382,10 +390,6 @@ begin exit(Self); end; - if step is LabelEncoder(var enc) then - if enc.ColumnName = fTarget then - ArgumentError(ER_LABELENCODER_TARGET_NOT_ALLOWED, fTarget); - // --- Matrix transformer if step is ITransformer then begin