From 38599a310fc473fd1935df2bee34edc3746651e1 Mon Sep 17 00:00:00 2001 From: Mikhalkovich Stanislav Date: Sat, 7 Feb 2026 13:06:49 +0300 Subject: [PATCH] =?UTF-8?q?DataFrameABC=201.1=20-=20=D0=BF=D0=BE=D0=BB?= =?UTF-8?q?=D0=BD=D0=BE=D1=81=D1=82=D1=8C=D1=8E=20=D0=BD=D0=B0=20DataFrame?= =?UTF-8?q?Schema?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Configuration/GlobalAssemblyInfo.cs | 2 +- Configuration/Version.defs | 2 +- Release/pabcversion.txt | 2 +- ReleaseGenerators/PascalABCNET_version.nsh | 2 +- bin/Lib/DataFrameABC.pas | 647 ++++++++++++++------- bin/Lib/DataFrameABCCore.pas | 382 ++++++++++-- 6 files changed, 767 insertions(+), 270 deletions(-) diff --git a/Configuration/GlobalAssemblyInfo.cs b/Configuration/GlobalAssemblyInfo.cs index 593f9ab6f..9739fd326 100644 --- a/Configuration/GlobalAssemblyInfo.cs +++ b/Configuration/GlobalAssemblyInfo.cs @@ -15,7 +15,7 @@ internal static class RevisionClass public const string Major = "3"; public const string Minor = "11"; public const string Build = "1"; - public const string Revision = "3746"; + public const string Revision = "3748"; public const string MainVersion = Major + "." + Minor; public const string FullVersion = Major + "." + Minor + "." + Build + "." + Revision; diff --git a/Configuration/Version.defs b/Configuration/Version.defs index 148a6fa59..b7650be0f 100644 --- a/Configuration/Version.defs +++ b/Configuration/Version.defs @@ -1,4 +1,4 @@ %COREVERSION%=1 -%REVISION%=3746 +%REVISION%=3748 %MINOR%=11 %MAJOR%=3 diff --git a/Release/pabcversion.txt b/Release/pabcversion.txt index ff5b0ccad..c9ca7cea2 100644 --- a/Release/pabcversion.txt +++ b/Release/pabcversion.txt @@ -1 +1 @@ -3.11.1.3746 +3.11.1.3748 diff --git a/ReleaseGenerators/PascalABCNET_version.nsh b/ReleaseGenerators/PascalABCNET_version.nsh index 2e04e4233..57e634ba4 100644 --- a/ReleaseGenerators/PascalABCNET_version.nsh +++ b/ReleaseGenerators/PascalABCNET_version.nsh @@ -1 +1 @@ -!define VERSION '3.11.1.3746' +!define VERSION '3.11.1.3748' diff --git a/bin/Lib/DataFrameABC.pas b/bin/Lib/DataFrameABC.pas index d800804b2..9aac5d667 100644 --- a/bin/Lib/DataFrameABC.pas +++ b/bin/Lib/DataFrameABC.pas @@ -1,7 +1,8 @@ // Copyright (c) Ivan Bondarev, Stanislav Mikhalkovich (for details please see \doc\copyright.txt) // This code is distributed under the GNU LGPL (for details please see \doc\license.txt) +// DataFrameABC v.1.1 -/// Стандартный модуль для работы с табличными данными (датасетами) +/// Стандартный модуль для работы с табличными данными (датасетами) /// !! DataFrame module for tabular data processing unit DataFrameABC; @@ -34,20 +35,19 @@ type DataFrame = class private columns: List; - columnIndexByName: Dictionary; - - procedure RebuildColumnIndex; - function GetColumnIndex(name: string): integer; - function ResolveKeyIndices(keys: array of string): array of integer; + fschema: DataFrameSchema; + + procedure RebuildSchema; // Join методы - function CreateInnerJoinResult(other: DataFrame; leftKeyIdx, rightKeyIdx: array of integer): DataFrame; - procedure AddColumnSchema(src: Column); // это сервисный метод добавления схемы - не данных! и только для join! + procedure AppendJoinedRow(leftCur, rightCur: DataFrameCursor; leftKeyIdx, rightKeyIdx: array of integer); procedure AppendLeftOnlyRow(leftCur: DataFrameCursor; leftKeyIdx, rightKeyIdx: array of integer); procedure AppendRightOnlyRow(rightCur: DataFrameCursor; leftKeyIdx, rightKeyIdx: array of integer; leftColumnCount: integer); // Single key методы + {function DataFrame.JoinInnerSingleKey(other: DataFrame; leftKey, rightKey: integer; + resultSchema: DataFrameSchema): DataFrame;} function JoinInnerSingleKey(other: DataFrame; key: string): DataFrame; function JoinInnerSingleKeyInt(other: DataFrame; leftKey, rightKey: integer): DataFrame; function JoinInnerSingleKeyFloat(other: DataFrame; leftKey, rightKey: integer): DataFrame; @@ -60,6 +60,9 @@ type function LeftJoinSingleKeyStr(other: DataFrame; leftKey, rightKey: integer): DataFrame; function LeftJoinSingleKeyBool(other: DataFrame; leftKey, rightKey: integer): DataFrame; + function ReorderBySchema(schema: DataFrameSchema): DataFrame; + function RightJoinViaSchema(other: DataFrame; keys: array of string): DataFrame; + function FullJoinSingleKey(other: DataFrame; key: string): DataFrame; // Multi key методы @@ -74,11 +77,22 @@ type procedure AssertSchemaConsistent; // Проверка инвариантов в Debug + constructor Create(cols: List; schema: DataFrameSchema); + + function BuildJoinSchema(right: DataFrame; leftKeys, rightKeys: array of integer; + rightPrefix: string): DataFrameSchema; + function BuildJoinSchema(right: DataFrame; leftKeys, rightKeys: array of string): DataFrameSchema; + + function CreateEmptyBySchema(schema: DataFrameSchema): DataFrame; + public + /// Схема DataFrame (имена/типы/categorical), не содержит данных + property Schema: DataFrameSchema read fschema; + /// Добавляет в DataFrame столбец-представление (view), /// использующий те же данные, что и исходный столбец procedure AddColumnView(src: Column); - public + /// Создает пустой DataFrame constructor Create; @@ -86,8 +100,6 @@ type function RowCount: integer; /// Возвращает количество столбцов в DataFrame function ColumnCount: integer; - /// Возвращает массив имен всех столбцов - function ColumnNames: array of string; /// Возвращает тип столбца по номеру function GetColumnType(colIndex: integer): ColumnType; /// Возвращает индекс столбца по имени @@ -360,25 +372,22 @@ type constructor DataFrame.Create; begin columns := []; + fschema := new DataFrameSchema([], []); end; -function DataFrame.GetColumnIndex(name: string): integer; +constructor DataFrame.Create(cols: List; schema: DataFrameSchema); begin - if columnIndexByName.ContainsKey(name) then - Result := columnIndexByName[name] - else - Result := -1; -end; + if cols = nil then + raise new System.ArgumentException('cols is nil'); + if schema = nil then + raise new System.ArgumentException('schema is nil'); + if cols.Count <> schema.ColumnCount then + raise new System.ArgumentException('Columns count and schema mismatch'); -function DataFrame.ResolveKeyIndices(keys: array of string): array of integer; -begin - Result := new integer[keys.Length]; - for var i := 0 to keys.Length - 1 do - begin - if not columnIndexByName.ContainsKey(keys[i]) then - raise new Exception('Column not found: ' + keys[i]); - Result[i] := columnIndexByName[keys[i]]; - end; + self.columns := cols; + self.fSchema := schema; + + RebuildSchema; end; function DataFrame.BuildJoinKey(cur: DataFrameCursor; layout: JoinKeyLayout; var hasNA: boolean): JoinKey; @@ -455,37 +464,6 @@ begin end; end; -function DataFrame.CreateInnerJoinResult(other: DataFrame; leftKeyIdx, rightKeyIdx: array of integer): DataFrame; -begin - Result := new DataFrame; - - // ключевые столбцы (слева) - for var i := 0 to leftKeyIdx.Length - 1 do - Result.AddColumnSchema(columns[leftKeyIdx[i]]); - - // остальные столбцы слева - for var j := 0 to columns.Count - 1 do - if not leftKeyIdx.Contains(j) then - Result.AddColumnSchema(columns[j]); - - // остальные столбцы справа - for var j := 0 to other.columns.Count - 1 do - if not rightKeyIdx.Contains(j) then - Result.AddColumnSchema(other.columns[j]); - - Result.RebuildColumnIndex; -end; - -procedure DataFrame.AddColumnSchema(src: Column); -begin - case src.Info.ColType of - ctInt: AddIntColumn(src.Info.Name, new integer[0], nil, src.Info.IsCategorical); - ctFloat: AddFloatColumn(src.Info.Name, new real[0], nil); - ctStr: AddStrColumn(src.Info.Name, new string[0], nil, src.Info.IsCategorical); - ctBool: AddBoolColumn(src.Info.Name, new boolean[0], nil); - end; -end; - procedure DataFrame.AppendJoinedRow(leftCur, rightCur: DataFrameCursor; leftKeyIdx, rightKeyIdx: array of integer); begin var col := 0; @@ -572,28 +550,28 @@ end; function DataFrame.LeftJoinSingleKey(other: DataFrame; key: string): DataFrame; begin - var leftKey := ResolveKeyIndices([key])[0]; - var rightKey := other.ResolveKeyIndices([key])[0]; + // 1. индексы ключей — через Schema + var leftKey := fSchema.IndexOf(key); + var rightKey := other.fSchema.IndexOf(key); - var lt := columns[leftKey].Info.ColType; - var rt := other.columns[rightKey].Info.ColType; + // 2. проверка типов ключей — через Schema + var lt := fSchema.ColumnTypeAt(leftKey); + var rt := other.fSchema.ColumnTypeAt(rightKey); if lt <> rt then raise new Exception('Join key types mismatch'); + // 3. типоспецифичный алгоритм (КАК РАНЬШЕ) case lt of - ctInt: - Result := LeftJoinSingleKeyInt(other, leftKey, rightKey); - ctFloat: - Result := LeftJoinSingleKeyFloat(other, leftKey, rightKey); - ctStr: - Result := LeftJoinSingleKeyStr(other, leftKey, rightKey); - ctBool: - Result := LeftJoinSingleKeyBool(other, leftKey, rightKey); + ctInt: Result := LeftJoinSingleKeyInt(other, leftKey, rightKey); + ctFloat: Result := LeftJoinSingleKeyFloat(other, leftKey, rightKey); + ctStr: Result := LeftJoinSingleKeyStr(other, leftKey, rightKey); + ctBool: Result := LeftJoinSingleKeyBool(other, leftKey, rightKey); end; end; + function DataFrame.LeftJoinSingleKeyInt(other: DataFrame; leftKey, rightKey: integer): DataFrame; begin var index := new Dictionary>; @@ -609,7 +587,16 @@ begin index[k].Add(rcur.Position); end; - var res := CreateInnerJoinResult(other, [leftKey], [rightKey]); + //=== Заменяем на Schema + var schema := DataFrameSchema.Merge( + fSchema, + other.fSchema, + [leftKey], + [rightKey], + 'right_' + ); + var res := CreateEmptyBySchema(schema); + //=== var lcur := GetCursor; while lcur.MoveNext do @@ -652,7 +639,16 @@ begin index[k].Add(rcur.Position); end; - var res := CreateInnerJoinResult(other, [leftKey], [rightKey]); + //=== + var schema := DataFrameSchema.Merge( + fSchema, + other.fSchema, + [leftKey], + [rightKey], + 'right_' + ); + var res := CreateEmptyBySchema(schema); + //=== var lcur := GetCursor; while lcur.MoveNext do @@ -693,7 +689,16 @@ begin index[k].Add(rcur.Position); end; - var res := CreateInnerJoinResult(other, [leftKey], [rightKey]); + //=== + var schema := DataFrameSchema.Merge( + fSchema, + other.fSchema, + [leftKey], + [rightKey], + 'right_' + ); + var res := CreateEmptyBySchema(schema); + //=== var lcur := GetCursor; while lcur.MoveNext do @@ -734,7 +739,16 @@ begin index[k].Add(rcur.Position); end; - var res := CreateInnerJoinResult(other, [leftKey], [rightKey]); + //=== + var schema := DataFrameSchema.Merge( + fSchema, + other.fSchema, + [leftKey], + [rightKey], + 'right_' + ); + var res := CreateEmptyBySchema(schema); + //=== var lcur := GetCursor; while lcur.MoveNext do @@ -763,15 +777,42 @@ end; function DataFrame.LeftJoinMultiKey(other: DataFrame; keys: array of string): DataFrame; begin - var leftKeyIdx := ResolveKeyIndices(keys); - var rightKeyIdx := other.ResolveKeyIndices(keys); + // 1. индексы ключей — через Schema + var n := keys.Length; + var leftKeyIdx := new integer[n]; + var rightKeyIdx := new integer[n]; + for var i := 0 to n - 1 do + begin + leftKeyIdx[i] := fSchema.IndexOf(keys[i]); + rightKeyIdx[i] := other.fSchema.IndexOf(keys[i]); + end; + + // 2. проверка типов ключей — через Schema + for var i := 0 to n - 1 do + if fSchema.ColumnTypeAt(leftKeyIdx[i]) <> + other.fSchema.ColumnTypeAt(rightKeyIdx[i]) then + raise new Exception('Join key types mismatch'); + + // 3. layout'ы (как раньше) var leftLayout := BuildJoinKeyLayout(leftKeyIdx); var rightLayout := other.BuildJoinKeyLayout(rightKeyIdx); + // 4. hash index по правой таблице var hash := other.BuildHashIndex(rightLayout); - var res := CreateInnerJoinResult(other, leftKeyIdx, rightKeyIdx); + // 5. создаём результат + var schema := DataFrameSchema.Merge( + fSchema, + other.fSchema, + leftKeyIdx, + rightKeyIdx, + 'right_' + ); + var res := CreateEmptyBySchema(schema); + //=== + + // 6. probe var lcur := GetCursor; var rcur := other.GetCursor; @@ -795,11 +836,41 @@ begin Result := res; end; +function DataFrame.ReorderBySchema(schema: DataFrameSchema): DataFrame; +begin + var cols := new Column[schema.ColumnCount]; + + for var i := 0 to schema.ColumnCount - 1 do + cols[i] := columns[fSchema.IndexOf(schema.Names[i])]; + + Result := new DataFrame(cols.ToList, schema); +end; + +function DataFrame.RightJoinViaSchema(other: DataFrame; keys: array of string): DataFrame; +begin + // 1. Схема в порядке Self + other + var schema := Self.BuildJoinSchema(other, keys, keys); + + // 2. Строки берём из перевёрнутого left join + var tmp := other.LeftJoinMultiKey(Self, keys); + + // 3. ПЕРЕСОБИРАЕМ колонки по схеме + Result := tmp.ReorderBySchema(schema); +end; + + + function DataFrame.FullJoinSingleKey(other: DataFrame; key: string): DataFrame; begin // 1. Индексы ключей - var leftKeyIdx := ResolveKeyIndices([key]); - var rightKeyIdx := other.ResolveKeyIndices([key]); + var li := fSchema.IndexOf(key); + var ri := other.fSchema.IndexOf(key); + + if (li < 0) or (ri < 0) then + raise new Exception($'Join key "{key}" not found'); + + var leftKeyIdx := [li]; + var rightKeyIdx := [ri]; // 2. Layout'ы ключей var leftLayout := BuildJoinKeyLayout(leftKeyIdx); @@ -809,7 +880,15 @@ begin var hash := other.BuildHashIndex(rightLayout); // 4. Результат (схема такая же, как у inner/left) - var res := CreateInnerJoinResult(other, leftKeyIdx, rightKeyIdx); + var schema := DataFrameSchema.Merge( + fSchema, + other.fSchema, + leftKeyIdx, + rightKeyIdx, + 'right_' + ); + var res := CreateEmptyBySchema(schema); + //=== // 5. Курсоры var leftCur := Self.GetCursor; @@ -862,9 +941,15 @@ end; function DataFrame.FullJoinMultiKey(other: DataFrame; keys: array of string): DataFrame; begin - // 1. Индексы ключей (ОТЛИЧИЕ №1) - var leftKeyIdx := ResolveKeyIndices(keys); - var rightKeyIdx := other.ResolveKeyIndices(keys); + // 1. Индексы ключей + var leftKeyIdx := new integer[keys.Length]; + var rightKeyIdx := new integer[keys.Length]; + + for var i := 0 to keys.Length - 1 do + begin + leftKeyIdx[i] := fSchema.IndexOf(keys[i]); + rightKeyIdx[i] := other.fSchema.IndexOf(keys[i]); + end; // 2. Layout'ы ключей (ОТЛИЧИЕ №2) var leftLayout := BuildJoinKeyLayout(leftKeyIdx); @@ -874,7 +959,15 @@ begin var hash := other.BuildHashIndex(rightLayout); // 4. Результат - var res := CreateInnerJoinResult(other, leftKeyIdx, rightKeyIdx); + var schema := DataFrameSchema.Merge( + fSchema, + other.fSchema, + leftKeyIdx, + rightKeyIdx, + 'right_' + ); + var res := CreateEmptyBySchema(schema); + //=== // 5. Курсоры var leftCur := Self.GetCursor; @@ -925,13 +1018,35 @@ begin Result := res; end; +{function DataFrame.JoinInnerSingleKey(other: DataFrame; leftKey, rightKey: integer; + resultSchema: DataFrameSchema): DataFrame; +begin + // типы ключей — ТОЛЬКО из Schema + var lt := fSchema.ColumnTypeAt(leftKey); + var rt := other.fSchema.ColumnTypeAt(rightKey); + + if lt <> rt then + raise new Exception('Join key types mismatch'); + + case lt of + ctInt: + Result := JoinInnerSingleKeyInt(other, leftKey, rightKey, resultSchema); + ctFloat: + Result := JoinInnerSingleKeyFloat(other, leftKey, rightKey, resultSchema); + ctStr: + Result := JoinInnerSingleKeyStr(other, leftKey, rightKey, resultSchema); + ctBool: + Result := JoinInnerSingleKeyBool(other, leftKey, rightKey, resultSchema); + end; +end;} + function DataFrame.JoinInnerSingleKey(other: DataFrame; key: string): DataFrame; begin - var leftKey := ResolveKeyIndices([key])[0]; - var rightKey := other.ResolveKeyIndices([key])[0]; + var leftKey := fSchema.IndexOf(key); + var rightKey := other.fSchema.IndexOf(key); - var lt := columns[leftKey].Info.ColType; - var rt := other.columns[rightKey].Info.ColType; + var lt := fSchema.ColumnTypeAt(leftKey); + var rt := other.fSchema.ColumnTypeAt(rightKey); if lt <> rt then raise new Exception('Join key types mismatch'); @@ -959,7 +1074,16 @@ begin index[k].Add(rcur.Position); end; - var res := CreateInnerJoinResult(other, [leftKey], [rightKey]); + //=== + var schema := DataFrameSchema.Merge( + fSchema, + other.fSchema, + [leftKey], + [rightKey], + 'right_' + ); + var res := CreateEmptyBySchema(schema); + //=== var lcur := GetCursor; while lcur.MoveNext do @@ -994,8 +1118,17 @@ begin index[k].Add(rcur.Position); end; - var res := CreateInnerJoinResult(other, [leftKey], [rightKey]); - + //=== + var schema := DataFrameSchema.Merge( + fSchema, + other.fSchema, + [leftKey], + [rightKey], + 'right_' + ); + var res := CreateEmptyBySchema(schema); + //=== + var lcur := GetCursor; while lcur.MoveNext do begin @@ -1029,8 +1162,17 @@ begin index[k].Add(rcur.Position); end; - var res := CreateInnerJoinResult(other, [leftKey], [rightKey]); - + //=== + var schema := DataFrameSchema.Merge( + fSchema, + other.fSchema, + [leftKey], + [rightKey], + 'right_' + ); + var res := CreateEmptyBySchema(schema); + //=== + var lcur := GetCursor; while lcur.MoveNext do begin @@ -1064,8 +1206,17 @@ begin index[k].Add(rcur.Position); end; - var res := CreateInnerJoinResult(other, [leftKey], [rightKey]); - + //=== + var schema := DataFrameSchema.Merge( + fSchema, + other.fSchema, + [leftKey], + [rightKey], + 'right_' + ); + var res := CreateEmptyBySchema(schema); + //=== + var lcur := GetCursor; while lcur.MoveNext do begin @@ -1086,21 +1237,42 @@ end; function DataFrame.JoinInnerMultiKey(other: DataFrame; keys: array of string): DataFrame; begin - // 1. разрешаем индексы ключей - var leftKeyIdx := ResolveKeyIndices(keys); - var rightKeyIdx := other.ResolveKeyIndices(keys); + // 1. индексы ключей — через Schema + var n := keys.Length; + var leftKeyIdx := new integer[n]; + var rightKeyIdx := new integer[n]; - // 2. строим layout'ы + for var i := 0 to n - 1 do + begin + leftKeyIdx[i] := fSchema.IndexOf(keys[i]); + rightKeyIdx[i] := other.fSchema.IndexOf(keys[i]); + end; + + // 2. проверка типов ключей — через Schema + for var i := 0 to n - 1 do + if fSchema.ColumnTypeAt(leftKeyIdx[i]) <> + other.fSchema.ColumnTypeAt(rightKeyIdx[i]) then + raise new Exception('Join key types mismatch'); + + // 3. строим layout'ы var leftLayout := BuildJoinKeyLayout(leftKeyIdx); var rightLayout := other.BuildJoinKeyLayout(rightKeyIdx); - // 3. hash index по правой таблице + // 4. hash index по правой таблице var hash := other.BuildHashIndex(rightLayout); - // 4. создаём результат - var res := CreateInnerJoinResult(other, leftKeyIdx, rightKeyIdx); - - // 5. probe + // 5. создаём результат + var schema := DataFrameSchema.Merge( + fSchema, + other.fSchema, + leftKeyIdx, + rightKeyIdx, + 'right_' + ); + var res := CreateEmptyBySchema(schema); + //=== + + // 6. probe var lcur := GetCursor; var rcur := other.GetCursor; @@ -1122,7 +1294,6 @@ begin Result := res; end; - function DataFrame.BuildJoinKeyLayout(keyIndices: array of integer): JoinKeyLayout; begin Result.ColIndices := keyIndices; @@ -1133,6 +1304,11 @@ end; function DataFrame.Join(other: DataFrame; keys: array of string; kind: JoinKind): DataFrame; begin + /// NOTE: + /// Join result structure is fully defined by DataFrameSchema. + /// Any post-hoc column reordering (Select, Rename) inside Join + /// is deprecated and must not be used. + if kind = jkInner then if keys.Length = 1 then exit(JoinInnerSingleKey(other, keys[0])) @@ -1146,15 +1322,7 @@ begin exit(LeftJoinMultiKey(other, keys)); if kind = jkRight then - begin - // 1. Меняем фреймы местами - var tmp := other.Join(Self, keys, jkLeft); - - // 2. Приводим порядок столбцов - var cols := Self.ColumnNames + other.ColumnNames; - - exit(tmp.Select(cols)); - end; + exit(RightJoinViaSchema(other, keys)); if kind = jkFull then if keys.Length = 1 then @@ -1190,18 +1358,29 @@ begin AssertSchemaConsistent; end; -procedure DataFrame.RebuildColumnIndex; +procedure DataFrame.RebuildSchema; begin - columnIndexByName := new Dictionary; - for var i := 0 to columns.Count - 1 do - columnIndexByName[columns[i].Info.Name] := i; -end; + var n := columns.Count; + var names := new string[n]; + var types := new ColumnType[n]; + var anyCat := false; -function DataFrame.ColumnNames: array of string; -begin - Result := new string[columns.Count]; - for var i := 0 to columns.Count - 1 do - Result[i] := columns[i].Info.Name; + for var i := 0 to n - 1 do + begin + var info := columns[i].Info; + names[i] := info.Name; + types[i] := info.ColType; + if info.IsCategorical then anyCat := true; + end; + + if not anyCat then + fschema := new DataFrameSchema(names, types) + else + begin + var cats := new boolean[n]; + for var i := 0 to n - 1 do cats[i] := columns[i].Info.IsCategorical; + fschema := new DataFrameSchema(names, types, cats); + end; end; function DataFrame.GetColumnType(colIndex: integer): ColumnType; @@ -1227,7 +1406,7 @@ begin end; function DataFrame.GetCursor: DataFrameCursor := - new DataFrameCursor(columns.ToArray,columnIndexByName); + new DataFrameCursor(columns.ToArray,fSchema); function DataFrame.GetIntColumn(name: string): array of integer; begin @@ -1250,7 +1429,7 @@ begin c.Data := data; c.IsValid := valid; columns.Add(c); - RebuildColumnIndex; + RebuildSchema; end; procedure DataFrame.AddFloatColumn(name: string; data: array of real; valid: array of boolean); @@ -1264,7 +1443,7 @@ begin c.IsValid := valid; columns.Add(c); - RebuildColumnIndex; + RebuildSchema; end; procedure DataFrame.AddStrColumn(name: string; data: array of string; valid: array of boolean; isCategorical: boolean); @@ -1278,7 +1457,7 @@ begin c.IsValid := valid; columns.Add(c); - RebuildColumnIndex; + RebuildSchema; end; procedure DataFrame.AddBoolColumn(name: string; data: array of boolean; valid: array of boolean); @@ -1292,7 +1471,7 @@ begin c.IsValid := valid; columns.Add(c); - RebuildColumnIndex; + RebuildSchema; end; @@ -1361,7 +1540,7 @@ function DataFrame.Mean(colName: string): real function DataFrame.Median(colIndex: integer): real; begin CheckColumnIndex(colIndex); - Result := Statistics.Median(Self, ColumnNames[colIndex]); + Result := Statistics.Median(Self, fSchema.Names[colIndex]); end; function DataFrame.Median(colName: string): real; @@ -1951,7 +2130,7 @@ begin end; -function DataFrame.Select(colIndices: array of integer): DataFrame; +{function DataFrame.Select(colIndices: array of integer): DataFrame; begin var res := new DataFrame; @@ -2010,59 +2189,46 @@ begin Result := res; AssertSchemaConsistent; +end;} + +function DataFrame.Select(colIndices: array of integer): DataFrame; +begin + foreach var i in colIndices do + CheckColumnIndex(i); + + var newSchema := fSchema.Select(colIndices); + var newColumns := new List; + + foreach var i in colIndices do + newColumns.Add(columns[i]); + + Result := new DataFrame(newColumns, newSchema); end; function DataFrame.Select(colNames: array of string): DataFrame; begin - Result := Select(colNames.Select(n -> ColumnIndex(n)).ToArray); + var indices := new integer[colNames.Length]; + + for var i := 0 to colNames.Length - 1 do + indices[i] := fSchema.IndexOf(colNames[i]); + + Result := Select(indices); end; + function DataFrame.Rename(colIndex: integer; newName: string): DataFrame; begin CheckColumnIndex(colIndex); - - var oldName := columns[colIndex].Info.Name; - if (newName <> oldName) and columnIndexByName.ContainsKey(newName) then - raise new Exception($'Column "{newName}" already exists'); - var res := new DataFrame; - for var i := 0 to columns.Count - 1 do - begin - var col := columns[i]; - var name := if i = colIndex then newName else col.Info.Name; + var oldName := fSchema.NameAt(colIndex); + if oldName = newName then + exit(Self); - case col.Info.ColType of - ctInt: - begin - var c := IntColumn(col); - res.AddIntColumn(name, c.Data, c.IsValid, c.Info.IsCategorical); - end; - - ctStr: - begin - var c := StrColumn(col); - res.AddStrColumn(name, c.Data, c.IsValid, c.Info.IsCategorical); - end; - - ctFloat: - begin - var c := FloatColumn(col); - res.AddFloatColumn(name, c.Data, c.IsValid); - end; - - ctBool: - begin - var c := BoolColumn(col); - res.AddBoolColumn(name, c.Data, c.IsValid); - end; - end; - end; - - Result := res; - - AssertSchemaConsistent; + var newSchema := fSchema.Rename(oldName, newName); + Result := new DataFrame(columns, newSchema); end; + function DataFrame.Rename(oldName, newName: string): DataFrame; begin Result := Rename(ColumnIndex(oldName), newName); @@ -2115,33 +2281,86 @@ end; function DataFrame.Drop(colIndices: array of integer): DataFrame; begin - // проверяем индексы foreach var i in colIndices do CheckColumnIndex(i); - // помечаем удаляемые столбцы var drop := new boolean[columns.Count]; foreach var i in colIndices do drop[i] := true; - // собираем список оставшихся var keep := new List; for var i := 0 to columns.Count - 1 do if not drop[i] then keep.Add(i); - // переиспользуем Select Result := Select(keep.ToArray); end; function DataFrame.Drop(colNames: array of string): DataFrame; begin - Result := Drop(colNames.Select(n -> ColumnIndex(n)).ToArray); + var indices := new integer[colNames.Length]; + + for var i := 0 to colNames.Length - 1 do + indices[i] := fSchema.IndexOf(colNames[i]); + + Result := Drop(indices); +end; + +function DataFrame.BuildJoinSchema(right: DataFrame; leftKeys, rightKeys: array of integer; + rightPrefix: string): DataFrameSchema; +begin + Result := DataFrameSchema.Merge( + fSchema, + right.fSchema, + leftKeys, + rightKeys, + rightPrefix + ); +end; + +function DataFrame.BuildJoinSchema(right: DataFrame; leftKeys, rightKeys: array of string): DataFrameSchema; +begin + var leftIdx := new integer[leftKeys.Length]; + var rightIdx := new integer[rightKeys.Length]; + + for var i := 0 to leftKeys.Length - 1 do + begin + leftIdx[i] := fSchema.IndexOf(leftKeys[i]); + rightIdx[i] := right.fSchema.IndexOf(rightKeys[i]); + end; + + Result := BuildJoinSchema( + right, + leftIdx, + rightIdx, + 'right_' + ); +end; + +function DataFrame.CreateEmptyBySchema(schema: DataFrameSchema): DataFrame; +begin + var cols := new List; + + for var i := 0 to schema.ColumnCount - 1 do + begin + case schema.Types[i] of + ctInt: + cols.Add(new IntColumn(schema.Names[i], schema.IsCategorical[i])); + ctFloat: + cols.Add(new FloatColumn(schema.Names[i])); + ctStr: + cols.Add(new StrColumn(schema.Names[i], schema.IsCategorical[i])); + ctBool: + cols.Add(new BoolColumn(schema.Names[i])); + end; + end; + + Result := new DataFrame(cols, schema); end; function DataFrame.WithColumnInt(name: string; f: DataFrameCursor -> integer): DataFrame; begin - if columnIndexByName.ContainsKey(name) then + if fSchema.HasColumn(name) then raise new Exception($'Column "{name}" already exists'); var res := new DataFrame; @@ -2218,7 +2437,7 @@ end; function DataFrame.WithColumnFloat(name: string; f: DataFrameCursor -> real): DataFrame; begin - if columnIndexByName.ContainsKey(name) then + if fSchema.HasColumn(name) then raise new Exception($'Column "{name}" already exists'); var res := new DataFrame; @@ -2269,7 +2488,7 @@ end; function DataFrame.WithColumnStr(name: string; f: DataFrameCursor -> string): DataFrame; begin - if columnIndexByName.ContainsKey(name) then + if fSchema.HasColumn(name) then raise new Exception($'Column "{name}" already exists'); var res := new DataFrame; @@ -2320,7 +2539,7 @@ end; function DataFrame.WithColumnBool(name: string; f: DataFrameCursor -> boolean): DataFrame; begin - if columnIndexByName.ContainsKey(name) then + if fSchema.HasColumn(name) then raise new Exception($'Column "{name}" already exists'); var res := new DataFrame; @@ -2528,6 +2747,7 @@ end;} procedure DataFrame.PrintPreview(maxRows: integer; headRows: integer; decimals: integer); begin + var ColumnSeparator := ' '; var colCount := columns.Count; if colCount = 0 then exit; @@ -2641,7 +2861,7 @@ begin if columns[j].Info.ColType = ctFloat then intWidth[j] + 1 + decimals else widths[j]; - PABCSystem.Print(columns[j].Info.Name.PadLeft(w) + ' '); + PABCSystem.Print(columns[j].Info.Name.PadLeft(w) + ColumnSeparator); end; PABCSystem.Println; @@ -2684,7 +2904,7 @@ begin begin cursor.MoveTo(i); for var j := 0 to colCount - 1 do - PABCSystem.Print(FormatValue(j) + ' '); + PABCSystem.Print(FormatValue(j) + ColumnSeparator); PABCSystem.Println; end; @@ -2701,7 +2921,7 @@ begin else w := widths[j]; - PABCSystem.Print($'…'.PadLeft(w) + ' '); + PABCSystem.Print($'…'.PadLeft(w) + ColumnSeparator); end; PABCSystem.Println; end; @@ -2712,7 +2932,7 @@ begin begin cursor.MoveTo(i); for var j := 0 to colCount - 1 do - PABCSystem.Print(FormatValue(j) + ' '); + PABCSystem.Print(FormatValue(j) + ColumnSeparator); PABCSystem.Println; end; end; @@ -2736,12 +2956,12 @@ end; procedure DataFrame.PrintSchema; begin - var nameWidth := ColumnNames.Max(s -> s.Length); + var nameWidth := fSchema.Names.Max(s -> s.Length); var typeWidth := 6; // Int / Float / Bool for var i := 0 to ColumnCount - 1 do begin - var name := ColumnNames[i].PadRight(nameWidth); + var name := fSchema.Names[i].PadRight(nameWidth); var typ := GetColumnType(i).ToString.Replace('ct','').PadRight(typeWidth); PABCSystem.Println($'{name} : {typ}'); end; @@ -2752,7 +2972,7 @@ begin PABCSystem.Println($'Rows : {RowCount}'); PABCSystem.Println($'Columns : {ColumnCount}'); - var nameWidth := ColumnNames.Max(s -> s.Length); + var nameWidth := fSchema.Names.Max(s -> s.Length); var typeWidth := 6; // Int / Float / Bool var infoWidth := nameWidth + 3 + typeWidth + 12; @@ -2760,7 +2980,7 @@ begin for var i := 0 to ColumnCount - 1 do begin - var name := ColumnNames[i].PadRight(nameWidth); + var name := fSchema.Names[i].PadRight(nameWidth); var typ := GetColumnType(i).ToString.Replace('ct','').PadRight(typeWidth); var cnt := Count(i); PABCSystem.Println($'{name} : {typ} ({cnt} non-NA)'); @@ -2773,14 +2993,6 @@ procedure DataFrame.AssertSchemaConsistent; begin {$IFDEF Test} - // --- 0. пустой DataFrame --- - if columns.Count = 0 then - begin - if columnIndexByName.Count <> 0 then - raise new Exception('Schema inconsistent: no columns but columnIndexByName not empty'); - exit; - end; - // --- 1. одинаковая RowCount у всех столбцов --- var rc := columns[0].RowCount; for var i := 1 to columns.Count - 1 do @@ -2789,26 +3001,26 @@ begin $'Schema inconsistent: column "{columns[i].Info.Name}" has RowCount={columns[i].RowCount}, expected {rc}' ); - // --- 2. columnIndexByName.Count = columns.Count --- - if columnIndexByName.Count <> columns.Count then + // --- 2. fschema.ColumnCount = columns.Count --- + if fschema.ColumnCount <> columns.Count then raise new Exception( - $'Schema inconsistent: columnIndexByName.Count={columnIndexByName.Count}, columns.Count={columns.Count}' - ); + $'Schema inconsistent: ColumnCount={fschema.ColumnCount}, columns.Count={columns.Count}' + ); // --- 3. имена уникальны и корректно индексированы --- for var i := 0 to columns.Count - 1 do begin var name := columns[i].Info.Name; - if not columnIndexByName.ContainsKey(name) then + if not fSchema.HasColumn(name) then raise new Exception( - $'Schema inconsistent: column "{name}" missing in columnIndexByName' + $'Schema inconsistent: column "{name}" missing in schema' ); - - var idx := columnIndexByName[name]; + + var idx := GetColumnIndex(name); if idx <> i then raise new Exception( - $'Schema inconsistent: columnIndexByName["{name}"]={idx}, expected {i}' + $'Schema inconsistent: GetColumnIndex("{name}")={idx}, expected {i}' ); end; @@ -3401,7 +3613,7 @@ begin // выбираем только числовые столбцы for var i := 0 to df.ColumnCount - 1 do if df.GetColumnType(i) in [ColumnType.ctInt, ColumnType.ctFloat] then - names.Add(df.ColumnNames[i]); + names.Add(df.fSchema.Names[i]); var n := names.Count; if n = 0 then @@ -3464,7 +3676,7 @@ begin means[i] := df.Mean(i); stds[i] := df.Std(i); if stds[i] = 0 then - raise new Exception($'Zero std in column {df.ColumnNames[i]}'); + raise new Exception($'Zero std in column {df.fSchema.Names[i]}'); isNumeric[i] := true; end; end; @@ -3473,7 +3685,7 @@ begin for var i := 0 to df.ColumnCount - 1 do begin if isNumeric[i] then - res.AddFloatColumn(df.ColumnNames[i], new real[df.RowCount], nil) + res.AddFloatColumn(df.fSchema.Names[i], new real[df.RowCount], nil) else res.AddColumnView(df.columns[i]); // private helper end; @@ -3545,7 +3757,7 @@ begin begin var (mn, mx) := df.MinMax(i); if mn = mx then - raise new Exception($'Zero range in column {df.ColumnNames[i]}'); + raise new Exception($'Zero range in column {df.fSchema.Names[i]}'); mins[i] := mn; maxs[i] := mx; isNumeric[i] := true; @@ -3556,7 +3768,7 @@ begin for var i := 0 to df.ColumnCount - 1 do begin if isNumeric[i] then - res.AddFloatColumn(df.ColumnNames[i], new real[df.RowCount], nil) + res.AddFloatColumn(df.fSchema.Names[i], new real[df.RowCount], nil) else res.AddColumnView(df.columns[i]); // private helper end; @@ -3999,12 +4211,25 @@ begin else missing := new HashSet(missingValues); + var raw := lines.ToArray; + + // исключаем пустые строки в начале и конце + var l := 0; + while (l < raw.Length) and (raw[l].Trim = '') do + l += 1; + + var r := raw.Length; + while (r > l) and (raw[r-1].Trim = '') do + r -= 1; + + var linesArray := raw[l:r]; + // ---------- PASS 1: headers + infer ---------- - var linesArray := lines.ToArray; - var headers: array of string := nil; var colCount := 0; var rowCount := linesArray.Count; + if hasHeader then + rowCount -= 1; var canBool, canInt, canFloat: array of boolean; (canBool, canInt, canFloat) := (nil, nil, nil); diff --git a/bin/Lib/DataFrameABCCore.pas b/bin/Lib/DataFrameABCCore.pas index ddd4db51b..b1fd37fac 100644 --- a/bin/Lib/DataFrameABCCore.pas +++ b/bin/Lib/DataFrameABCCore.pas @@ -11,6 +11,48 @@ type ColumnType = (ctInt, ctFloat, ctStr, ctBool); + /// Неизменяемое описание структуры столбцов DataFrame + DataFrameSchema = sealed class + private + fNames: array of string; + fTypes: array of ColumnType; + fIsCategorical: array of boolean; + fIndexByName: Dictionary; + + class function BuildIndex(names: array of string): Dictionary; + public + property ColumnCount: integer read fNames.Length; + property Names: array of string read fNames; + property Types: array of ColumnType read fTypes; + property IsCategorical: array of boolean read fIsCategorical; + + function IndexOf(name: string): integer; + function HasColumn(name: string): boolean; + + function ColumnTypeAt(i: integer): ColumnType; + function IsCategoricalAt(i: integer): boolean; + function NameAt(i: integer): string; + + constructor Create(names: array of string; types: array of ColumnType; + isCategorical: array of boolean := nil); + + { --- schema operations (immutable) --- } + function Select(indices: array of integer): DataFrameSchema; + function Drop(indices: array of integer): DataFrameSchema; + function Rename(oldName, newName: string): DataFrameSchema; + function WithCategorical(name: string; value: boolean := True): DataFrameSchema; + + { --- join helpers --- } + class function Merge( + left, right: DataFrameSchema; + leftKeys, rightKeys: array of integer; + rightPrefix: string + ): DataFrameSchema; + + { --- debug --- } + procedure AssertConsistent; + end; + ColumnInfo = auto class Name: string; ColType: ColumnType; @@ -36,6 +78,8 @@ type Data: array of integer; // Данные столбца IsValid: array of boolean; // Флаги валидности (может быть nil) public + constructor Create; begin end; + constructor Create(name: string; isCategorical: boolean); /// Возвращает количество строк в столбце function RowCount: integer; override := Data.Length; /// Добавляет невалидное (NA) значение в конец столбца @@ -49,6 +93,8 @@ type Data: array of real; // Данные столбца IsValid: array of boolean; // Флаги валидности public + constructor Create; begin end; + constructor Create(name: string); /// Возвращает количество строк в столбце function RowCount: integer; override := Data.Length; /// Добавляет невалидное (NA) значение в конец столбца @@ -61,7 +107,9 @@ type StrColumn = class(Column) Data: array of string; // Данные столбца IsValid: array of boolean; // Флаги валидности - public + public + constructor Create; begin end; + constructor Create(name: string; isCategorical: boolean); /// Возвращает количество строк в столбце function RowCount: integer; override := Data.Length; /// Добавляет невалидное (NA) значение в конец столбца @@ -75,6 +123,8 @@ type Data: array of boolean; // Данные столбца IsValid: array of boolean; // Флаги валидности public + constructor Create; begin end; + constructor Create(name: string); /// Возвращает количество строк в столбце function RowCount: integer; override := Data.Length; /// Добавляет невалидное (NA) значение в конец столбца @@ -111,7 +161,8 @@ type pos: integer; rowCnt: integer; colCnt: integer; - colIndexByName: Dictionary; + fSchema: DataFrameSchema; + intAcc: array of IntAccessor; floatAcc: array of FloatAccessor; strAcc: array of StrAccessor; @@ -119,7 +170,7 @@ type validAcc: array of ValidAccessor; public /// Создает курсор для указанных столбцов - constructor Create(cols: array of Column; colIndexByName: Dictionary); + constructor Create(cols: array of Column; schema: DataFrameSchema); /// Возвращает количество столбцов function ColumnCount: integer := colCnt; /// Возвращает количество строк @@ -199,10 +250,200 @@ begin raise new Exception('Column is not Bool'); end; +//----------------------------- +// DataFrameSchema +//----------------------------- +class function DataFrameSchema.BuildIndex(names: array of string): Dictionary; +begin + Result := new Dictionary; + for var i := 0 to names.Length - 1 do + begin + if Result.ContainsKey(names[i]) then + raise new System.ArgumentException($'Duplicate column name "{names[i]}"'); + Result.Add(names[i], i); + end; +end; + +constructor DataFrameSchema.Create(names: array of string; types: array of ColumnType; + isCategorical: array of boolean); +begin + if names = nil then raise new System.ArgumentException('names is nil'); + if types = nil then raise new System.ArgumentException('types is nil'); + if names.Length <> types.Length then + raise new System.ArgumentException('names and types length mismatch'); + if (isCategorical <> nil) and (isCategorical.Length <> names.Length) then + raise new System.ArgumentException('isCategorical length mismatch'); + + fNames := Copy(names); + fTypes := Copy(types); + fIsCategorical := if isCategorical = nil then nil else Copy(isCategorical); + fIndexByName := BuildIndex(fNames); + + AssertConsistent; +end; + +function DataFrameSchema.IndexOf(name: string): integer; +begin + if not fIndexByName.ContainsKey(name) then + raise new System.ArgumentException($'Column "{name}" does not exist'); + Result := fIndexByName[name]; +end; + +function DataFrameSchema.HasColumn(name: string): boolean := + fIndexByName.ContainsKey(name); + +function DataFrameSchema.NameAt(i: integer): string; +begin + if (i < 0) or (i >= ColumnCount) then + raise new System.ArgumentOutOfRangeException('i'); + Result := fNames[i]; +end; + +function DataFrameSchema.ColumnTypeAt(i: integer): ColumnType; +begin + if (i < 0) or (i >= ColumnCount) then + raise new System.ArgumentOutOfRangeException('i'); + Result := fTypes[i]; +end; + +function DataFrameSchema.IsCategoricalAt(i: integer): boolean; +begin + if (i < 0) or (i >= ColumnCount) then + raise new System.ArgumentOutOfRangeException('i'); + if fIsCategorical = nil then + Result := false + else + Result := fIsCategorical[i]; +end; + +function DataFrameSchema.Select(indices: array of integer): DataFrameSchema; +begin + if indices = nil then raise new System.ArgumentException('indices is nil'); + + var n := indices.Length; + var names := new string[n]; + var types := new ColumnType[n]; + var cats := if fIsCategorical = nil then nil else new boolean[n]; + + for var i := 0 to n - 1 do + begin + var k := indices[i]; + if (k < 0) or (k >= ColumnCount) then + raise new System.ArgumentOutOfRangeException('indices'); + names[i] := fNames[k]; + types[i] := fTypes[k]; + if cats <> nil then cats[i] := fIsCategorical[k]; + end; + + Result := new DataFrameSchema(names, types, cats); +end; + +function DataFrameSchema.Drop(indices: array of integer): DataFrameSchema; +begin + if indices = nil then raise new System.ArgumentException('indices is nil'); + + var drop := new boolean[ColumnCount]; + foreach var i in indices do + begin + if (i < 0) or (i >= ColumnCount) then + raise new System.ArgumentOutOfRangeException('indices'); + drop[i] := true; + end; + + var keep := new List; + for var i := 0 to ColumnCount - 1 do + if not drop[i] then + keep.Add(i); + + Result := Select(keep.ToArray); +end; + +function DataFrameSchema.Rename(oldName, newName: string): DataFrameSchema; +begin + if not HasColumn(oldName) then + raise new System.ArgumentException($'Column "{oldName}" does not exist'); + if HasColumn(newName) then + raise new System.ArgumentException($'Column "{newName}" already exists'); + + var names := Copy(fNames); + names[IndexOf(oldName)] := newName; + + Result := new DataFrameSchema(names, fTypes, fIsCategorical); +end; + +function DataFrameSchema.WithCategorical(name: string; value: boolean): DataFrameSchema; +begin + if not HasColumn(name) then + raise new System.ArgumentException($'Column "{name}" does not exist'); + + var cats := if fIsCategorical = nil then new boolean[ColumnCount] else Copy(fIsCategorical); + cats[IndexOf(name)] := value; + + Result := new DataFrameSchema(fNames, fTypes, cats); +end; + +class function DataFrameSchema.Merge(left, right: DataFrameSchema; + leftKeys, rightKeys: array of integer; rightPrefix: string): DataFrameSchema; +begin + if left = nil then raise new System.ArgumentException('left is nil'); + if right = nil then raise new System.ArgumentException('right is nil'); + if leftKeys.Length <> rightKeys.Length then + raise new System.ArgumentException('join keys length mismatch'); + + var skip := new boolean[right.ColumnCount]; + foreach var i in rightKeys do + begin + if (i < 0) or (i >= right.ColumnCount) then + raise new System.ArgumentOutOfRangeException('rightKeys'); + skip[i] := true; + end; + + var names := new List; + var types := new List; + var cats := new List; + + for var i := 0 to left.ColumnCount - 1 do + begin + names.Add(left.NameAt(i)); + types.Add(left.ColumnTypeAt(i)); + cats.Add(left.IsCategoricalAt(i)); + end; + + for var i := 0 to right.ColumnCount - 1 do + if not skip[i] then + begin + var name := right.NameAt(i); + if left.HasColumn(name) then name := rightPrefix + name; + names.Add(name); + types.Add(right.ColumnTypeAt(i)); + cats.Add(right.IsCategoricalAt(i)); + end; + + Result := new DataFrameSchema(names.ToArray, types.ToArray, cats.ToArray); +end; + +procedure DataFrameSchema.AssertConsistent; +begin + Assert(fNames.Length = fTypes.Length); + if fIsCategorical <> nil then Assert(fIsCategorical.Length = fNames.Length); + Assert(fIndexByName.Count = fNames.Length); +end; + + + //----------------------------- // Columns //----------------------------- +constructor IntColumn.Create(name: string; isCategorical: boolean); +begin + inherited Create; + Info := new ColumnInfo(name, ctInt, isCategorical); + + Data := new integer[0]; + IsValid := nil; +end; + procedure IntColumn.AppendInvalid; begin Data := Data + [0]; @@ -227,6 +468,14 @@ begin else AppendInvalid; end; +constructor FloatColumn.Create(name: string); +begin + inherited Create; + Info := new ColumnInfo(name, ctFloat, false); + Data := new real[0]; + IsValid := nil; +end; + procedure FloatColumn.AppendFromCursor(cur: DataFrameCursor; colIndex: integer); begin if cur.IsValid(colIndex) then @@ -251,6 +500,15 @@ begin IsValid := IsValid + [false]; end; +constructor StrColumn.Create(name: string; isCategorical: boolean); +begin + inherited Create; + Info := new ColumnInfo(name, ctStr, isCategorical); + + Data := new string[0]; + IsValid := nil; +end; + procedure StrColumn.AppendFromCursor(cur: DataFrameCursor; colIndex: integer); begin if cur.IsValid(colIndex) then @@ -275,6 +533,15 @@ begin IsValid := IsValid + [false]; end; +constructor BoolColumn.Create(name: string); +begin + inherited Create; + Info := new ColumnInfo(name, ctBool, false); + + Data := new boolean[0]; + IsValid := nil; +end; + procedure BoolColumn.AppendFromCursor(cur: DataFrameCursor; colIndex: integer); begin if cur.IsValid(colIndex) then @@ -349,14 +616,14 @@ end; // DataFrameCursor //----------------------------- -constructor DataFrameCursor.Create(cols: array of Column; colIndexByName: Dictionary); +constructor DataFrameCursor.Create(cols: array of Column; schema: DataFrameSchema); begin pos := -1; - self.colIndexByName := colIndexByName; + self.fSchema := schema; if cols.Length = 0 then rowCnt := 0 else - case cols[0].Info.ColType of + case fSchema.ColumnTypeAt(0) of ctInt: rowCnt := IntColumn(cols[0]).Data.Length; ctFloat: rowCnt := FloatColumn(cols[0]).Data.Length; ctStr: rowCnt := StrColumn(cols[0]).Data.Length; @@ -376,52 +643,57 @@ begin begin var col := cols[i]; - // IsValid - if col is IntColumn then - begin - var c := IntColumn(col); - if c.IsValid = nil then - validAcc[i] := pos -> true - else - validAcc[i] := pos -> c.IsValid[pos]; - - intAcc[i] := pos -> c.Data[pos]; - floatAcc[i] := pos -> c.Data[pos]; - end - else if col is FloatColumn then - begin - var c := FloatColumn(col); - if c.IsValid = nil then - validAcc[i] := pos -> true - else - validAcc[i] := pos -> c.IsValid[pos]; - - floatAcc[i] := pos -> c.Data[pos]; - intAcc[i] := NotInt; - end - else if col is StrColumn then - begin - var c := StrColumn(col); - if c.IsValid = nil then - validAcc[i] := pos -> true - else - validAcc[i] := pos -> c.IsValid[pos]; - - strAcc[i] := pos -> c.Data[pos]; - end - else if col is BoolColumn then - begin - var c := BoolColumn(col); - if c.IsValid = nil then - validAcc[i] := pos -> true - else - validAcc[i] := pos -> c.IsValid[pos]; - - boolAcc[i] := pos -> c.Data[pos]; - end + case fSchema.ColumnTypeAt(i) of + ctInt: + begin + var c := IntColumn(col); + if c.IsValid = nil then + validAcc[i] := pos -> true + else + validAcc[i] := pos -> c.IsValid[pos]; + + intAcc[i] := pos -> c.Data[pos]; + floatAcc[i] := pos -> c.Data[pos]; + end; + + ctFloat: + begin + var c := FloatColumn(col); + if c.IsValid = nil then + validAcc[i] := pos -> true + else + validAcc[i] := pos -> c.IsValid[pos]; + + floatAcc[i] := pos -> c.Data[pos]; + intAcc[i] := NotInt; + end; + + ctStr: + begin + var c := StrColumn(col); + if c.IsValid = nil then + validAcc[i] := pos -> true + else + validAcc[i] := pos -> c.IsValid[pos]; + + strAcc[i] := pos -> c.Data[pos]; + end; + + ctBool: + begin + var c := BoolColumn(col); + if c.IsValid = nil then + validAcc[i] := pos -> true + else + validAcc[i] := pos -> c.IsValid[pos]; + + boolAcc[i] := pos -> c.Data[pos]; + end; + else raise new Exception('Unknown column type'); + end; end; -end; +end; function DataFrameCursor.MoveNext: boolean; begin @@ -436,7 +708,7 @@ function DataFrameCursor.IsValid(i: integer): boolean := function DataFrameCursor.IsValid(name: string): boolean; begin - Result := IsValid(colIndexByName[name]); + Result := IsValid(fSchema.IndexOf(name)); end; function DataFrameCursor.Int(i: integer): integer := @@ -453,22 +725,22 @@ function DataFrameCursor.Bool(i: integer): boolean := function DataFrameCursor.Int(name: string): integer; begin - Result := Int(colIndexByName[name]); + Result := Int(fSchema.IndexOf(name)); end; function DataFrameCursor.Float(name: string): real; begin - Result := Float(colIndexByName[name]); + Result := Float(fSchema.IndexOf(name)); end; function DataFrameCursor.Str(name: string): string; begin - Result := Str(colIndexByName[name]); + Result := Str(fSchema.IndexOf(name)); end; function DataFrameCursor.Bool(name: string): boolean; begin - Result := Bool(colIndexByName[name]); + Result := Bool(fSchema.IndexOf(name)); end; procedure DataFrameCursor.MoveTo(p: integer);