From 74f69500fa08a4af1268175d7d1fa9b05cff7647 Mon Sep 17 00:00:00 2001 From: Mikhalkovich Stanislav Date: Sun, 21 Jun 2026 22:17:43 +0300 Subject: [PATCH] =?UTF-8?q?m.Print=20-=20=D0=B8=D1=81=D0=BF=D1=80=D0=B0?= =?UTF-8?q?=D0=B2=D0=BB=D0=B5=D0=BD=20=D0=B2=D1=8B=D0=B2=D0=BE=D0=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DataFrame - Stats, Describe Примеры на датафрейм --- .../MachineLearning/02_DataFrame/df23.pas | 15 +- .../MachineLearning/02_DataFrame/df24.pas | 7 +- .../MachineLearning/02_DataFrame/df25.pas | 7 +- .../MachineLearning/02_DataFrame/df26.pas | 5 +- .../MachineLearning/02_DataFrame/df27.pas | 7 +- .../MachineLearning/02_DataFrame/df28.pas | 7 +- .../MachineLearning/02_DataFrame/df29.pas | 7 +- ...luecounts_ties_and_count_name_contract.pas | 3 +- bin/Lib/DataFrameABC.pas | 283 ++++++++++++++---- bin/Lib/DataFrameABCCore.pas | 277 +++++++++++++++++ bin/Lib/MLABC.pas | 7 + bin/Lib/PABCSystem.pas | 29 +- 12 files changed, 545 insertions(+), 109 deletions(-) diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df23.pas b/InstallerSamples/MachineLearning/02_DataFrame/df23.pas index 1a1fb25b3..f8e93309e 100644 --- a/InstallerSamples/MachineLearning/02_DataFrame/df23.pas +++ b/InstallerSamples/MachineLearning/02_DataFrame/df23.pas @@ -6,23 +6,16 @@ id,created_at,name 1,2024-01-15,Alice 2,2024-01-16 12:30:00,Bob '''; - - var schema := new Dictionary; - schema['created_at'] := ColumnType.ctDateTime; - - var df1 := CsvLoader.LoadFromLines( - text.ToLines, - schema := schema + var df1 := DataFrame.FromCsvText( + text, + columnTypes := Dict('created_at' to ColumnType.ctDateTime) ); Println('Явная схема:'); df1.Print; Println(df1.DateTime('created_at')[0].Year); - var df2 := CsvLoader.LoadFromLines( - text.ToLines, - inferTypes := True - ); + var df2 := DataFrame.FromCsvText(text); Println; Println('Автоопределение:'); diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df24.pas b/InstallerSamples/MachineLearning/02_DataFrame/df24.pas index abbb25708..f2b44bd7f 100644 --- a/InstallerSamples/MachineLearning/02_DataFrame/df24.pas +++ b/InstallerSamples/MachineLearning/02_DataFrame/df24.pas @@ -1,4 +1,4 @@ -uses MLABC; +uses MLABC; begin var text := ''' @@ -7,10 +7,7 @@ id,created_at,name 2,16.01.2024 12:30:00,Bob '''; - var df := CsvLoader.LoadFromLines( - text.ToLines, - inferTypes := True - ); + var df := DataFrame.FromCsvText(text); df.Print; Println; diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df25.pas b/InstallerSamples/MachineLearning/02_DataFrame/df25.pas index e9112649e..6bb6800b0 100644 --- a/InstallerSamples/MachineLearning/02_DataFrame/df25.pas +++ b/InstallerSamples/MachineLearning/02_DataFrame/df25.pas @@ -1,4 +1,4 @@ -uses MLABC; +uses MLABC; begin var text := ''' @@ -8,10 +8,7 @@ id,created_at,name 3,17.01.2024 09:15:00,Charlie '''; - var df := CsvLoader.LoadFromLines( - text.ToLines, - inferTypes := True - ); + var df := DataFrame.FromCsvText(text); var sorted := df.SortBy('created_at'); diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df26.pas b/InstallerSamples/MachineLearning/02_DataFrame/df26.pas index 95bdab30c..2b084a7c5 100644 --- a/InstallerSamples/MachineLearning/02_DataFrame/df26.pas +++ b/InstallerSamples/MachineLearning/02_DataFrame/df26.pas @@ -8,10 +8,7 @@ id,created_at,name 3,17.01.2024 09:15:00,Charlie '''; - var df := CsvLoader.LoadFromLines( - text.ToLines, - inferTypes := True - ); + var df := DataFrame.FromCsvText(text); var filtered := df.Filter(cur -> cur.DateTime('created_at') >= DateTime.Create(2024, 1, 16)); diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df27.pas b/InstallerSamples/MachineLearning/02_DataFrame/df27.pas index 206c6194b..4108330e2 100644 --- a/InstallerSamples/MachineLearning/02_DataFrame/df27.pas +++ b/InstallerSamples/MachineLearning/02_DataFrame/df27.pas @@ -1,4 +1,4 @@ -uses MLABC; +uses MLABC; begin var text := ''' @@ -9,10 +9,7 @@ id,created_at,name 4,16.01.2024 12:30:00,Diana '''; - var df := CsvLoader.LoadFromLines( - text.ToLines, - inferTypes := True - ); + var df := DataFrame.FromCsvText(text); var grouped := df.GroupBy('created_at').Count; diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df28.pas b/InstallerSamples/MachineLearning/02_DataFrame/df28.pas index 1b97dc07b..fd57bf0f9 100644 --- a/InstallerSamples/MachineLearning/02_DataFrame/df28.pas +++ b/InstallerSamples/MachineLearning/02_DataFrame/df28.pas @@ -1,4 +1,4 @@ -uses MLABC; +uses MLABC; begin var text := ''' @@ -9,10 +9,7 @@ id,created_at,name 4,05.03.2024 09:15:00,Diana '''; - var df := CsvLoader.LoadFromLines( - text.ToLines, - inferTypes := True - ); + var df := DataFrame.FromCsvText(text); var df2 := df.WithColumnInt( 'year', diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df29.pas b/InstallerSamples/MachineLearning/02_DataFrame/df29.pas index e5a118297..d4d3228b7 100644 --- a/InstallerSamples/MachineLearning/02_DataFrame/df29.pas +++ b/InstallerSamples/MachineLearning/02_DataFrame/df29.pas @@ -1,4 +1,4 @@ -uses MLABC, DataFrameABC; +uses MLABC, DataFrameABC; begin var text := ''' @@ -8,10 +8,7 @@ id,created_at,name 3,05.03.2024 09:15:45,Charlie '''; - var df := CsvLoader.LoadFromLines( - text.ToLines, - inferTypes := True - ); + var df := DataFrame.FromCsvText(text); var df2 := df.WithDatePart('created_at', 'year', dpYear); var df3 := df2.WithDateParts('created_at', [ diff --git a/TestSuite/_MachineLearning/Core/060_dataframe_valuecounts_ties_and_count_name_contract.pas b/TestSuite/_MachineLearning/Core/060_dataframe_valuecounts_ties_and_count_name_contract.pas index 30da74bb7..cab9d1fa9 100644 --- a/TestSuite/_MachineLearning/Core/060_dataframe_valuecounts_ties_and_count_name_contract.pas +++ b/TestSuite/_MachineLearning/Core/060_dataframe_valuecounts_ties_and_count_name_contract.pas @@ -1,4 +1,4 @@ -uses MLABC, DataFrameABC, DataFrameABCCore; +uses MLABC, DataFrameABC, DataFrameABCCore; uses TestHelpers in '..\TestHelpers.pas'; begin @@ -34,3 +34,4 @@ Count Check(vc2.Int('Frequency')[0] = 2, 'ValueCounts conflict first frequency mismatch'); CheckSchemaMatchesColumns(vc2); end. + diff --git a/bin/Lib/DataFrameABC.pas b/bin/Lib/DataFrameABC.pas index 761228d45..fdd2a3ad7 100644 --- a/bin/Lib/DataFrameABC.pas +++ b/bin/Lib/DataFrameABC.pas @@ -301,15 +301,11 @@ type function MeanVariance(selector: DataFrameCursor -> DataValue): (real, real); /// Возвращает статистику столбца по индексу - function Describe(colIndex: integer): DescribeStats; + function Stats(colIndex: integer): DescribeStats; /// Возвращает статистику столбца по имени - function Describe(colName: string): DescribeStats; - /// Возвращает статистику по нескольким столбцам по именам - function Describe(colNames: array of string): Dictionary; - /// Возвращает статистику по нескольким столбцам по индексам - function Describe(colIndices: array of integer): Dictionary; - /// Возвращает статистику по всем числовым столбцам - function DescribeAll: Dictionary; + function Stats(colName: string): DescribeStats; + /// Возвращает таблицу описательной статистики по всем числовым столбцам + function Describe: DataFrame; /// Возвращает уникальные непустые значения столбца /// в порядке первого появления @@ -320,6 +316,8 @@ type /// Результат сортируется по убыванию Count, /// при равенстве частот сохраняется порядок первого появления function ValueCounts(colName: string): DataFrame; + /// Возвращает таблицу частот значений столбца с явным именем столбца счётчиков + function ValueCounts(colName: string; countColName: string): DataFrame; /// Возвращает число пропусков в столбце function MissingCount(colName: string): integer; /// Возвращает таблицу с числом пропусков по всем столбцам @@ -462,20 +460,40 @@ type procedure PrintInfo; /// Загружает DataFrame из CSV файла - static function FromCsv(filename: string): DataFrame; + static function FromCsv( + filename: string; + delimiter: char := ','; + hasHeader: boolean := true; + columnTypes: Dictionary := nil; + categoricalColumns: array of string := nil + ): DataFrame; /// Загружает DataFrame из многострочной строки в формате CSV - static function FromCsvText(text: string): DataFrame; + static function FromCsvText( + text: string; + delimiter: char := ','; + hasHeader: boolean := true; + columnTypes: Dictionary := nil; + categoricalColumns: array of string := nil + ): DataFrame; /// Загружает DataFrame из листа ODS-файла - static function FromODS(filename: string; sheet: string := ''): DataFrame; + static function FromODS( + filename: string; + sheet: string := ''; + hasHeader: boolean := true; + columnTypes: Dictionary := nil; + categoricalColumns: array of string := nil + ): DataFrame; /// Сохраняет DataFrame в csv-файл procedure ToCsv(filename: string); private /// Проверяет валидность индекса столбца procedure CheckColumnIndex(colIndex: integer); + function DescribeByIndices(colIndices: array of integer): Dictionary; + function DescribeAll: Dictionary; /// Добавляет строку из курсора //procedure AppendRowFromCursor(src: DataFrame; cur: DataFrameCursor); end; - + /// Тип операции группировки AggregationKind = (akCount, akSum, akMean, akStd, akMin, akMax); @@ -492,7 +510,6 @@ type function Min(colName: string): real; function Max(colName: string): real; end; - /// Интерфейс для группировки данных IGroupByContext = interface /// Возвращает DataFrame с количеством строк в каждой группе @@ -568,7 +585,7 @@ type /// Эквивалентна Quantile(..., 0.5) static function Median(df: DataFrame; colName: string): real; end; - + type /// Статический класс для загрузки данных из CSV файлов CsvLoader = static class @@ -581,7 +598,7 @@ type missingValues: array of string := nil; // Значения, считающиеся пропущенными trimWhitespace: boolean := true; strict: boolean := False; // Строгий режим (проверка формата) - бросать ли исключения - schema: Dictionary := nil; + columnTypes: Dictionary := nil; sampleSize: integer := 1000; ignoreColumns: array of string := nil; inferTypes: boolean := True; @@ -602,7 +619,7 @@ type missingValues: array of string := nil; trimWhitespace: boolean := true; strict: boolean := False; - schema: Dictionary := nil; + columnTypes: Dictionary := nil; sampleSize: integer := 1000; ignoreColumns: array of string := nil; inferTypes: boolean := True; @@ -763,6 +780,8 @@ const 'Нижняя граница Clip больше верхней!!Clip lower bound is greater than upper bound'; ER_CLIP_INT_BOUNDS_NON_INTEGER = 'Для целочисленного столбца границы Clip должны быть целыми!!Clip bounds for Int column must be integers'; + ER_VALUECOUNTS_COUNTCOL_EMPTY = + 'Имя столбца счётчиков пусто!!ValueCounts count column name is empty'; function FormatDateTimeForPrint(dt: System.DateTime; fmt: string): string; forward; function CollectSelectorValues(df: DataFrame; selector: DataFrameCursor -> DataValue): List; forward; @@ -3127,7 +3146,7 @@ begin Result := (mean, acc / (cnt - 1)); end; -function DataFrame.Describe(colIndex: integer): DescribeStats; +function DataFrame.Stats(colIndex: integer): DescribeStats; begin CheckColumnIndex(colIndex); @@ -3194,30 +3213,17 @@ begin Result.Max := mx; end; -function DataFrame.Describe(colName: string): DescribeStats; +function DataFrame.Stats(colName: string): DescribeStats; begin - Result := Describe(ColumnIndex(colName)); + Result := Stats(ColumnIndex(colName)); end; -function DataFrame.Describe(colNames: array of string): Dictionary; -begin - var res := new Dictionary; - - foreach var name in colNames do - begin - var idx := ColumnIndex(name); - res[name] := Describe(idx); - end; - - Result := res; -end; - -function DataFrame.Describe(colIndices: array of integer): Dictionary; +function DataFrame.DescribeByIndices(colIndices: array of integer): Dictionary; begin var res := new Dictionary; foreach var i in colIndices do - res[i] := Describe(i); + res[i] := Stats(i); Result := res; end; @@ -3229,12 +3235,57 @@ begin for var i := 0 to ColumnCount - 1 do case columns[i].Info.ColType of ctInt, ctFloat: - res[columns[i].Info.Name] := Describe(i); + res[columns[i].Info.Name] := Stats(i); end; Result := res; end; +function DataFrame.Describe: DataFrame; +begin + var stats := DescribeAll; + var names := new List; + var counts := new List; + var means := new List; + var stds := new List; + var mins := new List; + var maxs := new List; + var allInt := true; + + for var i := 0 to ColumnCount - 1 do + case columns[i].Info.ColType of + ctInt, ctFloat: + begin + var name := columns[i].Info.Name; + var st := stats[name]; + names.Add(name); + counts.Add(st.Count); + means.Add(st.Mean); + stds.Add(st.Std); + mins.Add(st.Min); + maxs.Add(st.Max); + if columns[i].Info.ColType <> ctInt then + allInt := false; + end; + end; + + Result := new DataFrame; + Result.AddStrColumn('Column', names.ToArray, nil); + Result.AddIntColumn('Count', counts.ToArray, nil); + Result.AddFloatColumn('Mean', means.ToArray, nil); + Result.AddFloatColumn('Std', stds.ToArray, nil); + if allInt then + begin + Result.AddIntColumn('Min', mins.Select(x -> Round(x)).ToArray, nil); + Result.AddIntColumn('Max', maxs.Select(x -> Round(x)).ToArray, nil); + end + else + begin + Result.AddFloatColumn('Min', mins.ToArray, nil); + Result.AddFloatColumn('Max', maxs.ToArray, nil); + end; +end; + function DataFrame.Unique(colName: string): array of DataValue; begin var ci := ColumnIndex(colName); @@ -3300,13 +3351,30 @@ begin end; function DataFrame.ValueCounts(colName: string): DataFrame; +begin + Result := ValueCounts(colName, 'Count'); +end; + +function ValueCounts(Self: Column): DataFrame; extensionmethod; +begin + var df := new DataFrame; + df.AddColumnAlias(Self); + Result := df.ValueCounts(Self.Info.Name); +end; + +function DataFrame.ValueCounts(colName: string; countColName: string): DataFrame; begin var ci := ColumnIndex(colName); var col := GetColumn(ci); var counts := new List; var order := new List; var res := new DataFrame; - var countColName := if colName = 'Count' then 'Frequency' else 'Count'; + if countColName = nil then + ArgumentNullError('countColName'); + if countColName = '' then + ArgumentError(ER_VALUECOUNTS_COUNTCOL_EMPTY); + if countColName = colName then + ArgumentError(ER_COLUMN_ALREADY_EXISTS, countColName); case col.Info.ColType of ctInt: @@ -5317,30 +5385,75 @@ begin Result := res; end; -static function DataFrame.FromCsv(filename: string): DataFrame; +static function DataFrame.FromCsv( + filename: string; + delimiter: char; + hasHeader: boolean; + columnTypes: Dictionary; + categoricalColumns: array of string +): DataFrame; begin - Result := CsvLoader.Load(filename); -end; - -static function DataFrame.FromCsvText(text: string): DataFrame; -begin - Result := CsvLoader.LoadFromLines(text.ToLines); -end; - -static function DataFrame.FromODS(filename: string; sheet: string): DataFrame; -begin - Result := LoadFromRows( - ODSReader.ReadSheet(filename, sheet), - true, + Result := CsvLoader.Load( + filename, + delimiter, + hasHeader, + nil, nil, true, false, - nil, + columnTypes, 1000, nil, true, nil, + categoricalColumns + ); +end; + +static function DataFrame.FromCsvText( + text: string; + delimiter: char; + hasHeader: boolean; + columnTypes: Dictionary; + categoricalColumns: array of string +): DataFrame; +begin + Result := CsvLoader.LoadFromLines( + text.ToLines, + delimiter, + hasHeader, nil, + true, + false, + columnTypes, + 1000, + nil, + true, + nil, + categoricalColumns + ); +end; + +static function DataFrame.FromODS( + filename: string; + sheet: string; + hasHeader: boolean; + columnTypes: Dictionary; + categoricalColumns: array of string +): DataFrame; +begin + Result := LoadFromRows( + ODSReader.ReadSheet(filename, sheet), + hasHeader, + nil, + true, + false, + columnTypes, + 1000, + nil, + true, + nil, + categoricalColumns, false, 100, 0.2, @@ -6429,7 +6542,17 @@ begin res.AddIntColumn(colNameAgg, counts[c], nil); akSum: - res.AddFloatColumn(colNameAgg, sums[c], nil); + begin + if colsIsInt[c] then + begin + var arr := new integer[n]; + for var g := 0 to n - 1 do + arr[g] := Round(sums[c][g]); + res.AddIntColumn(colNameAgg, arr, nil); + end + else + res.AddFloatColumn(colNameAgg, sums[c], nil); + end; akMean: begin @@ -6451,10 +6574,30 @@ begin end; akMin: - res.AddFloatColumn(colNameAgg, mins[c], nil); + begin + if colsIsInt[c] then + begin + var arr := new integer[n]; + for var g := 0 to n - 1 do + arr[g] := Round(mins[c][g]); + res.AddIntColumn(colNameAgg, arr, nil); + end + else + res.AddFloatColumn(colNameAgg, mins[c], nil); + end; akMax: - res.AddFloatColumn(colNameAgg, maxs[c], nil); + begin + if colsIsInt[c] then + begin + var arr := new integer[n]; + for var g := 0 to n - 1 do + arr[g] := Round(maxs[c][g]); + res.AddIntColumn(colNameAgg, arr, nil); + end + else + res.AddFloatColumn(colNameAgg, maxs[c], nil); + end; end; // метаданные @@ -6462,6 +6605,8 @@ begin if kind = akCount then types.Add(ctInt) + else if (kind in [akSum, akMin, akMax]) and colsIsInt[c] then + types.Add(ctInt) else types.Add(ctFloat); @@ -7638,6 +7783,10 @@ begin if hasHeader then begin headers := Copy(current); + if trimWhitespace then + for var j := 0 to headers.Length - 1 do + if headers[j] <> nil then + headers[j] := headers[j].Trim; originalColCount := headers.Length; end else @@ -8015,7 +8164,7 @@ static function CSVLoader.LoadFromLines( missingValues: array of string; trimWhitespace: boolean; strict: boolean; - schema: Dictionary; + columnTypes: Dictionary; sampleSize: integer; ignoreColumns: array of string; inferTypes: boolean; @@ -8109,6 +8258,9 @@ begin if (s.Length >= 2) and (s[1] = '"') and (s[s.Length] = '"') then s := s.Substring(1, s.Length - 2); + if trimWhitespace then + s := s.Trim; + parts[j] := s; end; @@ -8175,14 +8327,14 @@ begin begin var name := headers[j]; - if (schema <> nil) and schema.ContainsKey(name) then + if (columnTypes <> nil) and columnTypes.ContainsKey(name) then begin canBool[j] := false; canInt[j] := false; canFloat[j] := false; canDateTime[j] := false; - case schema[name] of + case columnTypes[name] of ctBool: canBool[j] := true; ctInt: canInt[j] := true; ctFloat: canFloat[j] := true; @@ -8231,6 +8383,8 @@ begin if (s.Length >= 2) and (s[1] = '"') and (s[s.Length] = '"') then s := s.Substring(1, s.Length - 2); + if trimWhitespace then + s := s.Trim; parts[j] := s; end; @@ -8266,7 +8420,7 @@ begin if not inferTypes then continue; - if (schema <> nil) and schema.ContainsKey(name) then + if (columnTypes <> nil) and columnTypes.ContainsKey(name) then continue; if (forceStrSet <> nil) and (name in forceStrSet) then @@ -8303,7 +8457,7 @@ begin begin var name := headers[j]; - if (schema <> nil) and schema.ContainsKey(name) then continue; + if (columnTypes <> nil) and columnTypes.ContainsKey(name) then continue; if (forceStrSet <> nil) and (name in forceStrSet) then continue; if (catSet <> nil) and (name in catSet) then continue; @@ -8601,7 +8755,7 @@ static function CSVLoader.Load(filename: string; missingValues: array of string; trimWhitespace: boolean; strict: boolean; - schema: Dictionary; + columnTypes: Dictionary; sampleSize: integer; ignoreColumns: array of string; inferTypes: boolean; @@ -8626,7 +8780,7 @@ begin missingValues, trimWhitespace, strict, - schema, + columnTypes, sampleSize, ignoreColumns, inferTypes, @@ -8706,3 +8860,10 @@ begin end; end. + + + + + + + diff --git a/bin/Lib/DataFrameABCCore.pas b/bin/Lib/DataFrameABCCore.pas index b4215f1f8..ffe36a732 100644 --- a/bin/Lib/DataFrameABCCore.pas +++ b/bin/Lib/DataFrameABCCore.pas @@ -206,7 +206,15 @@ type static function operator +(a, b: DataValue): DataValue; static function operator -(a, b: DataValue): DataValue; static function operator *(a, b: DataValue): DataValue; + static function operator *(a: DataValue; b: integer): real; + static function operator *(a: DataValue; b: real): real; + static function operator *(a: integer; b: DataValue): real; + static function operator *(a: real; b: DataValue): real; static function operator /(a, b: DataValue): DataValue; + static function operator /(a: DataValue; b: integer): real; + static function operator /(a: DataValue; b: real): real; + static function operator /(a: integer; b: DataValue): real; + static function operator /(a: real; b: DataValue): real; end; DataFrameCursor = class; @@ -223,6 +231,26 @@ type Info: ColumnInfo; public IsValid: array of boolean; // Флаги валидности (может быть nil) + /// Возвращает число валидных (non-NA) значений в столбце + function Count: integer; virtual; + /// Возвращает число пропусков в столбце + function MissingCount: integer; virtual; + /// Возвращает минимальное числовое значение столбца + function Min: real; virtual; + /// Возвращает максимальное числовое значение столбца + function Max: real; virtual; + /// Возвращает среднее числовое значение столбца + function Mean: real; virtual; + /// Возвращает медиану числовых значений столбца + function Median: real; virtual; + /// Возвращает выборочную дисперсию числовых значений столбца + function Variance: real; virtual; + /// Возвращает выборочное стандартное отклонение числовых значений столбца + function Std: real; virtual; + /// Возвращает уникальные непустые значения столбца в порядке первого появления + function Unique: array of DataValue; virtual; + /// Возвращает число различных непустых значений столбца + function NUnique: integer; virtual; /// Пытается извлечь i-тое данное из столбца как числовое если это возможно function TryGetNumericValue(i: integer; var value: real): boolean; virtual; abstract; /// Возвращает количество строк в столбце @@ -399,6 +427,7 @@ type end; function MergedRightColumnName(leftSchema, rightSchema: DataFrameSchema; rightIndex: integer): string; +function ColumnNumericValues(col: Column): List; implementation @@ -464,6 +493,10 @@ const 'Нельзя выполнять арифметические операции для типов {0} и {1}!!Cannot apply arithmetic to types {0} and {1}'; ER_DATAVALUE_DIVISION_BY_ZERO = 'Деление на ноль!!Division by zero'; + ER_COLUMN_NUMERIC_REQUIRED = + 'Столбец "{0}" должен быть числовым!!Column "{0}" must be numeric'; + ER_COLUMN_NO_VALID_VALUES = + 'Столбец "{0}" не содержит валидных значений!!Column "{0}" has no valid values'; //----------------------------- // Сервисные функции @@ -499,6 +532,28 @@ begin Error(ER_COLUMN_NOT_DATETIME); end; +procedure EnsureNumericColumn(col: Column); +begin + if (col = nil) or (col.Info = nil) then + ArgumentNullError('col'); + + if not (col.Info.ColType in [ctInt, ctFloat]) then + Error(ER_COLUMN_NUMERIC_REQUIRED, col.Info.Name); +end; + +function ColumnNumericValues(col: Column): List; +begin + EnsureNumericColumn(col); + + Result := new List; + for var i := 0 to col.RowCount - 1 do + begin + var value: real; + if col.TryGetNumericValue(i, value) then + Result.Add(value); + end; +end; + function ColumnTypeName(t: ColumnType): string; begin case t of @@ -834,6 +889,42 @@ begin Result := MakeNumeric(a.Name, a.Float * b.Float, t = ctInt); end; +static function DataValue.operator *(a: DataValue; b: integer): real; +begin + if System.Object.ReferenceEquals(a, nil) then + ArgumentNullError('a'); + if not a.fIsValid then + Error(ER_VALUE_IS_NA, a.fName); + Result := a.Float * b; +end; + +static function DataValue.operator *(a: DataValue; b: real): real; +begin + if System.Object.ReferenceEquals(a, nil) then + ArgumentNullError('a'); + if not a.fIsValid then + Error(ER_VALUE_IS_NA, a.fName); + Result := a.Float * b; +end; + +static function DataValue.operator *(a: integer; b: DataValue): real; +begin + if System.Object.ReferenceEquals(b, nil) then + ArgumentNullError('b'); + if not b.fIsValid then + Error(ER_VALUE_IS_NA, b.fName); + Result := a * b.Float; +end; + +static function DataValue.operator *(a: real; b: DataValue): real; +begin + if System.Object.ReferenceEquals(b, nil) then + ArgumentNullError('b'); + if not b.fIsValid then + Error(ER_VALUE_IS_NA, b.fName); + Result := a * b.Float; +end; + static function DataValue.operator /(a, b: DataValue): DataValue; begin ArithmeticType(a, b); @@ -842,6 +933,50 @@ begin Result := new DataValue(a.Name, a.Float / b.Float); end; +static function DataValue.operator /(a: DataValue; b: integer): real; +begin + if System.Object.ReferenceEquals(a, nil) then + ArgumentNullError('a'); + if not a.fIsValid then + Error(ER_VALUE_IS_NA, a.fName); + if b = 0 then + Error(ER_DATAVALUE_DIVISION_BY_ZERO); + Result := a.Float / b; +end; + +static function DataValue.operator /(a: DataValue; b: real): real; +begin + if System.Object.ReferenceEquals(a, nil) then + ArgumentNullError('a'); + if not a.fIsValid then + Error(ER_VALUE_IS_NA, a.fName); + if b = 0 then + Error(ER_DATAVALUE_DIVISION_BY_ZERO); + Result := a.Float / b; +end; + +static function DataValue.operator /(a: integer; b: DataValue): real; +begin + if System.Object.ReferenceEquals(b, nil) then + ArgumentNullError('b'); + if not b.fIsValid then + Error(ER_VALUE_IS_NA, b.fName); + if b.Float = 0 then + Error(ER_DATAVALUE_DIVISION_BY_ZERO); + Result := a / b.Float; +end; + +static function DataValue.operator /(a: real; b: DataValue): real; +begin + if System.Object.ReferenceEquals(b, nil) then + ArgumentNullError('b'); + if not b.fIsValid then + Error(ER_VALUE_IS_NA, b.fName); + if b.Float = 0 then + Error(ER_DATAVALUE_DIVISION_BY_ZERO); + Result := a / b.Float; +end; + procedure DataFrameSchema.Print; begin if fNames.Length = 0 then @@ -1052,6 +1187,146 @@ end; // Columns //----------------------------- +function Column.Count: integer; +begin + Result := 0; + for var i := 0 to RowCount - 1 do + if IsValid[i] then + Result += 1; +end; + +function Column.MissingCount: integer; +begin + Result := 0; + for var i := 0 to RowCount - 1 do + if not IsValid[i] then + Result += 1; +end; + +function Column.Min: real; +begin + var values := ColumnNumericValues(Self); + if values.Count = 0 then + Error(ER_COLUMN_NO_VALID_VALUES, Info.Name); + Result := values.Min; +end; + +function Column.Max: real; +begin + var values := ColumnNumericValues(Self); + if values.Count = 0 then + Error(ER_COLUMN_NO_VALID_VALUES, Info.Name); + Result := values.Max; +end; + +function Column.Mean: real; +begin + var values := ColumnNumericValues(Self); + if values.Count = 0 then + exit(0.0); + Result := values.Sum / values.Count; +end; + +function Column.Median: real; +begin + var values := ColumnNumericValues(Self); + if values.Count = 0 then + Error(ER_COLUMN_NO_VALID_VALUES, Info.Name); + + values.Sort; + var n := values.Count; + if n mod 2 = 1 then + Result := values[n div 2] + else + Result := (values[n div 2 - 1] + values[n div 2]) / 2.0; +end; + +function Column.Variance: real; +begin + var values := ColumnNumericValues(Self); + var cnt := values.Count; + + if cnt <= 1 then + exit(0.0); + + var mean := values.Sum / cnt; + var acc := 0.0; + foreach var v in values do + begin + var d := v - mean; + acc += d * d; + end; + + Result := acc / (cnt - 1); +end; + +function Column.Std: real; +begin + Result := Sqrt(Variance); +end; + +function Column.Unique: array of DataValue; +begin + var values := new List; + + case Info.ColType of + ctInt: + begin + var c := IntColumn(Self); + var seen := new HashSet; + for var i := 0 to c.Data.Length - 1 do + if c.IsValid[i] and seen.Add(c.Data[i]) then + values.Add(new DataValue(Info.Name, c.Data[i])); + end; + + ctFloat: + begin + var c := FloatColumn(Self); + var seen := new HashSet; + for var i := 0 to c.Data.Length - 1 do + if c.IsValid[i] and seen.Add(c.Data[i]) then + values.Add(new DataValue(Info.Name, c.Data[i])); + end; + + ctStr: + begin + var c := StrColumn(Self); + var seen := new HashSet; + for var i := 0 to c.Data.Length - 1 do + if c.IsValid[i] and seen.Add(c.Data[i]) then + values.Add(new DataValue(Info.Name, c.Data[i])); + end; + + ctBool: + begin + var c := BoolColumn(Self); + var seen := new HashSet; + for var i := 0 to c.Data.Length - 1 do + if c.IsValid[i] and seen.Add(c.Data[i]) then + values.Add(new DataValue(Info.Name, c.Data[i])); + end; + + ctDateTime: + begin + var c := DateTimeColumn(Self); + var seen := new HashSet; + for var i := 0 to c.Data.Length - 1 do + if c.IsValid[i] and seen.Add(c.Data[i]) then + values.Add(new DataValue(Info.Name, c.Data[i])); + end; + + else + Error(ER_UNKNOWN_COLUMN_TYPE); + end; + + Result := values.ToArray; +end; + +function Column.NUnique: integer; +begin + Result := Unique.Length; +end; + constructor IntColumn.Create(name: string; values: array of integer; valid: array of boolean); begin inherited Create; @@ -1475,3 +1750,5 @@ begin end; end. + + diff --git a/bin/Lib/MLABC.pas b/bin/Lib/MLABC.pas index ccadc29c8..9af091198 100644 --- a/bin/Lib/MLABC.pas +++ b/bin/Lib/MLABC.pas @@ -185,6 +185,12 @@ type AggregationKind = DataFrameABC.AggregationKind; const + ctInt = ColumnType.ctInt; + ctFloat = ColumnType.ctFloat; + ctStr = ColumnType.ctStr; + ctBool = ColumnType.ctBool; + ctDateTime = ColumnType.ctDateTime; + akMean = AggregationKind.akMean; akMin = AggregationKind.akMin; akMax = AggregationKind.akMax; @@ -226,3 +232,4 @@ implementation function EncodeLabels(labels: array of string): array of integer := MLUtilsABC.EncodeLabels(labels); end. + diff --git a/bin/Lib/PABCSystem.pas b/bin/Lib/PABCSystem.pas index 797a74731..5f60d06d5 100644 --- a/bin/Lib/PABCSystem.pas +++ b/bin/Lib/PABCSystem.pas @@ -1,4 +1,4 @@ -// Copyright (c) Ivan Bondarev, Stanislav Mikhalkovich (for details please see \doc\copyright.txt) +// Copyright (c) Ivan Bondarev, Stanislav Mikhalkovich (for details please see \doc\copyright.txt) // This code is distributed under the GNU LGPL (for details please see \doc\license.txt) /// Стандартный модуль @@ -13592,15 +13592,30 @@ end; /// Вывод двумерного вещественного массива по формату :w:f function Print(Self: array [,] of real; w: integer := 7; f: integer := 2): array [,] of real; extensionmethod; begin - for var i := 0 to Self.RowCount - 1 do + if PrintMatrixWithFormat then begin + var widths := new integer[Self.ColCount]; for var j := 0 to Self.ColCount - 1 do begin - if PrintMatrixWithFormat then - Write(FormatValue(Self[i, j], w, f)) - else Print(Self[i, j]); - end; - Writeln; + widths[j] := w; + for var i := 0 to Self.RowCount - 1 do + widths[j] := Max(widths[j], FormatValue(Self[i, j], 0, f).Length + 1); + end; + for var i := 0 to Self.RowCount - 1 do + begin + for var j := 0 to Self.ColCount - 1 do + Write(FormatValue(Self[i, j], 0, f).PadLeft(widths[j])); + Writeln; + end; + end + else + begin + for var i := 0 to Self.RowCount - 1 do + begin + for var j := 0 to Self.ColCount - 1 do + Print(Self[i, j]); + Writeln; + end; end; Result := Self; end;