From 828484f15a37844678f663367f7a9bf06e28ed2a Mon Sep 17 00:00:00 2001 From: Mikhalkovich Stanislav Date: Fri, 19 Jun 2026 22:06:08 +0300 Subject: [PATCH] =?UTF-8?q?=D0=9D=D0=BE=D0=B2=D1=8B=D0=B5=20ML=20=D0=BF?= =?UTF-8?q?=D1=80=D0=B8=D0=BC=D0=B5=D1=80=D1=8B=20=D0=B8=20=D1=82=D0=B5?= =?UTF-8?q?=D1=81=D1=82=D1=8B=20-=20=D0=B2=20=D0=BE=D1=81=D0=BD=D0=BE?= =?UTF-8?q?=D0=B2=D0=BD=D0=BE=D0=BC=20=D0=B2=20DataFrame=20EncodeLabels=20?= =?UTF-8?q?-=20=D1=83=D0=B1=D1=80=D0=B0=D0=BD=D0=B0=20=D0=BF=D1=80=D0=BE?= =?UTF-8?q?=D0=B2=D0=B5=D1=80=D0=BA=D0=B0=20=D0=BA=D0=B0=D1=82=D0=B5=D0=B3?= =?UTF-8?q?=D0=BE=D1=80=D0=B8=D0=B0=D0=BB=D1=8C=D0=BD=D0=BE=D1=81=D1=82?= =?UTF-8?q?=D0=B8,=20=D0=B4=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5=D0=BD=20?= =?UTF-8?q?=D1=82=D0=B8=D0=BF=20Bool=20=D0=92=20DataFrame=20=D0=B4=D0=BE?= =?UTF-8?q?=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5=D0=BD=20=D1=82=D0=B8=D0=BF=20Dat?= =?UTF-8?q?eTime=20=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB=D0=B5=D0=BD=20?= =?UTF-8?q?=D1=82=D0=B8=D0=BF=20DataValue=20=D0=BF=D0=BE=D0=B7=D0=B2=D0=BE?= =?UTF-8?q?=D0=BB=D1=8F=D1=8E=D1=89=D0=B8=D0=B9=20=D0=BF=D0=B8=D1=81=D0=B0?= =?UTF-8?q?=D1=82=D1=8C=20df.Filter(r=20->=20r['Col']=20>=205)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Configuration/GlobalAssemblyInfo.cs | 2 +- Configuration/Version.defs | 4 +- .../MachineLearning/02_DataFrame/df05.pas | 2 +- .../MachineLearning/02_DataFrame/df23.pas | 31 + .../MachineLearning/02_DataFrame/df24.pas | 20 + .../MachineLearning/02_DataFrame/df25.pas | 23 + .../MachineLearning/02_DataFrame/df26.pas | 23 + .../MachineLearning/02_DataFrame/df27.pas | 24 + .../MachineLearning/02_DataFrame/df28.pas | 29 + .../MachineLearning/02_DataFrame/df29.pas | 24 + .../MachineLearning/02_DataFrame/df30.pas | 15 + .../MachineLearning/02_DataFrame/df31.pas | 18 + .../MachineLearning/02_DataFrame/df32.pas | 18 + .../MachineLearning/02_DataFrame/df33.pas | 23 + .../MachineLearning/02_DataFrame/df34.pas | 26 + .../MachineLearning/02_DataFrame/df35.pas | 18 + .../10_DateTimeComponentsEncoder.pas | 23 + .../11_DateTimeOrdinalEncoder.pas | 23 + .../12_DateTimeCyclicEncoder.pas | 17 + Release/pabcversion.txt | 2 +- ReleaseGenerators/PascalABCNET_version.nsh | 2 +- .../031_dataadapters_encode_str_contract.pas | 1 - .../032_dataadapters_encode_int_contract.pas | 1 - ...datetime_schema_and_inference_contract.pas | 59 + ...taframe_print_datetime_format_contract.pas | 36 + .../Core/047_sortby_datetime_orders_rows.pas | 21 + ...048_filter_datetime_predicate_contract.pas | 21 + .../049_groupby_datetime_count_contract.pas | 25 + .../Core/050_withdatepart_contract.pas | 22 + .../Core/051_withdateparts_contract.pas | 26 + ...me_components_encoder_default_contract.pas | 23 + ...mponents_encoder_custom_parts_contract.pas | 23 + .../054_datetime_ordinal_encoder_contract.pas | 29 + .../055_datetime_cyclic_encoder_contract.pas | 38 + ...aframe_unique_and_valuecounts_contract.pas | 32 + .../Core/057_dataframe_missing_contract.pas | 30 + ...dataframe_dropduplicates_clip_contract.pas | 26 + ...059_dataframe_dropmissing_all_contract.pas | 18 + ...luecounts_ties_and_count_name_contract.pas | 36 + ...dataframe_clip_int_and_bounds_contract.pas | 26 + ...sification_requires_categorical_target.pas | 11 +- bin/Lib/DataAdapters.pas | 33 +- bin/Lib/DataFrameABC.pas | 1847 ++++++++++++++++- bin/Lib/DataFrameABCCore.pas | 517 ++++- bin/Lib/MLABC.pas | 21 + bin/Lib/MLPipelineABC.pas | 8 +- bin/Lib/PreprocessorABC.pas | 667 ++++++ 47 files changed, 3910 insertions(+), 54 deletions(-) create mode 100644 InstallerSamples/MachineLearning/02_DataFrame/df23.pas create mode 100644 InstallerSamples/MachineLearning/02_DataFrame/df24.pas create mode 100644 InstallerSamples/MachineLearning/02_DataFrame/df25.pas create mode 100644 InstallerSamples/MachineLearning/02_DataFrame/df26.pas create mode 100644 InstallerSamples/MachineLearning/02_DataFrame/df27.pas create mode 100644 InstallerSamples/MachineLearning/02_DataFrame/df28.pas create mode 100644 InstallerSamples/MachineLearning/02_DataFrame/df29.pas create mode 100644 InstallerSamples/MachineLearning/02_DataFrame/df30.pas create mode 100644 InstallerSamples/MachineLearning/02_DataFrame/df31.pas create mode 100644 InstallerSamples/MachineLearning/02_DataFrame/df32.pas create mode 100644 InstallerSamples/MachineLearning/02_DataFrame/df33.pas create mode 100644 InstallerSamples/MachineLearning/02_DataFrame/df34.pas create mode 100644 InstallerSamples/MachineLearning/02_DataFrame/df35.pas create mode 100644 InstallerSamples/MachineLearning/03_Preprocessing/10_DateTimeComponentsEncoder.pas create mode 100644 InstallerSamples/MachineLearning/03_Preprocessing/11_DateTimeOrdinalEncoder.pas create mode 100644 InstallerSamples/MachineLearning/03_Preprocessing/12_DateTimeCyclicEncoder.pas create mode 100644 TestSuite/_MachineLearning/Core/045_csv_datetime_schema_and_inference_contract.pas create mode 100644 TestSuite/_MachineLearning/Core/046_dataframe_print_datetime_format_contract.pas create mode 100644 TestSuite/_MachineLearning/Core/047_sortby_datetime_orders_rows.pas create mode 100644 TestSuite/_MachineLearning/Core/048_filter_datetime_predicate_contract.pas create mode 100644 TestSuite/_MachineLearning/Core/049_groupby_datetime_count_contract.pas create mode 100644 TestSuite/_MachineLearning/Core/050_withdatepart_contract.pas create mode 100644 TestSuite/_MachineLearning/Core/051_withdateparts_contract.pas create mode 100644 TestSuite/_MachineLearning/Core/052_datetime_components_encoder_default_contract.pas create mode 100644 TestSuite/_MachineLearning/Core/053_datetime_components_encoder_custom_parts_contract.pas create mode 100644 TestSuite/_MachineLearning/Core/054_datetime_ordinal_encoder_contract.pas create mode 100644 TestSuite/_MachineLearning/Core/055_datetime_cyclic_encoder_contract.pas create mode 100644 TestSuite/_MachineLearning/Core/056_dataframe_unique_and_valuecounts_contract.pas create mode 100644 TestSuite/_MachineLearning/Core/057_dataframe_missing_contract.pas create mode 100644 TestSuite/_MachineLearning/Core/058_dataframe_dropduplicates_clip_contract.pas create mode 100644 TestSuite/_MachineLearning/Core/059_dataframe_dropmissing_all_contract.pas create mode 100644 TestSuite/_MachineLearning/Core/060_dataframe_valuecounts_ties_and_count_name_contract.pas create mode 100644 TestSuite/_MachineLearning/Core/061_dataframe_clip_int_and_bounds_contract.pas diff --git a/Configuration/GlobalAssemblyInfo.cs b/Configuration/GlobalAssemblyInfo.cs index d0576404b..7abd20a81 100644 --- a/Configuration/GlobalAssemblyInfo.cs +++ b/Configuration/GlobalAssemblyInfo.cs @@ -15,7 +15,7 @@ internal static class RevisionClass public const string Major = "3"; public const string Minor = "11"; public const string Build = "1"; - public const string Revision = "3830"; + public const string Revision = "3833"; public const string MainVersion = Major + "." + Minor; public const string FullVersion = Major + "." + Minor + "." + Build + "." + Revision; diff --git a/Configuration/Version.defs b/Configuration/Version.defs index 416c9680e..eb80c3ecb 100644 --- a/Configuration/Version.defs +++ b/Configuration/Version.defs @@ -1,4 +1,4 @@ -%COREVERSION%=1 -%REVISION%=3830 %MINOR%=11 +%REVISION%=3833 +%COREVERSION%=1 %MAJOR%=3 diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df05.pas b/InstallerSamples/MachineLearning/02_DataFrame/df05.pas index 07b7f799d..e0af357f8 100644 --- a/InstallerSamples/MachineLearning/02_DataFrame/df05.pas +++ b/InstallerSamples/MachineLearning/02_DataFrame/df05.pas @@ -9,5 +9,5 @@ begin Charlie,21,78 '''); - df.Filter(row -> row.Int('age') >= 21).Print; + df.Filter(row -> row['age'] >= 21).Print; end. \ No newline at end of file diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df23.pas b/InstallerSamples/MachineLearning/02_DataFrame/df23.pas new file mode 100644 index 000000000..1a1fb25b3 --- /dev/null +++ b/InstallerSamples/MachineLearning/02_DataFrame/df23.pas @@ -0,0 +1,31 @@ +uses MLABC; + +begin + var text := ''' +id,created_at,name +1,2024-01-15,Alice +2,2024-01-16 12:30:00,Bob +'''; + + var schema := new Dictionary; + schema['created_at'] := ColumnType.ctDateTime; + + var df1 := CsvLoader.LoadFromLines( + text.ToLines, + schema := schema + ); + + Println('Явная схема:'); + df1.Print; + Println(df1.DateTime('created_at')[0].Year); + + var df2 := CsvLoader.LoadFromLines( + text.ToLines, + inferTypes := True + ); + + Println; + Println('Автоопределение:'); + df2.Print(dateTimeFormat := 'dd.MM.yy'); + Println(df2.GetColumnType('created_at')); +end. diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df24.pas b/InstallerSamples/MachineLearning/02_DataFrame/df24.pas new file mode 100644 index 000000000..abbb25708 --- /dev/null +++ b/InstallerSamples/MachineLearning/02_DataFrame/df24.pas @@ -0,0 +1,20 @@ +uses MLABC; + +begin + var text := ''' +id,created_at,name +1,15.01.2024,Alice +2,16.01.2024 12:30:00,Bob +'''; + + var df := CsvLoader.LoadFromLines( + text.ToLines, + inferTypes := True + ); + + df.Print; + Println; + Println(df.GetColumnType('created_at')); + Println(df.DateTime('created_at')[0].Year); + Println(df.DateTime('created_at')[1].Hour); +end. diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df25.pas b/InstallerSamples/MachineLearning/02_DataFrame/df25.pas new file mode 100644 index 000000000..e9112649e --- /dev/null +++ b/InstallerSamples/MachineLearning/02_DataFrame/df25.pas @@ -0,0 +1,23 @@ +uses MLABC; + +begin + var text := ''' +id,created_at,name +1,16.01.2024 12:30:00,Bob +2,15.01.2024,Alice +3,17.01.2024 09:15:00,Charlie +'''; + + var df := CsvLoader.LoadFromLines( + text.ToLines, + inferTypes := True + ); + + var sorted := df.SortBy('created_at'); + + Println('Исходный DataFrame:'); + df.Print; + Println; + Println('После SortBy(created_at):'); + sorted.Print; +end. diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df26.pas b/InstallerSamples/MachineLearning/02_DataFrame/df26.pas new file mode 100644 index 000000000..95bdab30c --- /dev/null +++ b/InstallerSamples/MachineLearning/02_DataFrame/df26.pas @@ -0,0 +1,23 @@ +uses MLABC; + +begin + var text := ''' +id,created_at,name +1,15.01.2024,Alice +2,16.01.2024 12:30:00,Bob +3,17.01.2024 09:15:00,Charlie +'''; + + var df := CsvLoader.LoadFromLines( + text.ToLines, + inferTypes := True + ); + + var filtered := df.Filter(cur -> cur.DateTime('created_at') >= DateTime.Create(2024, 1, 16)); + + Println('Исходный DataFrame:'); + df.Print; + Println; + Println('Дата >= 16.01.2024:'); + filtered.Print; +end. diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df27.pas b/InstallerSamples/MachineLearning/02_DataFrame/df27.pas new file mode 100644 index 000000000..206c6194b --- /dev/null +++ b/InstallerSamples/MachineLearning/02_DataFrame/df27.pas @@ -0,0 +1,24 @@ +uses MLABC; + +begin + var text := ''' +id,created_at,name +1,15.01.2024,Alice +2,16.01.2024 12:30:00,Bob +3,15.01.2024,Charlie +4,16.01.2024 12:30:00,Diana +'''; + + var df := CsvLoader.LoadFromLines( + text.ToLines, + inferTypes := True + ); + + var grouped := df.GroupBy('created_at').Count; + + Println('Исходный DataFrame:'); + df.Print; + Println; + Println('GroupBy(created_at).Count:'); + grouped.Print; +end. diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df28.pas b/InstallerSamples/MachineLearning/02_DataFrame/df28.pas new file mode 100644 index 000000000..1b97dc07b --- /dev/null +++ b/InstallerSamples/MachineLearning/02_DataFrame/df28.pas @@ -0,0 +1,29 @@ +uses MLABC; + +begin + var text := ''' +id,created_at,name +1,15.01.2023,Alice +2,16.01.2024 12:30:00,Bob +3,20.02.2023,Charlie +4,05.03.2024 09:15:00,Diana +'''; + + var df := CsvLoader.LoadFromLines( + text.ToLines, + inferTypes := True + ); + + var df2 := df.WithColumnInt( + 'year', + cur -> cur.DateTime('created_at').Year + ); + + var grouped := df2.GroupBy('year').Count; + + Println('DataFrame с колонкой year:'); + df2.Print; + Println; + Println('GroupBy(year).Count:'); + grouped.Print; +end. diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df29.pas b/InstallerSamples/MachineLearning/02_DataFrame/df29.pas new file mode 100644 index 000000000..e5a118297 --- /dev/null +++ b/InstallerSamples/MachineLearning/02_DataFrame/df29.pas @@ -0,0 +1,24 @@ +uses MLABC, DataFrameABC; + +begin + var text := ''' +id,created_at,name +1,15.01.2023 10:20:30,Alice +2,16.02.2024 12:30:00,Bob +3,05.03.2024 09:15:45,Charlie +'''; + + var df := CsvLoader.LoadFromLines( + text.ToLines, + inferTypes := True + ); + + var df2 := df.WithDatePart('created_at', 'year', dpYear); + var df3 := df2.WithDateParts('created_at', [ + ('month', dpMonth), + ('day', dpDay), + ('date_only', dpDate) + ]); + + df3.Print; +end. diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df30.pas b/InstallerSamples/MachineLearning/02_DataFrame/df30.pas new file mode 100644 index 000000000..563e75ffc --- /dev/null +++ b/InstallerSamples/MachineLearning/02_DataFrame/df30.pas @@ -0,0 +1,15 @@ +uses MLABC; + +begin + var df := DataFrame.FromCsvText(''' +product,count,pack_price +milk,2,120 +bread,1,55 +milk,3,118 +tea,1,340 +'''); + + var total := df.Sum(r -> r['count'] * r['pack_price']); + + Println('Общая стоимость упаковок:', total:0:2); +end. diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df31.pas b/InstallerSamples/MachineLearning/02_DataFrame/df31.pas new file mode 100644 index 000000000..75aa07eba --- /dev/null +++ b/InstallerSamples/MachineLearning/02_DataFrame/df31.pas @@ -0,0 +1,18 @@ +uses MLABC; + +begin + var df := DataFrame.FromCsvText(''' +name,district,operation,amount +Alice,North,sale,120 +Bob,North,refund,30 +Clara,South,sale,210 +Dmitry,North,sale,180 +'''); + + var saleCount := df.Count(r -> + (r['district'] = 'North') and + (r['operation'] = 'sale') + ); + + Println('Продаж в районе North:', saleCount); +end. diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df32.pas b/InstallerSamples/MachineLearning/02_DataFrame/df32.pas new file mode 100644 index 000000000..91819ea6a --- /dev/null +++ b/InstallerSamples/MachineLearning/02_DataFrame/df32.pas @@ -0,0 +1,18 @@ +uses MLABC; + +begin + var df := DataFrame.FromCsvText(''' +product,count,price +milk,2,120 +bread,1,55 +milk,3,118 +tea,1,340 +coffee,2,260 +'''); + + Println('Средний чек по строке:', df.Mean(r -> r['count'] * r['price']):0:2); + Println('Медианный чек по строке:', df.Median(r -> r['count'] * r['price']):0:2); + Println('Минимальный чек по строке:', df.Min(r -> r['count'] * r['price']):0:2); + Println('Максимальный чек по строке:', df.Max(r -> r['count'] * r['price']):0:2); + Println('Стандартное отклонение:', df.Std(r -> r['count'] * r['price']):0:2); +end. diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df33.pas b/InstallerSamples/MachineLearning/02_DataFrame/df33.pas new file mode 100644 index 000000000..296e7e04a --- /dev/null +++ b/InstallerSamples/MachineLearning/02_DataFrame/df33.pas @@ -0,0 +1,23 @@ +uses MLABC; + +begin + var df := DataFrame.FromCsvText(''' +Город +Москва +Казань +Москва +Омск +Казань +Москва +'''); + + var values := df.Unique('Город'); + Println('Уникальные города:'); + foreach var v in values do + Println(v.Str); + + Println; + Println('Число различных городов: ', df.NUnique('Город')); + Println; + df.ValueCounts('Город').Print; +end. diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df34.pas b/InstallerSamples/MachineLearning/02_DataFrame/df34.pas new file mode 100644 index 000000000..86a09b8f4 --- /dev/null +++ b/InstallerSamples/MachineLearning/02_DataFrame/df34.pas @@ -0,0 +1,26 @@ +uses MLABC; + +begin + var df := DataFrame.FromCsvText(''' +name,age,income +Alice,20,100 +Bob,,120 +Clara,25, +Dmitry,30,150 +'''); + + Println('Пропусков в столбце age:', df.MissingCount('age')); + Println('Пропусков в столбце income:', df.MissingCount('income')); + Println; + + Println('Пропуски по столбцам:'); + df.MissingCounts.Print; + Println; + + Println('Строки без пропусков во всех столбцах:'); + df.DropMissing.Print; + Println; + + Println('Строки без пропусков в age и income:'); + df.DropMissing(['age', 'income']).Print; +end. diff --git a/InstallerSamples/MachineLearning/02_DataFrame/df35.pas b/InstallerSamples/MachineLearning/02_DataFrame/df35.pas new file mode 100644 index 000000000..f919e0b77 --- /dev/null +++ b/InstallerSamples/MachineLearning/02_DataFrame/df35.pas @@ -0,0 +1,18 @@ +uses MLABC; + +begin + var df := DataFrame.FromCsvText(''' +name,age,score +Alice,10,1.5 +Bob,25,4.8 +Alice,10,1.5 +Clara,30,8.2 +'''); + + Println('Без полных дубликатов:'); + df.DropDuplicates.Print; + Println; + + Println('После Clip(score, 0, 5):'); + df.Clip('score', 0, 5).Print; +end. diff --git a/InstallerSamples/MachineLearning/03_Preprocessing/10_DateTimeComponentsEncoder.pas b/InstallerSamples/MachineLearning/03_Preprocessing/10_DateTimeComponentsEncoder.pas new file mode 100644 index 000000000..f243e1789 --- /dev/null +++ b/InstallerSamples/MachineLearning/03_Preprocessing/10_DateTimeComponentsEncoder.pas @@ -0,0 +1,23 @@ +uses MLABC; + +begin + var df := DataFrame.FromCsvText(''' +id,created_at,name +1,15.01.2024 10:20:30,Alice +2,16.02.2024 12:30:00,Bob +3,17.03.2025 09:15:45,Charlie +'''); + + var enc1 := new DateTimeComponentsEncoder('created_at'); + var df1 := enc1.FitTransform(df); + + Println('Стандартные компоненты:'); + df1.Print; + Println; + + var enc2 := new DateTimeComponentsEncoder('created_at', [dpYear, dpMonth, dpDay, dpHour]); + var df2 := enc2.FitTransform(df); + + Println('Пользовательский набор компонент:'); + df2.Print; +end. diff --git a/InstallerSamples/MachineLearning/03_Preprocessing/11_DateTimeOrdinalEncoder.pas b/InstallerSamples/MachineLearning/03_Preprocessing/11_DateTimeOrdinalEncoder.pas new file mode 100644 index 000000000..b98e96691 --- /dev/null +++ b/InstallerSamples/MachineLearning/03_Preprocessing/11_DateTimeOrdinalEncoder.pas @@ -0,0 +1,23 @@ +uses MLABC; + +begin + var df := DataFrame.FromCsvText(''' +id,created_at,name +1,15.01.2024 00:00:00,Alice +2,16.01.2024 12:00:00,Bob +3,18.01.2024 06:00:00,Charlie +'''); + + var enc1 := new DateTimeOrdinalEncoder('created_at'); + var df1 := enc1.FitTransform(df); + + Println('Смещение в днях от минимальной даты:'); + df1.Print; + Println; + + var enc2 := new DateTimeOrdinalEncoder('created_at', 'created_at_hours', dtuHours); + var df2 := enc2.FitTransform(df); + + Println('Смещение в часах от минимальной даты:'); + df2.Print; +end. diff --git a/InstallerSamples/MachineLearning/03_Preprocessing/12_DateTimeCyclicEncoder.pas b/InstallerSamples/MachineLearning/03_Preprocessing/12_DateTimeCyclicEncoder.pas new file mode 100644 index 000000000..4c30f2c51 --- /dev/null +++ b/InstallerSamples/MachineLearning/03_Preprocessing/12_DateTimeCyclicEncoder.pas @@ -0,0 +1,17 @@ +uses MLABC; + +begin + var df := DataFrame.FromCsvText(''' +id,created_at,name +1,15.01.2024 00:00:00,Alice +2,15.01.2024 06:00:00,Bob +3,15.01.2024 12:00:00,Charlie +4,15.01.2024 18:00:00,Diana +5,15.01.2024 23:56:00,Eva +'''); + + var enc := new DateTimeCyclicEncoder('created_at', dpTimeOfDay); + var df2 := enc.FitTransform(df); + + df2.Print; +end. diff --git a/Release/pabcversion.txt b/Release/pabcversion.txt index ef7dcaed1..5788eb486 100644 --- a/Release/pabcversion.txt +++ b/Release/pabcversion.txt @@ -1 +1 @@ -3.11.1.3830 +3.11.1.3833 diff --git a/ReleaseGenerators/PascalABCNET_version.nsh b/ReleaseGenerators/PascalABCNET_version.nsh index 272caafca..542126095 100644 --- a/ReleaseGenerators/PascalABCNET_version.nsh +++ b/ReleaseGenerators/PascalABCNET_version.nsh @@ -1 +1 @@ -!define VERSION '3.11.1.3830' +!define VERSION '3.11.1.3833' diff --git a/TestSuite/_MachineLearning/Core/031_dataadapters_encode_str_contract.pas b/TestSuite/_MachineLearning/Core/031_dataadapters_encode_str_contract.pas index 3bd47aa3e..b65dd90bf 100644 --- a/TestSuite/_MachineLearning/Core/031_dataadapters_encode_str_contract.pas +++ b/TestSuite/_MachineLearning/Core/031_dataadapters_encode_str_contract.pas @@ -4,7 +4,6 @@ uses TestHelpers in '..\TestHelpers.pas'; begin var df := new DataFrame; df.AddStrColumn('Target', Arr('cat', 'dog', 'cat', 'fox')); - df := df.SetCategorical(['Target']); var classes: array of string; var y := df.EncodeLabels('Target', classes); diff --git a/TestSuite/_MachineLearning/Core/032_dataadapters_encode_int_contract.pas b/TestSuite/_MachineLearning/Core/032_dataadapters_encode_int_contract.pas index d59fb2760..597bef784 100644 --- a/TestSuite/_MachineLearning/Core/032_dataadapters_encode_int_contract.pas +++ b/TestSuite/_MachineLearning/Core/032_dataadapters_encode_int_contract.pas @@ -4,7 +4,6 @@ uses TestHelpers in '..\TestHelpers.pas'; begin var df := new DataFrame; df.AddIntColumn('Target', Arr(10, 20, 10, 30)); - df := df.SetCategorical(['Target']); var classes: array of string; var y := df.EncodeLabels('Target', classes); diff --git a/TestSuite/_MachineLearning/Core/045_csv_datetime_schema_and_inference_contract.pas b/TestSuite/_MachineLearning/Core/045_csv_datetime_schema_and_inference_contract.pas new file mode 100644 index 000000000..6cb5672c5 --- /dev/null +++ b/TestSuite/_MachineLearning/Core/045_csv_datetime_schema_and_inference_contract.pas @@ -0,0 +1,59 @@ +uses MLABC; +uses TestHelpers in '..\TestHelpers.pas'; + +begin + var textIso := ''' +id,created_at,name +1,2024-01-15,Alice +2,2024-01-16T12:30:00,Bob +'''; + + var textRu := ''' +id,created_at,name +1,15.01.2024,Alice +2,16.01.2024 12:30:00,Bob +'''; + + var schema := new Dictionary; + schema['created_at'] := ColumnType.ctDateTime; + + var dfSchema := CsvLoader.LoadFromLines( + textIso.ToLines, + schema := schema + ); + + Check(dfSchema.GetColumnType('created_at') = ColumnType.ctDateTime, 'Schema DateTime type expected'); + Check(dfSchema.DateTime('created_at')[0] = new System.DateTime(2024, 1, 15), 'Schema DateTime value mismatch'); + + var dfInfer := CsvLoader.LoadFromLines( + textIso.ToLines, + inferTypes := true + ); + + Check(dfInfer.GetColumnType('created_at') = ColumnType.ctDateTime, 'Inferred DateTime type expected'); + Check(dfInfer.DateTime('created_at')[1] = new System.DateTime(2024, 1, 16, 12, 30, 0), 'Inferred DateTime value mismatch'); + + var dfRu := CsvLoader.LoadFromLines( + textRu.ToLines, + inferTypes := true + ); + + Check(dfRu.GetColumnType('created_at') = ColumnType.ctDateTime, 'Russian DateTime inference expected'); + Check(dfRu.DateTime('created_at')[0] = new System.DateTime(2024, 1, 15), 'Russian DateTime date mismatch'); + Check(dfRu.DateTime('created_at')[1] = new System.DateTime(2024, 1, 16, 12, 30, 0), 'Russian DateTime value mismatch'); + + var tmp := System.IO.Path.Combine( + System.IO.Path.GetTempPath, + 'ml_datetime_' + System.Guid.NewGuid.ToString + '.csv' + ); + + dfInfer.ToCsv(tmp); + var dfRoundTrip := CsvLoader.Load(tmp); + + Check(dfRoundTrip.GetColumnType('created_at') = ColumnType.ctDateTime, 'Round-trip DateTime type expected'); + Check(dfRoundTrip.DateTime('created_at')[1] = dfInfer.DateTime('created_at')[1], 'Round-trip DateTime value mismatch'); + CheckSchemaMatchesColumns(dfRoundTrip); + + if System.IO.File.Exists(tmp) then + System.IO.File.Delete(tmp); +end. diff --git a/TestSuite/_MachineLearning/Core/046_dataframe_print_datetime_format_contract.pas b/TestSuite/_MachineLearning/Core/046_dataframe_print_datetime_format_contract.pas new file mode 100644 index 000000000..f7ca62f77 --- /dev/null +++ b/TestSuite/_MachineLearning/Core/046_dataframe_print_datetime_format_contract.pas @@ -0,0 +1,36 @@ +uses MLABC; +uses TestHelpers in '..\TestHelpers.pas'; + +begin + var df := new DataFrame; + df.AddDateTimeColumn('created_at', Arr( + new System.DateTime(2024, 1, 15), + new System.DateTime(2024, 1, 16, 12, 30, 0) + )); + + var oldOut := System.Console.Out; + var sw := new System.IO.StringWriter; + System.Console.SetOut(sw); + try + df.Print; + finally + System.Console.SetOut(oldOut); + end; + + var s := sw.ToString; + Check(s.Contains('2024-01-15'), 'Default Print must show date-only value'); + Check(s.Contains('2024-01-16 12:30:00'), 'Default Print must show date and time'); + Check(not s.Contains('2024-01-15 00:00:00'), 'Default Print must omit zero time'); + + sw := new System.IO.StringWriter; + System.Console.SetOut(sw); + try + df.Print(dateTimeFormat := 'yyyy/MM/dd'); + finally + System.Console.SetOut(oldOut); + end; + + s := sw.ToString; + Check(s.Contains('2024/01/15'), 'Custom Print format must be applied'); + Check(s.Contains('2024/01/16'), 'Custom Print format must be applied to all DateTime values'); +end. diff --git a/TestSuite/_MachineLearning/Core/047_sortby_datetime_orders_rows.pas b/TestSuite/_MachineLearning/Core/047_sortby_datetime_orders_rows.pas new file mode 100644 index 000000000..7ad07ecc4 --- /dev/null +++ b/TestSuite/_MachineLearning/Core/047_sortby_datetime_orders_rows.pas @@ -0,0 +1,21 @@ +uses MLABC; +uses TestHelpers in '..\TestHelpers.pas'; + +begin + var df := new DataFrame; + df.AddIntColumn('Id', Arr(1, 2, 3)); + df.AddDateTimeColumn('CreatedAt', Arr( + new System.DateTime(2024, 1, 16, 12, 30, 0), + new System.DateTime(2024, 1, 15), + new System.DateTime(2024, 1, 17, 9, 15, 0) + )); + + var res := df.SortBy('CreatedAt'); + + Check(res.RowCount = 3, 'SortBy(DateTime) row count mismatch'); + Check(res.Int('Id')[0] = 2, 'First sorted row mismatch'); + Check(res.Int('Id')[1] = 1, 'Second sorted row mismatch'); + Check(res.Int('Id')[2] = 3, 'Third sorted row mismatch'); + Check(res.DateTime('CreatedAt')[0] = new System.DateTime(2024, 1, 15), 'Sorted DateTime value mismatch'); + CheckSchemaMatchesColumns(res); +end. diff --git a/TestSuite/_MachineLearning/Core/048_filter_datetime_predicate_contract.pas b/TestSuite/_MachineLearning/Core/048_filter_datetime_predicate_contract.pas new file mode 100644 index 000000000..a42b1d6ce --- /dev/null +++ b/TestSuite/_MachineLearning/Core/048_filter_datetime_predicate_contract.pas @@ -0,0 +1,21 @@ +uses MLABC; +uses TestHelpers in '..\TestHelpers.pas'; + +begin + var df := new DataFrame; + df.AddIntColumn('Id', Arr(1, 2, 3)); + df.AddDateTimeColumn('CreatedAt', Arr( + new System.DateTime(2024, 1, 15), + new System.DateTime(2024, 1, 16, 12, 30, 0), + new System.DateTime(2024, 1, 17, 9, 15, 0) + )); + df.AddStrColumn('Name', Arr('Alice', 'Bob', 'Charlie')); + + var res := df.Filter(cur -> cur.DateTime('CreatedAt') >= new System.DateTime(2024, 1, 16)); + + Check(res.RowCount = 2, 'Filter(DateTime) row count mismatch'); + Check(res.Int('Id')[0] = 2, 'First filtered row mismatch'); + Check(res.Int('Id')[1] = 3, 'Second filtered row mismatch'); + Check(res.DateTime('CreatedAt')[0] = new System.DateTime(2024, 1, 16, 12, 30, 0), 'First filtered DateTime mismatch'); + CheckSchemaMatchesColumns(res); +end. diff --git a/TestSuite/_MachineLearning/Core/049_groupby_datetime_count_contract.pas b/TestSuite/_MachineLearning/Core/049_groupby_datetime_count_contract.pas new file mode 100644 index 000000000..1da525294 --- /dev/null +++ b/TestSuite/_MachineLearning/Core/049_groupby_datetime_count_contract.pas @@ -0,0 +1,25 @@ +uses MLABC, DataFrameABCCore; +uses TestHelpers in '..\TestHelpers.pas'; + +begin + var df := new DataFrame; + df.AddIntColumn('Id', Arr(1, 2, 3, 4)); + df.AddDateTimeColumn('CreatedAt', Arr( + new System.DateTime(2024, 1, 15), + new System.DateTime(2024, 1, 16, 12, 30, 0), + new System.DateTime(2024, 1, 15), + new System.DateTime(2024, 1, 16, 12, 30, 0) + )); + + var g := df.GroupBy('CreatedAt').Count; + + Check(g.Schema.ColumnCount = 2, 'Unexpected GroupBy(DateTime) column count'); + Check(g.Schema.NameAt(0) = 'CreatedAt', 'DateTime key column name mismatch'); + Check(g.Schema.ColumnTypeAt(0) = ColumnType.ctDateTime, 'DateTime key column type must stay DateTime'); + Check(g.GetColumn(0).Info.Name = 'CreatedAt', 'Physical DateTime key column name mismatch'); + Check(g.GetColumn(0).Info.ColType = ColumnType.ctDateTime, 'Physical DateTime key column type must stay DateTime'); + Check(g.RowCount = 2, 'GroupBy(DateTime) distinct key count mismatch'); + Check(g.DateTime('CreatedAt')[0] = new System.DateTime(2024, 1, 15), 'First grouped DateTime key mismatch'); + Check(g.GetIntColumn('count')[0] = 2, 'First grouped count mismatch'); + Check(g.GetIntColumn('count')[1] = 2, 'Second grouped count mismatch'); +end. diff --git a/TestSuite/_MachineLearning/Core/050_withdatepart_contract.pas b/TestSuite/_MachineLearning/Core/050_withdatepart_contract.pas new file mode 100644 index 000000000..9384944d6 --- /dev/null +++ b/TestSuite/_MachineLearning/Core/050_withdatepart_contract.pas @@ -0,0 +1,22 @@ +uses MLABC, DataFrameABC, DataFrameABCCore; +uses TestHelpers in '..\TestHelpers.pas'; + +begin + var df := new DataFrame; + df.AddIntColumn('Id', Arr(1, 2)); + df.AddDateTimeColumn('CreatedAt', Arr( + new System.DateTime(2024, 1, 15, 10, 20, 30), + new System.DateTime(2025, 2, 16, 12, 30, 40) + )); + + var resYear := df.WithDatePart('CreatedAt', 'Year', dpYear); + Check(resYear.GetColumnType('Year') = ColumnType.ctInt, 'Year part type mismatch'); + Check(resYear.Int('Year')[0] = 2024, 'First year mismatch'); + Check(resYear.Int('Year')[1] = 2025, 'Second year mismatch'); + + var resDate := df.WithDatePart('CreatedAt', 'OnlyDate', dpDate); + Check(resDate.GetColumnType('OnlyDate') = ColumnType.ctDateTime, 'Date part type mismatch'); + Check(resDate.DateTime('OnlyDate')[0] = new System.DateTime(2024, 1, 15), 'First date part mismatch'); + Check(resDate.DateTime('OnlyDate')[1] = new System.DateTime(2025, 2, 16), 'Second date part mismatch'); + CheckSchemaMatchesColumns(resDate); +end. diff --git a/TestSuite/_MachineLearning/Core/051_withdateparts_contract.pas b/TestSuite/_MachineLearning/Core/051_withdateparts_contract.pas new file mode 100644 index 000000000..032dc3b45 --- /dev/null +++ b/TestSuite/_MachineLearning/Core/051_withdateparts_contract.pas @@ -0,0 +1,26 @@ +uses MLABC, DataFrameABC, DataFrameABCCore; +uses TestHelpers in '..\TestHelpers.pas'; + +begin + var df := new DataFrame; + df.AddIntColumn('Id', Arr(1, 2)); + df.AddDateTimeColumn('CreatedAt', Arr( + new System.DateTime(2024, 1, 15, 10, 20, 30), + new System.DateTime(2025, 2, 16, 12, 30, 40) + )); + + var res := df.WithDateParts('CreatedAt', [ + ('Year', dpYear), + ('Month', dpMonth), + ('OnlyDate', dpDate) + ]); + + Check(res.ColumnCount = 5, 'WithDateParts column count mismatch'); + Check(res.GetColumnType('Year') = ColumnType.ctInt, 'Year type mismatch'); + Check(res.GetColumnType('Month') = ColumnType.ctInt, 'Month type mismatch'); + Check(res.GetColumnType('OnlyDate') = ColumnType.ctDateTime, 'OnlyDate type mismatch'); + Check(res.Int('Year')[0] = 2024, 'Year[0] mismatch'); + Check(res.Int('Month')[1] = 2, 'Month[1] mismatch'); + Check(res.DateTime('OnlyDate')[0] = new System.DateTime(2024, 1, 15), 'OnlyDate[0] mismatch'); + CheckSchemaMatchesColumns(res); +end. diff --git a/TestSuite/_MachineLearning/Core/052_datetime_components_encoder_default_contract.pas b/TestSuite/_MachineLearning/Core/052_datetime_components_encoder_default_contract.pas new file mode 100644 index 000000000..9eb16cedb --- /dev/null +++ b/TestSuite/_MachineLearning/Core/052_datetime_components_encoder_default_contract.pas @@ -0,0 +1,23 @@ +uses MLABC, DataFrameABC; +uses TestHelpers in '..\TestHelpers.pas'; + +begin + var df := CsvLoader.LoadFromLines(''' +id,created_at,name +1,15.01.2024 10:20:30,Alice +2,16.02.2025 12:30:40,Bob +'''.ToLines, inferTypes := True); + + var enc := new DateTimeComponentsEncoder('created_at'); + var res := enc.FitTransform(df); + + Check(not res.HasColumn('created_at'), 'Source DateTime column must be replaced'); + Check(res.HasColumn('created_at_year'), 'Year column missing'); + Check(res.HasColumn('created_at_month'), 'Month column missing'); + Check(res.HasColumn('created_at_day'), 'Day column missing'); + Check(res.GetColumnType('created_at_year') = ColumnType.ctInt, 'Year column type mismatch'); + Check(res.Int('created_at_year')[0] = 2024, 'First year mismatch'); + Check(res.Int('created_at_month')[1] = 2, 'Second month mismatch'); + Check(res.Int('created_at_day')[1] = 16, 'Second day mismatch'); + CheckSchemaMatchesColumns(res); +end. diff --git a/TestSuite/_MachineLearning/Core/053_datetime_components_encoder_custom_parts_contract.pas b/TestSuite/_MachineLearning/Core/053_datetime_components_encoder_custom_parts_contract.pas new file mode 100644 index 000000000..95648ea34 --- /dev/null +++ b/TestSuite/_MachineLearning/Core/053_datetime_components_encoder_custom_parts_contract.pas @@ -0,0 +1,23 @@ +uses MLABC, DataFrameABC; +uses TestHelpers in '..\TestHelpers.pas'; + +begin + var df := CsvLoader.LoadFromLines(''' +id,created_at,name +1,15.01.2024 10:20:30,Alice +2,16.02.2025 12:30:40,Bob +'''.ToLines, inferTypes := True); + + var enc := new DateTimeComponentsEncoder('created_at', [dpDate, dpHour, dpDayOfWeek]); + var res := enc.FitTransform(df); + + Check(not res.HasColumn('created_at'), 'Source DateTime column must be replaced'); + Check(res.HasColumn('created_at_date'), 'Date component column missing'); + Check(res.HasColumn('created_at_hour'), 'Hour component column missing'); + Check(res.HasColumn('created_at_dayofweek'), 'DayOfWeek component column missing'); + Check(res.GetColumnType('created_at_date') = ColumnType.ctDateTime, 'Date component type mismatch'); + Check(res.GetColumnType('created_at_hour') = ColumnType.ctInt, 'Hour component type mismatch'); + Check(res.DateTime('created_at_date')[0] = new System.DateTime(2024, 1, 15), 'Date component value mismatch'); + Check(res.Int('created_at_hour')[1] = 12, 'Hour component value mismatch'); + CheckSchemaMatchesColumns(res); +end. diff --git a/TestSuite/_MachineLearning/Core/054_datetime_ordinal_encoder_contract.pas b/TestSuite/_MachineLearning/Core/054_datetime_ordinal_encoder_contract.pas new file mode 100644 index 000000000..97e43202d --- /dev/null +++ b/TestSuite/_MachineLearning/Core/054_datetime_ordinal_encoder_contract.pas @@ -0,0 +1,29 @@ +uses MLABC; +uses TestHelpers in '..\TestHelpers.pas'; + +begin + var df := CsvLoader.LoadFromLines(''' +id,created_at,name +1,15.01.2024 00:00:00,Alice +2,16.01.2024 12:00:00,Bob +3,18.01.2024 06:00:00,Charlie +'''.ToLines, inferTypes := True); + + var encDays := new DateTimeOrdinalEncoder('created_at'); + var daysDf := encDays.FitTransform(df); + + Check(not daysDf.HasColumn('created_at'), 'Source DateTime column must be replaced'); + Check(daysDf.GetColumnType('created_at_days') = ColumnType.ctFloat, 'Ordinal DateTime column must become float'); + Check(Abs(daysDf.Float('created_at_days')[0] - 0.0) < 1e-9, 'First day offset mismatch'); + Check(Abs(daysDf.Float('created_at_days')[1] - 1.5) < 1e-9, 'Second day offset mismatch'); + Check(Abs(daysDf.Float('created_at_days')[2] - 3.25) < 1e-9, 'Third day offset mismatch'); + + var encHours := new DateTimeOrdinalEncoder('created_at', 'created_at_hours', dtuHours); + var hoursDf := encHours.FitTransform(df); + + Check(hoursDf.GetColumnType('created_at_hours') = ColumnType.ctFloat, 'Hour ordinal column type mismatch'); + Check(Abs(hoursDf.Float('created_at_hours')[0] - 0.0) < 1e-9, 'First hour offset mismatch'); + Check(Abs(hoursDf.Float('created_at_hours')[1] - 36.0) < 1e-9, 'Second hour offset mismatch'); + Check(Abs(hoursDf.Float('created_at_hours')[2] - 78.0) < 1e-9, 'Third hour offset mismatch'); + CheckSchemaMatchesColumns(hoursDf); +end. diff --git a/TestSuite/_MachineLearning/Core/055_datetime_cyclic_encoder_contract.pas b/TestSuite/_MachineLearning/Core/055_datetime_cyclic_encoder_contract.pas new file mode 100644 index 000000000..30b05d738 --- /dev/null +++ b/TestSuite/_MachineLearning/Core/055_datetime_cyclic_encoder_contract.pas @@ -0,0 +1,38 @@ +uses MLABC; +uses TestHelpers in '..\TestHelpers.pas'; + +begin + var df := CsvLoader.LoadFromLines(''' +id,created_at,name +1,15.01.2024 00:00:00,Alice +2,15.01.2024 06:00:00,Bob +3,15.01.2024 12:00:00,Charlie +4,15.01.2024 18:00:00,Diana +5,15.01.2024 23:59:59,Eva +'''.ToLines, inferTypes := True); + + var enc := new DateTimeCyclicEncoder('created_at', dpTimeOfDay); + var res := enc.FitTransform(df); + + Check(not res.HasColumn('created_at'), 'Source DateTime column must be replaced'); + Check(res.HasColumn('created_at_cyc_timeofday_sin'), 'TimeOfDay sin column missing'); + Check(res.HasColumn('created_at_cyc_timeofday_cos'), 'TimeOfDay cos column missing'); + Check(res.GetColumnType('created_at_cyc_timeofday_sin') = ColumnType.ctFloat, 'TimeOfDay sin type mismatch'); + Check(res.GetColumnType('created_at_cyc_timeofday_cos') = ColumnType.ctFloat, 'TimeOfDay cos type mismatch'); + + Check(Abs(res.Float('created_at_cyc_timeofday_sin')[0] - 0.0) < 1e-9, '00:00 sin mismatch'); + Check(Abs(res.Float('created_at_cyc_timeofday_cos')[0] - 1.0) < 1e-9, '00:00 cos mismatch'); + + Check(Abs(res.Float('created_at_cyc_timeofday_sin')[1] - 1.0) < 1e-9, '06:00 sin mismatch'); + Check(Abs(res.Float('created_at_cyc_timeofday_cos')[1] - 0.0) < 1e-9, '06:00 cos mismatch'); + + Check(Abs(res.Float('created_at_cyc_timeofday_sin')[2] - 0.0) < 1e-9, '12:00 sin mismatch'); + Check(Abs(res.Float('created_at_cyc_timeofday_cos')[2] + 1.0) < 1e-9, '12:00 cos mismatch'); + + Check(Abs(res.Float('created_at_cyc_timeofday_sin')[3] + 1.0) < 1e-9, '18:00 sin mismatch'); + Check(Abs(res.Float('created_at_cyc_timeofday_cos')[3] - 0.0) < 1e-9, '18:00 cos mismatch'); + + Check(Abs(res.Float('created_at_cyc_timeofday_sin')[4]) < 1e-3, '23:59:59 sin must be close to 0'); + Check(Abs(res.Float('created_at_cyc_timeofday_cos')[4] - 1.0) < 1e-6, '23:59:59 cos must be close to 1'); + CheckSchemaMatchesColumns(res); +end. diff --git a/TestSuite/_MachineLearning/Core/056_dataframe_unique_and_valuecounts_contract.pas b/TestSuite/_MachineLearning/Core/056_dataframe_unique_and_valuecounts_contract.pas new file mode 100644 index 000000000..7af1b2fc8 --- /dev/null +++ b/TestSuite/_MachineLearning/Core/056_dataframe_unique_and_valuecounts_contract.pas @@ -0,0 +1,32 @@ +uses MLABC, DataFrameABC, DataFrameABCCore; +uses TestHelpers in '..\TestHelpers.pas'; + +begin + var df := DataFrame.FromCsvText(''' +city +Moscow +Kazan +Moscow +Omsk +Kazan +Moscow +'''); + + var values := df.Unique('city'); + Check(values.Length = 3, 'Unique length mismatch'); + Check(values[0].Str = 'Moscow', 'Unique first value mismatch'); + Check(values[1].Str = 'Kazan', 'Unique second value mismatch'); + Check(values[2].Str = 'Omsk', 'Unique third value mismatch'); + + Check(df.NUnique('city') = 3, 'NUnique mismatch'); + + var vc := df.ValueCounts('city'); + Check(vc.RowCount = 3, 'ValueCounts row count mismatch'); + Check(vc.Str('city')[0] = 'Moscow', 'ValueCounts first key mismatch'); + Check(vc.Int('Count')[0] = 3, 'ValueCounts first count mismatch'); + Check(vc.Str('city')[1] = 'Kazan', 'ValueCounts second key mismatch'); + Check(vc.Int('Count')[1] = 2, 'ValueCounts second count mismatch'); + Check(vc.Str('city')[2] = 'Omsk', 'ValueCounts third key mismatch'); + Check(vc.Int('Count')[2] = 1, 'ValueCounts third count mismatch'); + CheckSchemaMatchesColumns(vc); +end. diff --git a/TestSuite/_MachineLearning/Core/057_dataframe_missing_contract.pas b/TestSuite/_MachineLearning/Core/057_dataframe_missing_contract.pas new file mode 100644 index 000000000..130a3d92e --- /dev/null +++ b/TestSuite/_MachineLearning/Core/057_dataframe_missing_contract.pas @@ -0,0 +1,30 @@ +uses MLABC, DataFrameABC, DataFrameABCCore; +uses TestHelpers in '..\TestHelpers.pas'; + +begin + var df := DataFrame.FromCsvText(''' +name,age,income +Alice,20,100 +Bob,,120 +Clara,25, +Dmitry,30,150 +'''); + + Check(df.MissingCount('age') = 1, 'MissingCount(age) mismatch'); + Check(df.MissingCount('income') = 1, 'MissingCount(income) mismatch'); + + var mc := df.MissingCounts; + Check(mc.RowCount = 3, 'MissingCounts row count mismatch'); + Check(mc.Str('Column')[0] = 'name', 'MissingCounts first column mismatch'); + Check(mc.Int('MissingCount')[0] = 0, 'MissingCounts first count mismatch'); + Check(mc.Str('Column')[1] = 'age', 'MissingCounts second column mismatch'); + Check(mc.Int('MissingCount')[1] = 1, 'MissingCounts second count mismatch'); + Check(mc.Str('Column')[2] = 'income', 'MissingCounts third column mismatch'); + Check(mc.Int('MissingCount')[2] = 1, 'MissingCounts third count mismatch'); + + var dropped := df.DropMissing(['age', 'income']); + Check(dropped.RowCount = 2, 'DropMissing selected row count mismatch'); + Check(dropped.Str('name')[0] = 'Alice', 'DropMissing first row mismatch'); + Check(dropped.Str('name')[1] = 'Dmitry', 'DropMissing second row mismatch'); + CheckSchemaMatchesColumns(dropped); +end. diff --git a/TestSuite/_MachineLearning/Core/058_dataframe_dropduplicates_clip_contract.pas b/TestSuite/_MachineLearning/Core/058_dataframe_dropduplicates_clip_contract.pas new file mode 100644 index 000000000..629c60fca --- /dev/null +++ b/TestSuite/_MachineLearning/Core/058_dataframe_dropduplicates_clip_contract.pas @@ -0,0 +1,26 @@ +uses MLABC, DataFrameABC, DataFrameABCCore; +uses TestHelpers in '..\TestHelpers.pas'; + +begin + var df := DataFrame.FromCsvText(''' +name,age,score +Alice,10,1.5 +Bob,25,4.8 +Alice,10,1.5 +Clara,30,8.2 +'''); + + var dedup := df.DropDuplicates; + Check(dedup.RowCount = 3, 'DropDuplicates row count mismatch'); + Check(dedup.Str('name')[0] = 'Alice', 'DropDuplicates first row mismatch'); + Check(dedup.Str('name')[1] = 'Bob', 'DropDuplicates second row mismatch'); + Check(dedup.Str('name')[2] = 'Clara', 'DropDuplicates third row mismatch'); + CheckSchemaMatchesColumns(dedup); + + var clipped := df.Clip('score', 0, 5); + Check(clipped.RowCount = 4, 'Clip row count mismatch'); + Check(Abs(clipped.Float('score')[0] - 1.5) < 1e-9, 'Clip first value mismatch'); + Check(Abs(clipped.Float('score')[1] - 4.8) < 1e-9, 'Clip second value mismatch'); + Check(Abs(clipped.Float('score')[3] - 5.0) < 1e-9, 'Clip clipped value mismatch'); + CheckSchemaMatchesColumns(clipped); +end. diff --git a/TestSuite/_MachineLearning/Core/059_dataframe_dropmissing_all_contract.pas b/TestSuite/_MachineLearning/Core/059_dataframe_dropmissing_all_contract.pas new file mode 100644 index 000000000..f1d7e335a --- /dev/null +++ b/TestSuite/_MachineLearning/Core/059_dataframe_dropmissing_all_contract.pas @@ -0,0 +1,18 @@ +uses MLABC, DataFrameABC, DataFrameABCCore; +uses TestHelpers in '..\TestHelpers.pas'; + +begin + var df := DataFrame.FromCsvText(''' +name,age,income +Alice,20,100 +Bob,,120 +Clara,25, +Dmitry,30,150 +'''); + + var dropped := df.DropMissing; + Check(dropped.RowCount = 2, 'DropMissing all row count mismatch'); + Check(dropped.Str('name')[0] = 'Alice', 'DropMissing all first row mismatch'); + Check(dropped.Str('name')[1] = 'Dmitry', 'DropMissing all second row mismatch'); + CheckSchemaMatchesColumns(dropped); +end. diff --git a/TestSuite/_MachineLearning/Core/060_dataframe_valuecounts_ties_and_count_name_contract.pas b/TestSuite/_MachineLearning/Core/060_dataframe_valuecounts_ties_and_count_name_contract.pas new file mode 100644 index 000000000..30da74bb7 --- /dev/null +++ b/TestSuite/_MachineLearning/Core/060_dataframe_valuecounts_ties_and_count_name_contract.pas @@ -0,0 +1,36 @@ +uses MLABC, DataFrameABC, DataFrameABCCore; +uses TestHelpers in '..\TestHelpers.pas'; + +begin + var df := DataFrame.FromCsvText(''' +city +Moscow +Kazan +Moscow +Kazan +Omsk +'''); + + var vc := df.ValueCounts('city'); + Check(vc.RowCount = 3, 'ValueCounts tie row count mismatch'); + Check(vc.Str('city')[0] = 'Moscow', 'ValueCounts tie first key mismatch'); + Check(vc.Int('Count')[0] = 2, 'ValueCounts tie first count mismatch'); + Check(vc.Str('city')[1] = 'Kazan', 'ValueCounts tie second key mismatch'); + Check(vc.Int('Count')[1] = 2, 'ValueCounts tie second count mismatch'); + Check(vc.Str('city')[2] = 'Omsk', 'ValueCounts tie third key mismatch'); + Check(vc.Int('Count')[2] = 1, 'ValueCounts tie third count mismatch'); + + var df2 := DataFrame.FromCsvText(''' +Count +1 +2 +1 +'''); + + var vc2 := df2.ValueCounts('Count'); + Check(vc2.HasColumn('Count'), 'ValueCounts must keep source column name'); + Check(vc2.HasColumn('Frequency'), 'ValueCounts must rename counter column on conflict'); + Check(vc2.Int('Count')[0] = 1, 'ValueCounts conflict first key mismatch'); + Check(vc2.Int('Frequency')[0] = 2, 'ValueCounts conflict first frequency mismatch'); + CheckSchemaMatchesColumns(vc2); +end. diff --git a/TestSuite/_MachineLearning/Core/061_dataframe_clip_int_and_bounds_contract.pas b/TestSuite/_MachineLearning/Core/061_dataframe_clip_int_and_bounds_contract.pas new file mode 100644 index 000000000..dbc055beb --- /dev/null +++ b/TestSuite/_MachineLearning/Core/061_dataframe_clip_int_and_bounds_contract.pas @@ -0,0 +1,26 @@ +uses MLABC, DataFrameABC, DataFrameABCCore; +uses TestHelpers in '..\TestHelpers.pas'; + +begin + var df := DataFrame.FromCsvText(''' +age +-5 +10 +30 +'''); + + var clipped := df.Clip('age', 0, 20); + Check(clipped.RowCount = 3, 'Clip int row count mismatch'); + Check(clipped.Int('age')[0] = 0, 'Clip int first value mismatch'); + Check(clipped.Int('age')[1] = 10, 'Clip int second value mismatch'); + Check(clipped.Int('age')[2] = 20, 'Clip int third value mismatch'); + CheckSchemaMatchesColumns(clipped); + + CheckRaises(procedure -> begin + var tmp := df.Clip('age', 0.5, 20); + end, 'Clip must reject non-integer bounds for int column'); + + CheckRaises(procedure -> begin + var tmp := df.Clip('age', 10, 0); + end, 'Clip must reject lower > upper'); +end. diff --git a/TestSuite/_MachineLearning/Regression/019_pipeline_classification_requires_categorical_target.pas b/TestSuite/_MachineLearning/Regression/019_pipeline_classification_requires_categorical_target.pas index 8c73a4e4b..72ff8f073 100644 --- a/TestSuite/_MachineLearning/Regression/019_pipeline_classification_requires_categorical_target.pas +++ b/TestSuite/_MachineLearning/Regression/019_pipeline_classification_requires_categorical_target.pas @@ -12,7 +12,14 @@ begin new LogisticRegression ); - CheckRaises(procedure -> begin pipe.Fit(df); end, - 'Classification pipeline must require categorical target'); + pipe.Fit(df); + + var pred := pipe.Predict(df); + var labels := pipe.PredictLabels(df); + var classes := pipe.GetClassLabels; + + Check(pred.Length = df.RowCount, 'Pipeline prediction length mismatch'); + Check(labels.Length = df.RowCount, 'Pipeline label prediction length mismatch'); + Check(classes.Length = 2, 'Pipeline class count mismatch'); end. diff --git a/bin/Lib/DataAdapters.pas b/bin/Lib/DataAdapters.pas index 4a5068a6c..80bbf49a8 100644 --- a/bin/Lib/DataAdapters.pas +++ b/bin/Lib/DataAdapters.pas @@ -134,9 +134,6 @@ begin if not Self.HasColumn(target) then ArgumentError(ER_COLUMN_NOT_FOUND, target); - if not Self.IsCategorical(target) then - ArgumentError(ER_ENCODELABELS_NOT_CATEGORICAL, target); - var col := Self[target]; for var i := 0 to Self.RowCount - 1 do if not col.IsValid[i] then @@ -157,6 +154,13 @@ begin exit; end; + if Self.GetColumnType(target) = ColumnType.ctBool then + begin + var labels := Self.GetBoolColumn(target).ToArray.Select(x -> x.ToString).ToArray; + Result := EncodeLabels(labels); + exit; + end; + ArgumentError(ER_ENCODELABELS_UNSUPPORTED_TYPE, target); end; @@ -166,7 +170,7 @@ end; /// Каждому уникальному значению присваивается номер 0,1,2,... /// Порядок кодирования соответствует порядку первого появления значений в столбце. /// В параметр classes возвращается массив уникальных значений в порядке кодирования. -/// Работает только для категориальных столбцов типов string и integer. +/// Работает для столбцов типов string, integer и boolean. /// Для целочисленных категориальных столбцов значения преобразуются в строки. /// Используется при подготовке данных для задач классификации и визуализации. function EncodeLabels(Self: DataFrame; target: string; var classes: array of string): array of integer; extensionmethod; @@ -180,9 +184,6 @@ begin if not Self.HasColumn(target) then ArgumentError(ER_COLUMN_NOT_FOUND, target); - if not Self.IsCategorical(target) then - ArgumentError(ER_ENCODELABELS_NOT_CATEGORICAL, target); - var col := Self[target]; for var i := 0 to Self.RowCount - 1 do if not col.IsValid[i] then @@ -204,6 +205,11 @@ begin // если API требует string classes: classes := intClasses.Select(x -> x.ToString).ToArray; end; + ColumnType.ctBool: + begin + var labels := Self.GetBoolColumn(target).ToArray.Select(x -> x.ToString).ToArray; + Result := EncodeLabels(labels, classes); + end; else ArgumentError(ER_ENCODELABELS_UNSUPPORTED_TYPE, target); end; @@ -228,9 +234,6 @@ begin if not Self.HasColumn(target) then ArgumentError(ER_COLUMN_NOT_FOUND, target); - if not Self.IsCategorical(target) then - ArgumentError(ER_ENCODELABELS_NOT_CATEGORICAL, target); - var col := Self[target]; for var i := 0 to Self.RowCount - 1 do if not col.IsValid[i] then @@ -247,6 +250,16 @@ begin var data := Self.GetIntColumn(target).ToArray; var strData := new string[data.Length]; + for var i := 0 to data.Length - 1 do + strData[i] := data[i].ToString; + + Result := TransformLabels(strData, classes); + end; + ColumnType.ctBool: + begin + var data := Self.GetBoolColumn(target).ToArray; + var strData := new string[data.Length]; + for var i := 0 to data.Length - 1 do strData[i] := data[i].ToString; diff --git a/bin/Lib/DataFrameABC.pas b/bin/Lib/DataFrameABC.pas index 7ad7d8bb5..761228d45 100644 --- a/bin/Lib/DataFrameABC.pas +++ b/bin/Lib/DataFrameABC.pas @@ -11,6 +11,9 @@ /// !! DataFrame module for tabular data processing unit DataFrameABC; +{$reference System.Xml.dll} +{$reference System.IO.Compression.dll} + interface uses DataFrameABCCore; @@ -23,8 +26,14 @@ type JoinKind = (jkInner, jkLeft, jkRight, jkFull); /// Тип столбца ColumnType = DataFrameABCCore.ColumnType; + /// Извлекаемая часть значения DateTime + DatePartKind = (dpYear, dpMonth, dpDay, dpHour, dpMinute, dpSecond, dpDayOfWeek, dpTimeOfDay, dpDate); /// Информация о столбце ColumnInfo = DataFrameABCCore.ColumnInfo; + /// Типизированное значение ячейки DataFrame + DataValue = DataFrameABCCore.DataValue; + /// Массив строковых значений + StringArray = DataFrameABCCore.StringArray; /// Курсор для итерации по строкам DataFrame DataFrameCursor = DataFrameABCCore.DataFrameCursor; /// Предикат для фильтрации строк @@ -36,6 +45,7 @@ type FloatColumn = DataFrameABCCore.FloatColumn; BoolColumn = DataFrameABCCore.BoolColumn; StrColumn = DataFrameABCCore.StrColumn; + DateTimeColumn = DataFrameABCCore.DateTimeColumn; IGroupByContext = interface; @@ -192,6 +202,10 @@ type /// Переданные массивы сохраняются как есть. /// Не изменяйте их после передачи в DataFrame. procedure AddBoolColumn(name: string; data: array of boolean; valid: array of boolean := nil); + /// Добавляет столбец DateTime. + /// Переданные массивы сохраняются как есть. + /// Не изменяйте их после передачи в DataFrame. + procedure AddDateTimeColumn(name: string; data: array of System.DateTime; valid: array of boolean := nil); /// Возвращает целочисленный массив значений столбца с данным именем. function GetIntColumn(name: string): array of integer; @@ -201,6 +215,8 @@ type function GetStrColumn(name: string): array of string; /// Возвращает логический массив значений столбца с данным именем. function GetBoolColumn(name: string): array of boolean; + /// Возвращает массив DateTime-значений столбца с данным именем. + function GetDateTimeColumn(name: string): array of System.DateTime; /// Возвращает целочисленный массив значений столбца с данным именем. function Int(name: string): array of integer; @@ -210,47 +226,79 @@ type function Str(name: string): array of string; /// Возвращает логический массив значений столбца с данным именем. function Bool(name: string): array of boolean; + /// Возвращает массив DateTime-значений столбца с данным именем. + function DateTime(name: string): array of System.DateTime; /// Вычисляет сумму значений столбца по индексу function Sum(colIndex: integer): real; /// Вычисляет сумму значений столбца по имени - function Sum(colName: string): real; + function Sum(colName: string): real; + /// Вычисляет сумму значений, задаваемых выражением от строки. + /// Невалидные значения (NA) пропускаются. + function Sum(selector: DataFrameCursor -> DataValue): real; /// Подсчитывает количество валидных значений в столбце по индексу function Count(colIndex: integer): integer; /// Подсчитывает количество валидных значений в столбце по имени - function Count(colName: string): integer; + function Count(colName: string): integer; + /// Подсчитывает количество строк, для которых выражение от строки истинно. + /// Учитываются только валидные логические значения True. + function Count(selector: DataFrameCursor -> DataValue): integer; /// Вычисляет среднее значение столбца по индексу function Mean(colIndex: integer): real; /// Вычисляет среднее значение столбца по имени - function Mean(colName: string): real; + function Mean(colName: string): real; + /// Вычисляет среднее значений, задаваемых выражением от строки. + /// Невалидные значения (NA) пропускаются. + function Mean(selector: DataFrameCursor -> DataValue): real; /// Вычисляет медиану по валидным (non-NA) значениям столбца по индексу function Median(colIndex: integer): real; /// Вычисляет медиану по валидным (non-NA) значениям столбца по имени function Median(colName: string): real; + /// Вычисляет медиану значений, задаваемых выражением от строки. + /// Невалидные значения (NA) пропускаются. + function Median(selector: DataFrameCursor -> DataValue): real; /// Находит минимальное значение столбца по индексу function Min(colIndex: integer): real; /// Находит минимальное значение столбца по имени - function Min(colName: string): real; + function Min(colName: string): real; + /// Находит минимум значений, задаваемых выражением от строки. + /// Невалидные значения (NA) пропускаются. + function Min(selector: DataFrameCursor -> DataValue): real; /// Находит максимальное значение столбца по индексу function Max(colIndex: integer): real; /// Находит максимальное значение столбца по имени function Max(colName: string): real; + /// Находит максимум значений, задаваемых выражением от строки. + /// Невалидные значения (NA) пропускаются. + function Max(selector: DataFrameCursor -> DataValue): real; /// Находит минимальное и максимальное значения столбца по индексу function MinMax(colIndex: integer): (real, real); /// Находит минимальное и максимальное значения столбца по имени - function MinMax(colName: string): (real, real); + function MinMax(colName: string): (real, real); + /// Находит минимум и максимум значений, задаваемых выражением от строки. + /// Невалидные значения (NA) пропускаются. + function MinMax(selector: DataFrameCursor -> DataValue): (real, real); /// Вычисляет дисперсию значений столбца по индексу function Variance(colIndex: integer): real; /// Вычисляет дисперсию значений столбца по имени - function Variance(colName: string): real; + function Variance(colName: string): real; + /// Вычисляет дисперсию значений, задаваемых выражением от строки. + /// Невалидные значения (NA) пропускаются. + function Variance(selector: DataFrameCursor -> DataValue): real; /// Вычисляет стандартное отклонение столбца по индексу function Std(colIndex: integer): real; /// Вычисляет стандартное отклонение столбца по имени - function Std(colName: string): real; + function Std(colName: string): real; + /// Вычисляет стандартное отклонение значений, задаваемых выражением от строки. + /// Невалидные значения (NA) пропускаются. + function Std(selector: DataFrameCursor -> DataValue): real; /// Вычисляет среднее и дисперсию столбца по индексу function MeanVariance(colIndex: integer): (real, real); /// Вычисляет среднее и дисперсию столбца по имени - function MeanVariance(colName: string): (real, real); + function MeanVariance(colName: string): (real, real); + /// Вычисляет среднее и дисперсию значений, задаваемых выражением от строки. + /// Невалидные значения (NA) пропускаются. + function MeanVariance(selector: DataFrameCursor -> DataValue): (real, real); /// Возвращает статистику столбца по индексу function Describe(colIndex: integer): DescribeStats; @@ -261,25 +309,47 @@ type /// Возвращает статистику по нескольким столбцам по индексам function Describe(colIndices: array of integer): Dictionary; /// Возвращает статистику по всем числовым столбцам - function DescribeAll: Dictionary; + function DescribeAll: Dictionary; + + /// Возвращает уникальные непустые значения столбца + /// в порядке первого появления + function Unique(colName: string): array of DataValue; + /// Возвращает число различных непустых значений столбца + function NUnique(colName: string): integer; + /// Возвращает таблицу частот значений столбца. + /// Результат сортируется по убыванию Count, + /// при равенстве частот сохраняется порядок первого появления + function ValueCounts(colName: string): DataFrame; + /// Возвращает число пропусков в столбце + function MissingCount(colName: string): integer; + /// Возвращает таблицу с числом пропусков по всем столбцам + function MissingCounts: DataFrame; + /// Удаляет строки, содержащие пропуски хотя бы в одном столбце + function DropMissing: DataFrame; + /// Удаляет строки, содержащие пропуски в указанных столбцах + function DropMissing(columns: array of string): DataFrame; + /// Удаляет полные дубликаты строк + function DropDuplicates: DataFrame; + /// Ограничивает числовой столбец диапазоном [lower, upper] + function Clip(colName: string; lower, upper: real): DataFrame; /// Группирует данные по столбцу по индексу. - /// Поддерживаемые типы ключей: integer, string, boolean. + /// Поддерживаемые типы ключей: integer, string, boolean, DateTime. /// Вещественные столбцы (float) не поддерживаются из-за численной нестабильности. function GroupBy(colIndex: integer): IGroupByContext; /// Группирует данные по столбцу по имени. - /// Поддерживаемые типы ключей: integer, string, boolean. + /// Поддерживаемые типы ключей: integer, string, boolean, DateTime. /// Вещественные столбцы (float) не поддерживаются из-за численной нестабильности. function GroupBy(colName: string): IGroupByContext; /// Группирует данные по нескольким столбцам по индексам. - /// Все столбцы-ключи должны иметь тип integer, string или boolean. + /// Все столбцы-ключи должны иметь тип integer, string, boolean или DateTime. /// Использование float-столбцов в качестве ключей не поддерживается. function GroupBy(colIndices: array of integer): IGroupByContext; /// Группирует данные по нескольким столбцам по именам. - /// Все столбцы-ключи должны иметь тип integer, string или boolean. + /// Все столбцы-ключи должны иметь тип integer, string, boolean или DateTime. /// Использование float-столбцов в качестве ключей не поддерживается. function GroupBy(colNames: array of string): IGroupByContext; @@ -325,6 +395,11 @@ type function WithColumnStr(name: string; f: DataFrameCursor -> string): DataFrame; /// Добавляет вычисляемый логический столбец function WithColumnBool(name: string; f: DataFrameCursor -> boolean): DataFrame; + /// Добавляет новый столбец, вычисляя указанную часть DateTime-столбца. + /// Для dpDate создаётся столбец DateTime, для остальных частей - целочисленный столбец. + function WithDatePart(sourceColumn, newColumnName: string; part: DatePartKind): DataFrame; + /// Добавляет несколько столбцов, извлекая части из одного DateTime-столбца. + function WithDateParts(sourceColumn: string; parts: array of (string, DatePartKind)): DataFrame; /// Заменяет существующий числовой столбец, пересчитывая его по функции от курсора /// Пропущенные значения (NA) сохраняются @@ -377,8 +452,10 @@ type /// Возвращает новый DataFrame. static function Concat(params dfs: array of DataFrame): DataFrame; - /// Выводит DataFrame с настраиваемым числом строк - procedure Print(maxRows: integer := 10; headRows: integer := -1; decimals: integer := 2); + /// Выводит DataFrame с настраиваемым числом строк. + /// Для столбцов DateTime можно задать свой формат через dateTimeFormat. + procedure Print(maxRows: integer := 10; headRows: integer := -1; + decimals: integer := 2; dateTimeFormat: string := nil); /// Выводит схему датафрейма procedure PrintSchema; /// Выводит размер, схему и количество валидных значений @@ -388,6 +465,8 @@ type static function FromCsv(filename: string): DataFrame; /// Загружает DataFrame из многострочной строки в формате CSV static function FromCsvText(text: string): DataFrame; + /// Загружает DataFrame из листа ODS-файла + static function FromODS(filename: string; sheet: string := ''): DataFrame; /// Сохраняет DataFrame в csv-файл procedure ToCsv(filename: string); private @@ -536,6 +615,14 @@ type minFrequency: integer := 5 ): DataFrame; end; + + /// Читает листы ODS и возвращает их как массив строк ячеек. + /// Первая версия не учитывает слитые ячейки и форматирование. + ODSReader = static class + /// Читает указанный лист ODS-файла. + /// Если sheet = '', используется первый лист. + static function ReadSheet(filename: string; sheet: string := ''): array of StringArray; + end; CsvSaver = static class static procedure Save(df: DataFrame; filename: string; @@ -544,7 +631,7 @@ type implementation -uses MLExceptions; +uses MLExceptions, System.Globalization, System.Text, System.Xml, System.IO, System.IO.Compression; const ER_COLS_NULL = @@ -597,6 +684,8 @@ const 'Столбец не является числовым!!Column is not numeric'; ER_GROUPBY_UNSUPPORTED_KEY_TYPE = 'Неподдерживаемый тип ключа GroupBy: {0}!!Unsupported GroupBy key type: {0}'; + ER_JOIN_DATETIME_NOT_SUPPORTED = + 'Join по ключам DateTime пока не поддерживается!!Join on DateTime keys is not supported yet'; ER_ZERO_VARIANCE = 'Нулевая дисперсия при вычислении корреляции!!Zero variance in correlation'; ER_NO_VALID_PAIRS = @@ -605,6 +694,15 @@ const 'Нет числовых столбцов для матрицы корреляции!!No numeric columns for correlation matrix'; ER_ZERO_STD_STANDARDIZE = 'Нулевое стандартное отклонение при вызове Standardize!!Zero standard deviation in Standardize'; + ER_ODS_FILENAME_EMPTY = + 'Имя ODS-файла пусто!!ODS filename is empty'; + ER_ODS_CONTENT_XML_NOT_FOUND = + 'В ODS-файле не найден content.xml!!content.xml not found in ODS file'; + ER_ODS_NO_SHEETS = + 'ODS-файл не содержит листов!!ODS file contains no sheets'; + ER_ODS_SHEET_NOT_FOUND = + 'Лист "{0}" не найден в ODS-файле!!Sheet "{0}" not found in ODS file'; + ER_INVALID_VALUE_IN_COLUMN = 'Недопустимое значение в столбце "{0}"!!' + 'Invalid value in column "{0}"'; @@ -645,6 +743,8 @@ const 'Столбец {0} содержит нецелое значение {1} в строке {2}!!Column {0} contains non-integer value {1} at row {2}'; ER_FEATURES_EMPTY = 'Список признаков пуст!!Feature list is empty'; + ER_DATEPARTS_EMPTY = + 'Список частей даты пуст!!Date parts list is empty'; ER_AGGREGATIONS_EMPTY = 'Список агрегатов пуст!!Aggregation list is empty'; ER_COLUMN_OUT_OF_RANGE = @@ -657,6 +757,35 @@ const 'Неподдерживаемый тип столбца!!Unsupported column type'; ER_JOIN_FLOAT_KEY_NOT_SUPPORTED = 'Соединение по вещественным ключам не поддерживается из-за ошибок точности!!Join on float keys is not supported due to precision issues'; + ER_COLUMN_NOT_DATETIME = + 'Столбец "{0}" не является DateTime!!Column "{0}" is not DateTime'; + ER_CLIP_LOWER_GREATER_UPPER = + 'Нижняя граница Clip больше верхней!!Clip lower bound is greater than upper bound'; + ER_CLIP_INT_BOUNDS_NON_INTEGER = + 'Для целочисленного столбца границы Clip должны быть целыми!!Clip bounds for Int column must be integers'; + +function FormatDateTimeForPrint(dt: System.DateTime; fmt: string): string; forward; +function CollectSelectorValues(df: DataFrame; selector: DataFrameCursor -> DataValue): List; forward; +function IsIntegralReal(x: real): boolean; forward; +function BuildRowSignature(df: DataFrame; rowIndex: integer): string; forward; +function LoadFromRows( + rows: array of StringArray; + hasHeader: boolean; + missingValues: array of string; + trimWhitespace: boolean; + strict: boolean; + schema: Dictionary; + sampleSize: integer; + ignoreColumns: array of string; + inferTypes: boolean; + forceStringColumns: array of string; + categoricalColumns: array of string; + inferCategorical: boolean; + maxCategoricalCardinality: integer; + maxCategoricalRatio: real; + idThreshold: real; + minFrequency: integer +): DataFrame; forward; type GroupKey = class(IComparable) @@ -957,6 +1086,36 @@ begin res.AddBoolColumn(name, data, valid); end; +procedure BuildDateTimeColumnFromJoin( + res: DataFrame; + name: string; + src: DateTimeColumn; + idx: array of integer +); +begin + var n := idx.Length; + var data := new System.DateTime[n]; + var valid := new boolean[n]; + + for var i := 0 to n - 1 do + begin + var j := idx[i]; + + if j < 0 then + begin + data[i] := default(System.DateTime); + valid[i] := False; + end + else + begin + data[i] := src.Data[j]; + valid[i] := src.IsValid[j]; + end; + end; + + res.AddDateTimeColumn(name, data, valid); +end; + procedure BuildColumnFromJoin(res: DataFrame; name: string; col: Column; idx: array of integer); begin case col.Info.ColType of @@ -968,6 +1127,8 @@ begin BuildStrColumnFromJoin(res, name, StrColumn(col), idx); ctBool: BuildBoolColumnFromJoin(res, name, BoolColumn(col), idx); + ctDateTime: + BuildDateTimeColumnFromJoin(res, name, DateTimeColumn(col), idx); else Error(ER_UNSUPPORTED_COLUMN_TYPE, col.Info.ColType); end; @@ -1119,6 +1280,8 @@ begin if lt = ctFloat then Error(ER_JOIN_FLOAT_KEY_NOT_SUPPORTED); + if lt = ctDateTime then + Error(ER_JOIN_DATETIME_NOT_SUPPORTED); case lt of ctInt: Result := LeftJoinSingleKeyInt(other, leftKey, rightKey); @@ -1680,6 +1843,8 @@ begin if lt = ctFloat then Error(ER_JOIN_FLOAT_KEY_NOT_SUPPORTED); + if lt = ctDateTime then + Error(ER_JOIN_DATETIME_NOT_SUPPORTED); case lt of ctInt: Result := JoinInnerSingleKeyInt(other, leftKey, rightKey); @@ -1935,6 +2100,8 @@ begin if t = ctFloat then Error(ER_JOIN_FLOAT_KEY_NOT_SUPPORTED); + if t = ctDateTime then + Error(ER_JOIN_DATETIME_NOT_SUPPORTED); Result.ColTypes[i] := t; end; @@ -1993,6 +2160,8 @@ begin if lt = ctFloat then Error(ER_JOIN_FLOAT_KEY_NOT_SUPPORTED); + if lt = ctDateTime then + Error(ER_JOIN_DATETIME_NOT_SUPPORTED); case kind of jkInner: @@ -2316,6 +2485,13 @@ begin Result := c.Data; end; +function DataFrame.GetDateTimeColumn(name: string): array of System.DateTime; +begin + var i := ColumnIndex(name); + var c := DateTimeColumn(columns[i]); + Result := c.Data; +end; + function DataFrame.Int(name: string): array of integer; begin Result := GetIntColumn(name); @@ -2336,6 +2512,11 @@ begin Result := GetBoolColumn(name); end; +function DataFrame.DateTime(name: string): array of System.DateTime; +begin + Result := GetDateTimeColumn(name); +end; + function DataFrame.TrainTestSplit(testRatio: real; shuffle: boolean; seed: integer): (DataFrame, DataFrame); begin if Self = nil then @@ -2544,6 +2725,18 @@ begin CommitAddedColumn(c); end; +procedure DataFrame.AddDateTimeColumn(name: string; data: array of System.DateTime; valid: array of boolean); +begin + if HasColumn(name) then + ArgumentError(ER_COLUMN_ALREADY_EXISTS, name); + + if (columns.Count > 0) and (Length(data) <> RowCount) then + DimensionError(ER_ADD_COLUMN_ROW_MISMATCH); + + var c := new DateTimeColumn(name, data, valid); + CommitAddedColumn(c); +end; + procedure DataFrame.CheckColumnIndex(colIndex: integer); begin if (colIndex < 0) or (colIndex >= ColumnCount) then @@ -2566,6 +2759,24 @@ end; function DataFrame.Sum(colName: string): real := Sum(ColumnIndex(colName)); + +function DataFrame.Sum(selector: DataFrameCursor -> DataValue): real; +begin + if selector = nil then + ArgumentNullError('selector'); + + var cursor := GetCursor; + var s: real := 0.0; + + while cursor.MoveNext do + begin + var v := selector(cursor); + if (not System.Object.ReferenceEquals(v, nil)) and v.IsValid then + s += v.Float; + end; + + Result := s; +end; function DataFrame.Count(colIndex: integer): integer; begin @@ -2584,6 +2795,24 @@ end; function DataFrame.Count(colName: string): integer := Count(ColumnIndex(colName)); +function DataFrame.Count(selector: DataFrameCursor -> DataValue): integer; +begin + if selector = nil then + ArgumentNullError('selector'); + + var cursor := GetCursor; + var c := 0; + + while cursor.MoveNext do + begin + var v := selector(cursor); + if (not System.Object.ReferenceEquals(v, nil)) and v.IsValid and v.Bool then + c += 1; + end; + + Result := c; +end; + function DataFrame.Mean(colIndex: integer): real; begin CheckColumnIndex(colIndex); @@ -2605,6 +2834,15 @@ end; function DataFrame.Mean(colName: string): real := Mean(ColumnIndex(colName)); + +function DataFrame.Mean(selector: DataFrameCursor -> DataValue): real; +begin + var values := CollectSelectorValues(Self, selector); + if values.Count = 0 then + exit(0.0); + + Result := values.Sum / values.Count; +end; function DataFrame.Median(colIndex: integer): real; begin @@ -2617,6 +2855,21 @@ begin Result := Statistics.Median(Self, colName); end; +function DataFrame.Median(selector: DataFrameCursor -> DataValue): real; +begin + var values := CollectSelectorValues(Self, selector); + if values.Count = 0 then + Error(ER_NO_VALID_VALUES_QUANTILE); + + values.Sort; + + var n := values.Count; + if n mod 2 = 1 then + Result := values[n div 2] + else + Result := (values[n div 2 - 1] + values[n div 2]) / 2.0; +end; + function DataFrame.Min(colIndex: integer): real; begin CheckColumnIndex(colIndex); @@ -2647,6 +2900,15 @@ end; function DataFrame.Min(colName: string): real := Min(ColumnIndex(colName)); +function DataFrame.Min(selector: DataFrameCursor -> DataValue): real; +begin + var values := CollectSelectorValues(Self, selector); + if values.Count = 0 then + Error(ER_NO_VALID_VALUES_COLUMN, 'selector'); + + Result := values.Min; +end; + function DataFrame.Max(colIndex: integer): real; begin CheckColumnIndex(colIndex); @@ -2676,6 +2938,15 @@ end; function DataFrame.Max(colName: string): real := Max(ColumnIndex(colName)); + +function DataFrame.Max(selector: DataFrameCursor -> DataValue): real; +begin + var values := CollectSelectorValues(Self, selector); + if values.Count = 0 then + Error(ER_NO_VALID_VALUES_COLUMN, 'selector'); + + Result := values.Max; +end; function DataFrame.MinMax(colIndex: integer): (real, real); begin @@ -2712,6 +2983,15 @@ function DataFrame.MinMax(colName: string): (real, real); begin Result := MinMax(ColumnIndex(colName)); end; + +function DataFrame.MinMax(selector: DataFrameCursor -> DataValue): (real, real); +begin + var values := CollectSelectorValues(Self, selector); + if values.Count = 0 then + Error(ER_NO_VALID_VALUES_COLUMN, 'selector'); + + Result := (values.Min, values.Max); +end; function DataFrame.Variance(colIndex: integer): real; begin @@ -2751,11 +3031,36 @@ end; function DataFrame.Variance(colName: string): real := Variance(ColumnIndex(colName)); +function DataFrame.Variance(selector: DataFrameCursor -> DataValue): real; +begin + var values := CollectSelectorValues(Self, selector); + var cnt := values.Count; + + if cnt <= 1 then + exit(0.0); + + var mean := values.Sum / cnt; + var acc := 0.0; + + foreach var v in values do + begin + var d := v - mean; + acc += d * d; + end; + + Result := acc / (cnt - 1); +end; + function DataFrame.Std(colIndex: integer): real := Sqrt(Variance(colIndex)); function DataFrame.Std(colName: string): real := Std(ColumnIndex(colName)); + +function DataFrame.Std(selector: DataFrameCursor -> DataValue): real; +begin + Result := Sqrt(Variance(selector)); +end; function DataFrame.MeanVariance(colIndex: integer): (real, real); begin @@ -2798,6 +3103,29 @@ end; function DataFrame.MeanVariance(colName: string): (real, real) := MeanVariance(ColumnIndex(colName)); + +function DataFrame.MeanVariance(selector: DataFrameCursor -> DataValue): (real, real); +begin + var values := CollectSelectorValues(Self, selector); + var cnt := values.Count; + + if cnt = 0 then + exit((0.0, 0.0)); + + var mean := values.Sum / cnt; + + if cnt <= 1 then + exit((mean, 0.0)); + + var acc := 0.0; + foreach var v in values do + begin + var d := v - mean; + acc += d * d; + end; + + Result := (mean, acc / (cnt - 1)); +end; function DataFrame.Describe(colIndex: integer): DescribeStats; begin @@ -2907,6 +3235,413 @@ begin Result := res; end; +function DataFrame.Unique(colName: string): array of DataValue; +begin + var ci := ColumnIndex(colName); + var col := GetColumn(ci); + var values := new List; + + case col.Info.ColType of + ctInt: + begin + var c := IntColumn(col); + var seen := new HashSet; + for var i := 0 to c.Data.Length - 1 do + if c.IsValid[i] and seen.Add(c.Data[i]) then + values.Add(new DataValue(colName, c.Data[i])); + end; + + ctFloat: + begin + var c := FloatColumn(col); + var seen := new HashSet; + for var i := 0 to c.Data.Length - 1 do + if c.IsValid[i] and seen.Add(c.Data[i]) then + values.Add(new DataValue(colName, c.Data[i])); + end; + + ctStr: + begin + var c := StrColumn(col); + var seen := new HashSet; + for var i := 0 to c.Data.Length - 1 do + if c.IsValid[i] and seen.Add(c.Data[i]) then + values.Add(new DataValue(colName, c.Data[i])); + end; + + ctBool: + begin + var c := BoolColumn(col); + var seen := new HashSet; + for var i := 0 to c.Data.Length - 1 do + if c.IsValid[i] and seen.Add(c.Data[i]) then + values.Add(new DataValue(colName, c.Data[i])); + end; + + ctDateTime: + begin + var c := DateTimeColumn(col); + var seen := new HashSet; + for var i := 0 to c.Data.Length - 1 do + if c.IsValid[i] and seen.Add(c.Data[i]) then + values.Add(new DataValue(colName, c.Data[i])); + end; + + else + Error(ER_UNKNOWN_COLUMN_TYPE); + end; + + Result := values.ToArray; +end; + +function DataFrame.NUnique(colName: string): integer; +begin + Result := Unique(colName).Length; +end; + +function DataFrame.ValueCounts(colName: string): DataFrame; +begin + var ci := ColumnIndex(colName); + var col := GetColumn(ci); + var counts := new List; + var order := new List; + var res := new DataFrame; + var countColName := if colName = 'Count' then 'Frequency' else 'Count'; + + case col.Info.ColType of + ctInt: + begin + var c := IntColumn(col); + var index := new Dictionary; + var values := new List; + + for var i := 0 to c.Data.Length - 1 do + if c.IsValid[i] then + begin + var pos: integer; + if index.TryGetValue(c.Data[i], pos) then + counts[pos] += 1 + else + begin + pos := values.Count; + index[c.Data[i]] := pos; + values.Add(c.Data[i]); + counts.Add(1); + end; + end; + + for var i := 0 to values.Count - 1 do + order.Add(i); + + order.Sort((a, b) -> + begin + Result := counts[b].CompareTo(counts[a]); + if Result = 0 then + Result := a.CompareTo(b); + end); + + var sortedValues := new integer[values.Count]; + var sortedCounts := new integer[values.Count]; + for var i := 0 to order.Count - 1 do + begin + sortedValues[i] := values[order[i]]; + sortedCounts[i] := counts[order[i]]; + end; + + res.AddIntColumn(colName, sortedValues); + res.AddIntColumn(countColName, sortedCounts); + end; + + ctFloat: + begin + var c := FloatColumn(col); + var index := new Dictionary; + var values := new List; + + for var i := 0 to c.Data.Length - 1 do + if c.IsValid[i] then + begin + var pos: integer; + if index.TryGetValue(c.Data[i], pos) then + counts[pos] += 1 + else + begin + pos := values.Count; + index[c.Data[i]] := pos; + values.Add(c.Data[i]); + counts.Add(1); + end; + end; + + for var i := 0 to values.Count - 1 do + order.Add(i); + + order.Sort((a, b) -> + begin + Result := counts[b].CompareTo(counts[a]); + if Result = 0 then + Result := a.CompareTo(b); + end); + + var sortedValues := new real[values.Count]; + var sortedCounts := new integer[values.Count]; + for var i := 0 to order.Count - 1 do + begin + sortedValues[i] := values[order[i]]; + sortedCounts[i] := counts[order[i]]; + end; + + res.AddFloatColumn(colName, sortedValues); + res.AddIntColumn(countColName, sortedCounts); + end; + + ctStr: + begin + var c := StrColumn(col); + var index := new Dictionary; + var values := new List; + + for var i := 0 to c.Data.Length - 1 do + if c.IsValid[i] then + begin + var pos: integer; + if index.TryGetValue(c.Data[i], pos) then + counts[pos] += 1 + else + begin + pos := values.Count; + index[c.Data[i]] := pos; + values.Add(c.Data[i]); + counts.Add(1); + end; + end; + + for var i := 0 to values.Count - 1 do + order.Add(i); + + order.Sort((a, b) -> + begin + Result := counts[b].CompareTo(counts[a]); + if Result = 0 then + Result := a.CompareTo(b); + end); + + var sortedValues := new string[values.Count]; + var sortedCounts := new integer[values.Count]; + for var i := 0 to order.Count - 1 do + begin + sortedValues[i] := values[order[i]]; + sortedCounts[i] := counts[order[i]]; + end; + + res.AddStrColumn(colName, sortedValues); + res.AddIntColumn(countColName, sortedCounts); + end; + + ctBool: + begin + var c := BoolColumn(col); + var index := new Dictionary; + var values := new List; + + for var i := 0 to c.Data.Length - 1 do + if c.IsValid[i] then + begin + var pos: integer; + if index.TryGetValue(c.Data[i], pos) then + counts[pos] += 1 + else + begin + pos := values.Count; + index[c.Data[i]] := pos; + values.Add(c.Data[i]); + counts.Add(1); + end; + end; + + for var i := 0 to values.Count - 1 do + order.Add(i); + + order.Sort((a, b) -> + begin + Result := counts[b].CompareTo(counts[a]); + if Result = 0 then + Result := a.CompareTo(b); + end); + + var sortedValues := new boolean[values.Count]; + var sortedCounts := new integer[values.Count]; + for var i := 0 to order.Count - 1 do + begin + sortedValues[i] := values[order[i]]; + sortedCounts[i] := counts[order[i]]; + end; + + res.AddBoolColumn(colName, sortedValues); + res.AddIntColumn(countColName, sortedCounts); + end; + + ctDateTime: + begin + var c := DateTimeColumn(col); + var index := new Dictionary; + var values := new List; + + for var i := 0 to c.Data.Length - 1 do + if c.IsValid[i] then + begin + var pos: integer; + if index.TryGetValue(c.Data[i], pos) then + counts[pos] += 1 + else + begin + pos := values.Count; + index[c.Data[i]] := pos; + values.Add(c.Data[i]); + counts.Add(1); + end; + end; + + for var i := 0 to values.Count - 1 do + order.Add(i); + + order.Sort((a, b) -> + begin + Result := counts[b].CompareTo(counts[a]); + if Result = 0 then + Result := a.CompareTo(b); + end); + + var sortedValues := new System.DateTime[values.Count]; + var sortedCounts := new integer[values.Count]; + for var i := 0 to order.Count - 1 do + begin + sortedValues[i] := values[order[i]]; + sortedCounts[i] := counts[order[i]]; + end; + + res.AddDateTimeColumn(colName, sortedValues); + res.AddIntColumn(countColName, sortedCounts); + end; + + else + Error(ER_UNKNOWN_COLUMN_TYPE); + end; + + var cats := new boolean[2]; + cats[0] := fSchema.IsCategoricalAt(ci); + cats[1] := false; + res.SetSchema(new DataFrameSchema([colName, countColName], [fSchema.ColumnTypeAt(ci), ctInt], cats)); + Result := res; +end; + +function DataFrame.MissingCount(colName: string): integer; +begin + var col := GetColumn(colName); + Result := 0; + for var i := 0 to col.IsValid.Length - 1 do + if not col.IsValid[i] then + Result += 1; +end; + +function DataFrame.MissingCounts: DataFrame; +begin + var names := fSchema.ColumnNames; + var counts := new integer[names.Length]; + + for var i := 0 to names.Length - 1 do + counts[i] := MissingCount(names[i]); + + var res := new DataFrame; + res.AddStrColumn('Column', names); + res.AddIntColumn('MissingCount', counts); + res.SetSchema(new DataFrameSchema(['Column', 'MissingCount'], [ctStr, ctInt], [false, false])); + Result := res; +end; + +function DataFrame.DropMissing: DataFrame; +begin + var cols := fSchema.ColumnNames; + Result := DropMissing(cols); +end; + +function DataFrame.DropMissing(columns: array of string): DataFrame; +begin + if columns = nil then + ArgumentNullError('columns'); + + if columns.Length = 0 then + begin + if RowCount = 0 then + exit(TakeRows([])); + exit(TakeRows(Arr(0..RowCount - 1))); + end; + + var indices := new integer[columns.Length]; + for var i := 0 to columns.Length - 1 do + indices[i] := ColumnIndex(columns[i]); + + var selected := new List; + var cur := GetCursor; + while cur.MoveNext do + begin + var ok := true; + for var j := 0 to indices.Length - 1 do + if not cur.IsValid(indices[j]) then + begin + ok := false; + break; + end; + + if ok then + selected.Add(cur.Position); + end; + + Result := TakeRows(selected.ToArray); +end; + +function DataFrame.DropDuplicates: DataFrame; +begin + var seen := new HashSet; + var selected := new List; + + for var i := 0 to RowCount - 1 do + begin + var key := BuildRowSignature(Self, i); + if seen.Add(key) then + selected.Add(i); + end; + + Result := TakeRows(selected.ToArray); +end; + +function DataFrame.Clip(colName: string; lower, upper: real): DataFrame; +begin + if lower > upper then + ArgumentError(ER_CLIP_LOWER_GREATER_UPPER); + + var ci := ColumnIndex(colName); + var t := fSchema.ColumnTypeAt(ci); + + case t of + ctInt: + begin + if (not IsIntegralReal(lower)) or (not IsIntegralReal(upper)) then + ArgumentError(ER_CLIP_INT_BOUNDS_NON_INTEGER); + + var lo := Round(lower); + var hi := Round(upper); + Result := MapIntColumnData(colName, v -> PABCSystem.Max(lo, PABCSystem.Min(hi, v))); + end; + + ctFloat: + Result := MapFloatColumnData(colName, v -> PABCSystem.Max(lower, PABCSystem.Min(upper, v))); + + else + Error(ER_COLUMN_NOT_NUMERIC); + end; +end; + function DataFrame.GroupBy(colIndex: integer): IGroupByContext; begin CheckColumnIndex(colIndex); @@ -2980,6 +3715,7 @@ type FloatVals: array of real; StrVals: array of string; BoolVals: array of boolean; + DateTimeVals: array of System.DateTime; end; function DataFrame.SortBy(colIndices: array of integer; descending: array of boolean): DataFrame; @@ -3004,6 +3740,7 @@ begin k.FloatVals := new real[colIndices.Length]; k.StrVals := new string[colIndices.Length]; k.BoolVals := new boolean[colIndices.Length]; + k.DateTimeVals := new System.DateTime[colIndices.Length]; for var i := 0 to colIndices.Length - 1 do begin @@ -3017,6 +3754,7 @@ begin ctFloat: k.FloatVals[i] := cur.Float(c); ctStr: k.StrVals[i] := cur.Str(c); ctBool: k.BoolVals[i] := cur.Bool(c); + ctDateTime: k.DateTimeVals[i] := cur.DateTime(c); end; end; @@ -3042,7 +3780,8 @@ begin ctFloat: cmp := a.FloatVals[i].CompareTo(b.FloatVals[i]); ctStr: cmp := a.StrVals[i].CompareTo(b.StrVals[i]); ctBool: cmp := a.BoolVals[i].CompareTo(b.BoolVals[i]); - end; + ctDateTime: cmp := a.DateTimeVals[i].CompareTo(b.DateTimeVals[i]); + end; if cmp <> 0 then begin @@ -3214,6 +3953,22 @@ begin res.AddBoolColumn(src.Info.Name, data, valid); end + else if col is DateTimeColumn then + begin + var src := DateTimeColumn(col); + + var data := new System.DateTime[newCount]; + var valid := new boolean[newCount]; + + for var k := 0 to newCount - 1 do + begin + var i := mask[k]; + data[k] := src.Data[i]; + valid[k] := src.IsValid[i]; + end; + + res.AddDateTimeColumn(src.Info.Name, data, valid); + end else Error(ER_UNKNOWN_COLUMN_TYPE); end; @@ -3314,6 +4069,12 @@ begin var c := BoolColumn(col); res.AddBoolColumn(newName, c.Data, c.IsValid); end; + + ctDateTime: + begin + var c := DateTimeColumn(col); + res.AddDateTimeColumn(newName, c.Data, c.IsValid); + end; end; end; @@ -3408,6 +4169,8 @@ begin cols.Add(new StrColumn(schema.ColumnNames[i])); ctBool: cols.Add(new BoolColumn(schema.ColumnNames[i])); + ctDateTime: + cols.Add(new DateTimeColumn(schema.ColumnNames[i])); end; end; @@ -3452,6 +4215,12 @@ begin var c := BoolColumn(col); Result.AddBoolColumn(c.Info.Name, c.Data, c.IsValid); end; + + ctDateTime: + begin + var c := DateTimeColumn(col); + Result.AddDateTimeColumn(c.Info.Name, c.Data, c.IsValid); + end; end; end; end; @@ -3564,6 +4333,71 @@ begin Result.AssertSchemaConsistent; end; +function DataFrame.WithDatePart(sourceColumn, newColumnName: string; part: DatePartKind): DataFrame; +begin + if sourceColumn = nil then + ArgumentNullError(ER_ARG_NULL, 'sourceColumn'); + if newColumnName = nil then + ArgumentNullError(ER_ARG_NULL, 'newColumnName'); + if not HasColumn(sourceColumn) then + ArgumentError(ER_COLUMN_NOT_FOUND, sourceColumn); + if GetColumnType(sourceColumn) <> ColumnType.ctDateTime then + ArgumentError(ER_COLUMN_NOT_DATETIME, sourceColumn); + + if part <> dpDate then + begin + case part of + dpYear: + exit(WithColumnInt(newColumnName, cur -> cur.DateTime(sourceColumn).Year)); + dpMonth: + exit(WithColumnInt(newColumnName, cur -> cur.DateTime(sourceColumn).Month)); + dpDay: + exit(WithColumnInt(newColumnName, cur -> cur.DateTime(sourceColumn).Day)); + dpHour: + exit(WithColumnInt(newColumnName, cur -> cur.DateTime(sourceColumn).Hour)); + dpMinute: + exit(WithColumnInt(newColumnName, cur -> cur.DateTime(sourceColumn).Minute)); + dpSecond: + exit(WithColumnInt(newColumnName, cur -> cur.DateTime(sourceColumn).Second)); + dpDayOfWeek: + exit(WithColumnInt(newColumnName, cur -> integer(cur.DateTime(sourceColumn).DayOfWeek))); + end; + end; + + var res := CloneWithSharedColumns; + + var data := new System.DateTime[RowCount]; + var valid := new boolean[RowCount]; + + var cur := GetCursor; + var i := 0; + + while cur.MoveNext do + begin + data[i] := cur.DateTime(sourceColumn).Date; + valid[i] := True; + i += 1; + end; + + res.AddDateTimeColumn(newColumnName, data, valid); + res.SetSchema(ExtendSchema(newColumnName, ctDateTime, false)); + + Result := res; + Result.AssertSchemaConsistent; +end; + +function DataFrame.WithDateParts(sourceColumn: string; parts: array of (string, DatePartKind)): DataFrame; +begin + if sourceColumn = nil then + ArgumentNullError(ER_ARG_NULL, 'sourceColumn'); + if (parts = nil) or (parts.Length = 0) then + ArgumentError(ER_DATEPARTS_EMPTY); + + Result := Self; + foreach var partInfo in parts do + Result := Result.WithDatePart(sourceColumn, partInfo[0], partInfo[1]); +end; + function DataFrame.ExtendSchema( name: string; colType: ColumnType; @@ -4076,6 +4910,23 @@ begin res.AddBoolColumn(name, data, validDst); end; + ctDateTime: + begin + var src := DateTimeColumn(col); + var data := new System.DateTime[k]; + var validDst := new boolean[k]; + + for var j := 0 to k - 1 do + begin + var i := indices[j]; + + data[j] := src.Data[i]; + validDst[j] := src.IsValid[i]; + end; + + res.AddDateTimeColumn(name, data, validDst); + end; + else Error(ER_UNSUPPORTED_COLUMN_TYPE, colType); end; @@ -4094,7 +4945,8 @@ begin Result := res; end; -procedure DataFrame.Print(maxRows: integer; headRows: integer; decimals: integer); +procedure DataFrame.Print(maxRows: integer; headRows: integer; decimals: integer; + dateTimeFormat: string); begin var colCount := columns.Count; if colCount = 0 then exit; @@ -4153,6 +5005,7 @@ begin ctFloat: s := cursor.Float(j).ToString('F' + decimals); ctStr: s := cursor.Str(j); ctBool: s := cursor.Bool(j).ToString; + ctDateTime: s := FormatDateTimeForPrint(cursor.DateTime(j), dateTimeFormat); end; if s.Length > widths[j] then @@ -4179,6 +5032,7 @@ begin ctFloat: s := cursor.Float(j).ToString('F' + decimals); ctStr: s := cursor.Str(j); ctBool: s := cursor.Bool(j).ToString; + ctDateTime: s := FormatDateTimeForPrint(cursor.DateTime(j), dateTimeFormat); end; if s.Length > widths[j] then @@ -4213,6 +5067,7 @@ begin ctFloat: s := cursor.Float(j).ToString('F' + decimals); ctStr: s := cursor.Str(j); ctBool: s := cursor.Bool(j).ToString; + ctDateTime: s := FormatDateTimeForPrint(cursor.DateTime(j), dateTimeFormat); end; PABCSystem.Print(s.PadLeft(widths[j])); @@ -4255,6 +5110,7 @@ begin ctFloat: s := cursor.Float(j).ToString('F' + decimals); ctStr: s := cursor.Str(j); ctBool: s := cursor.Bool(j).ToString; + ctDateTime: s := FormatDateTimeForPrint(cursor.DateTime(j), dateTimeFormat); end; PABCSystem.Print(s.PadLeft(widths[j])); @@ -4287,6 +5143,7 @@ begin ctFloat: Result := 'float'; ctStr: Result := 'string'; ctBool: Result := 'bool'; + ctDateTime: Result := 'datetime'; end; end; @@ -4391,6 +5248,8 @@ begin AddStrColumn(src.Info.Name, StrColumn(src).Data, StrColumn(src).IsValid); ctBool: AddBoolColumn(src.Info.Name, BoolColumn(src).Data, BoolColumn(src).IsValid); + ctDateTime: + AddDateTimeColumn(src.Info.Name, DateTimeColumn(src).Data, DateTimeColumn(src).IsValid); end; end; @@ -4468,6 +5327,28 @@ begin Result := CsvLoader.LoadFromLines(text.ToLines); end; +static function DataFrame.FromODS(filename: string; sheet: string): DataFrame; +begin + Result := LoadFromRows( + ODSReader.ReadSheet(filename, sheet), + true, + nil, + true, + false, + nil, + 1000, + nil, + true, + nil, + nil, + false, + 100, + 0.2, + 0.8, + 5 + ); +end; + static function DataFrame.Concat(params dfs: array of DataFrame): DataFrame; begin if (dfs = nil) or (dfs.Length = 0) then @@ -4584,6 +5465,26 @@ begin res.AddBoolColumn(name, data, valid); end; + + ctDateTime: + begin + var data := new System.DateTime[totalRows]; + var valid := new boolean[totalRows]; + var pos := 0; + + for var di := 0 to dfs.Length - 1 do + begin + var col := DateTimeColumn(dfs[di].columns[ci]); + for var r := 0 to col.Data.Length - 1 do + begin + data[pos] := col.Data[r]; + valid[pos] := col.IsValid[r]; + pos += 1; + end; + end; + + res.AddDateTimeColumn(name, data, valid); + end; end; end; @@ -4891,6 +5792,7 @@ begin ctInt: key := cursor.Int(keyColumn); ctStr: key := cursor.Str(keyColumn); ctBool: key := cursor.Bool(keyColumn); + ctDateTime: key := cursor.DateTime(keyColumn); else Error(ER_GROUPBY_UNSUPPORTED_KEY_TYPE, df.columns[keyColumn].Info.ColType); end; @@ -4929,6 +5831,7 @@ begin ctInt: values[i] := cursor.Int(c); ctStr: values[i] := cursor.Str(c); ctBool: values[i] := cursor.Bool(c); + ctDateTime: values[i] := cursor.DateTime(c); else Error(ER_GROUPBY_UNSUPPORTED_KEY_TYPE, df.columns[c].Info.ColType); end; end; @@ -4972,6 +5875,8 @@ begin keys := groups1.Keys.OrderBy(k -> string(k)).Select(k -> object(k)).ToArray; ctBool: keys := groups1.Keys.OrderBy(k -> boolean(k)).Select(k -> object(k)).ToArray; + ctDateTime: + keys := groups1.Keys.OrderBy(k -> System.DateTime(k)).Select(k -> object(k)).ToArray; else Error(ER_GROUPBY_UNSUPPORTED_KEY_TYPE); end; @@ -5000,6 +5905,12 @@ begin res.AddBoolColumn(keyName, keys.Select(k -> boolean(k)).ToArray, nil); types.Add(ctBool); end; + + ctDateTime: + begin + res.AddDateTimeColumn(keyName, keys.Select(k -> System.DateTime(k)).ToArray, nil); + types.Add(ctDateTime); + end; end; names.Add(keyName); @@ -5042,6 +5953,12 @@ begin res.AddBoolColumn(colName, keys.Select(key -> boolean(key.Values[k])).ToArray, nil); types.Add(ctBool); end; + + ctDateTime: + begin + res.AddDateTimeColumn(colName, keys.Select(key -> System.DateTime(key.Values[k])).ToArray, nil); + types.Add(ctDateTime); + end; else Error(ER_GROUPBY_UNSUPPORTED_KEY_TYPE); @@ -5133,6 +6050,7 @@ begin ctInt: keys := groups1.Keys.OrderBy(k -> integer(k)).Select(k -> object(k)).ToArray; ctStr: keys := groups1.Keys.OrderBy(k -> string(k)).Select(k -> object(k)).ToArray; ctBool: keys := groups1.Keys.OrderBy(k -> boolean(k)).Select(k -> object(k)).ToArray; + ctDateTime: keys := groups1.Keys.OrderBy(k -> System.DateTime(k)).Select(k -> object(k)).ToArray; else Error(ER_GROUPBY_UNSUPPORTED_KEY_TYPE); end; @@ -5295,7 +6213,13 @@ begin .OrderBy(k -> boolean(k)) .Select(k -> object(k)) .ToArray; - + + ctDateTime: + keys1 := groups1.Keys + .OrderBy(k -> System.DateTime(k)) + .Select(k -> object(k)) + .ToArray; + else Error(ER_GROUPBY_UNSUPPORTED_KEY_TYPE); end; @@ -5429,6 +6353,8 @@ begin res.AddStrColumn(keyName, keys1.Select(k -> string(k)).ToArray, nil); ctBool: res.AddBoolColumn(keyName, keys1.Select(k -> boolean(k)).ToArray, nil); + ctDateTime: + res.AddDateTimeColumn(keyName, keys1.Select(k -> System.DateTime(k)).ToArray, nil); else Error(ER_GROUPBY_UNSUPPORTED_KEY_TYPE, source.fSchema.ColumnTypeAt(keyColumn)); end; @@ -5452,6 +6378,8 @@ begin res.AddStrColumn(keyName, keysN.Select(key -> string(key.Values[k])).ToArray, nil); ctBool: res.AddBoolColumn(keyName, keysN.Select(key -> boolean(key.Values[k])).ToArray, nil); + ctDateTime: + res.AddDateTimeColumn(keyName, keysN.Select(key -> System.DateTime(key.Values[k])).ToArray, nil); else Error(ER_GROUPBY_UNSUPPORTED_KEY_TYPE, source.fSchema.ColumnTypeAt(ci)); end; @@ -6257,6 +7185,828 @@ begin begin value := False; exit(True) end; end; +function TryStrToDateTime(s: string; var value: System.DateTime): boolean; +begin + var t := s; + if t <> nil then + t := t.Trim; + + if (t = nil) or (t = '') then + exit(False); + + var ru := CultureInfo.GetCultureInfo('ru-RU'); + + var formats := [ + 'yyyy-MM-dd', + 'yyyy-MM-dd HH:mm:ss', + 'yyyy-MM-dd HH:mm', + 'yyyy-MM-ddTHH:mm:ss', + 'yyyy-MM-ddTHH:mm', + 'yyyy-MM-ddTHH:mm:ss.FFF', + 'yyyy-MM-ddTHH:mm:ss.fffffff', + 'dd.MM.yyyy', + 'dd.MM.yyyy HH:mm:ss', + 'dd.MM.yyyy HH:mm', + 's', + 'o' + ]; + + Result := System.DateTime.TryParseExact( + t, + formats, + CultureInfo.InvariantCulture, + DateTimeStyles.None, + value + ); + + if Result then + exit; + + Result := System.DateTime.TryParseExact( + t, + formats, + ru, + DateTimeStyles.None, + value + ); + + if Result then + exit; + + Result := System.DateTime.TryParse( + t, + CultureInfo.InvariantCulture, + DateTimeStyles.None, + value + ); + + if Result then + exit; + + Result := System.DateTime.TryParse( + t, + ru, + DateTimeStyles.None, + value + ); +end; + +function TryStrToBoolStrict(s: string; var value: boolean): boolean; +begin + if s = nil then + exit(False); + Result := TryStrToBoolStrictRange(s, 1, s.Length, value); +end; + +function IsBlankRow(row: array of string): boolean; +begin + if row = nil then + exit(True); + + foreach var s in row do + if (s <> nil) and (s.Trim <> '') then + exit(False); + + Result := True; +end; + +function IsBlankStringArray(row: StringArray): boolean; +begin + Result := IsBlankRow(row); +end; + +function CollectSelectorValues(df: DataFrame; selector: DataFrameCursor -> DataValue): List; +begin + if selector = nil then + ArgumentNullError('selector'); + + Result := new List; + + var cursor := df.GetCursor; + while cursor.MoveNext do + begin + var v := selector(cursor); + if (not System.Object.ReferenceEquals(v, nil)) and v.IsValid then + Result.Add(v.Float); + end; +end; + +function IsIntegralReal(x: real): boolean; +begin + Result := Abs(x - Round(x)) < 1e-9; +end; + +function BuildRowSignature(df: DataFrame; rowIndex: integer): string; +begin + var sb := new StringBuilder; + var schema := df.Schema; + var cur := df.GetCursor; + cur.MoveTo(rowIndex); + + for var i := 0 to schema.ColumnCount - 1 do + begin + sb.Append('|'); + + if not cur.IsValid(i) then + begin + sb.Append('NA'); + continue; + end; + + case schema.ColumnTypeAt(i) of + ctInt: + sb.Append('I:').Append(cur.Int(i)); + ctFloat: + sb.Append('F:').Append(cur.Float(i).ToString('R', CultureInfo.InvariantCulture)); + ctStr: + begin + var s := cur.Str(i); + if s = nil then + s := ''; + sb.Append('S:').Append(s.Length).Append(':').Append(s); + end; + ctBool: + sb.Append('B:').Append(if cur.Bool(i) then '1' else '0'); + ctDateTime: + sb.Append('D:').Append(cur.DateTime(i).Ticks); + else + Error(ER_UNKNOWN_COLUMN_TYPE); + end; + end; + + Result := sb.ToString; +end; + +function ParseCsvField(raw: string): string; +begin + Result := raw; + if Result = nil then + exit; + + if (Result.Length >= 2) and (Result[1] = '"') and (Result[Result.Length] = '"') then + Result := Result.Substring(1, Result.Length - 2); +end; + +function ParseCsvLine(line: string; delimiter: char; strict: boolean): array of string; +begin + var tmpStarts := new integer[64]; + var tmpLens := new integer[64]; + var actualCount: integer; + var unclosedQuote: boolean; + + ScanFieldsQuoted(line, delimiter, tmpStarts, tmpLens, actualCount, unclosedQuote); + + if unclosedQuote and strict then + Error(ER_CSV_UNCLOSED_QUOTE); + + Result := new string[actualCount]; + + for var j := 0 to actualCount - 1 do + begin + var s := line.Substring(tmpStarts[j] - 1, tmpLens[j]); + Result[j] := ParseCsvField(s); + end; +end; + +function ParseCsvLinesToRows(lines: sequence of string; delimiter: char; strict: boolean): array of StringArray; +begin + var raw := lines.ToArray; + Result := new StringArray[raw.Length]; + + for var i := 0 to raw.Length - 1 do + Result[i] := ParseCsvLine(raw[i], delimiter, strict); +end; + +function GetXmlAttr(node: XmlNode; localName: string): string; +begin + if (node = nil) or (node.Attributes = nil) then + exit(nil); + + for var i := 0 to node.Attributes.Count - 1 do + begin + var attr := node.Attributes[i]; + if (attr.LocalName = localName) or (attr.Name = localName) then + exit(attr.Value); + end; + + Result := nil; +end; + +function GetRepeatedCount(node: XmlNode; localName: string): integer; +begin + Result := 1; + var s := GetXmlAttr(node, localName); + if (s = nil) or (s = '') then + exit; + + var n: integer; + if System.Int32.TryParse(s, n) and (n > 0) then + Result := n; +end; + +function OdsCellValue(cell: XmlNode): string; +begin + if cell.LocalName = 'covered-table-cell' then + exit(''); + + var textValue := cell.InnerText; + var valueType := GetXmlAttr(cell, 'value-type'); + + if valueType = 'boolean' then + begin + Result := GetXmlAttr(cell, 'boolean-value'); + if (Result = nil) or (Result = '') then + Result := textValue; + if Result <> nil then + Result := Result.ToLower; + exit; + end; + + if (valueType = 'float') or (valueType = 'currency') or (valueType = 'percentage') then + begin + Result := GetXmlAttr(cell, 'value'); + if (Result = nil) or (Result = '') then + Result := textValue; + exit; + end; + + if valueType = 'date' then + begin + Result := GetXmlAttr(cell, 'date-value'); + if (Result = nil) or (Result = '') then + Result := textValue; + exit; + end; + + if valueType = 'time' then + begin + Result := GetXmlAttr(cell, 'time-value'); + if (Result = nil) or (Result = '') then + Result := textValue; + exit; + end; + + Result := GetXmlAttr(cell, 'string-value'); + if (Result = nil) or (Result = '') then + Result := textValue; + if Result = nil then + Result := ''; +end; + +function FindOdsTableNode(doc: XmlDocument; sheet: string): XmlNode; +begin + Result := nil; + var firstTable: XmlNode := nil; + + var nodes := doc.GetElementsByTagName('*'); + for var i := 0 to nodes.Count - 1 do + begin + var node := nodes[i]; + if node.LocalName <> 'table' then + continue; + + if firstTable = nil then + firstTable := node; + + var name := GetXmlAttr(node, 'name'); + if (sheet <> nil) and (sheet <> '') and (name = sheet) then + begin + Result := node; + exit; + end; + end; + + if firstTable = nil then + Error(ER_ODS_NO_SHEETS); + + if (sheet = nil) or (sheet = '') then + Result := firstTable + else + Error(ER_ODS_SHEET_NOT_FOUND, sheet); +end; + +static function ODSReader.ReadSheet(filename: string; sheet: string): array of StringArray; +begin + if (filename = nil) or (filename.Trim = '') then + ArgumentError(ER_ODS_FILENAME_EMPTY); + + var fs := new FileStream(filename, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + try + var zip := new ZipArchive(fs, ZipArchiveMode.Read); + try + var entry := zip.GetEntry('content.xml'); + if entry = nil then + Error(ER_ODS_CONTENT_XML_NOT_FOUND); + + var doc := new XmlDocument; + var stream := entry.Open; + try + doc.Load(stream); + finally + stream.Close; + end; + + var tableNode := FindOdsTableNode(doc, sheet); + var rowsList := new List; + var pendingBlankRows := 0; + + foreach var rowNodeObj in tableNode.ChildNodes do + begin + var rowNode := XmlNode(rowNodeObj); + if rowNode.LocalName <> 'table-row' then + continue; + + var rowCells := new List; + var pendingEmptyCells := 0; + + foreach var cellNodeObj in rowNode.ChildNodes do + begin + var cellNode := XmlNode(cellNodeObj); + if (cellNode.LocalName <> 'table-cell') and (cellNode.LocalName <> 'covered-table-cell') then + continue; + + var repeatCols := GetRepeatedCount(cellNode, 'number-columns-repeated'); + var cellValue := OdsCellValue(cellNode); + + if cellValue = '' then + pendingEmptyCells += repeatCols + else + begin + for var k := 1 to pendingEmptyCells do + rowCells.Add(''); + pendingEmptyCells := 0; + + for var k := 1 to repeatCols do + rowCells.Add(cellValue); + end; + end; + + var repeatRows := GetRepeatedCount(rowNode, 'number-rows-repeated'); + var rowArray := rowCells.ToArray; + + if IsBlankStringArray(rowArray) then + pendingBlankRows += repeatRows + else + begin + for var k := 1 to pendingBlankRows do + rowsList.Add(new string[0]); + pendingBlankRows := 0; + + for var k := 1 to repeatRows do + rowsList.Add(Copy(rowArray)); + end; + end; + + Result := rowsList.ToArray; + finally + zip.Dispose; + end; + finally + fs.Close; + end; +end; + +function LoadFromRows( + rows: array of StringArray; + hasHeader: boolean; + missingValues: array of string; + trimWhitespace: boolean; + strict: boolean; + schema: Dictionary; + sampleSize: integer; + ignoreColumns: array of string; + inferTypes: boolean; + forceStringColumns: array of string; + categoricalColumns: array of string; + inferCategorical: boolean; + maxCategoricalCardinality: integer; + maxCategoricalRatio: real; + idThreshold: real; + minFrequency: integer +): DataFrame; +begin + var missing := if missingValues = nil then + new HashSet(Arr('', 'NA', 'NaN', 'null')) + else + new HashSet(missingValues); + + var l := 0; + while (l < rows.Length) and IsBlankRow(rows[l]) do + l += 1; + + var r := rows.Length; + while (r > l) and IsBlankRow(rows[r - 1]) do + r -= 1; + + var rowsArray := rows[l:r]; + + var headers: array of string := nil; + var originalColCount := 0; + var rowCount := rowsArray.Length; + if hasHeader then + rowCount -= 1; + + var canBool, canInt, canFloat, canDateTime: array of boolean; + (canBool, canInt, canFloat, canDateTime) := (nil, nil, nil, nil); + + var inferLimit := sampleSize; + if inferLimit <= 0 then + inferLimit := integer.MaxValue; + + var ignoreSet := if ignoreColumns = nil then nil else new HashSet(ignoreColumns); + var forceStrSet := if forceStringColumns = nil then nil else new HashSet(forceStringColumns); + var catSet := if categoricalColumns = nil then nil else new HashSet(categoricalColumns); + + var map: array of integer := nil; + var newColCount := 0; + + var uniqueSet: array of HashSet := nil; + var freqMap: array of Dictionary := nil; + var nonMissingCount: array of integer := nil; + var autoCat: array of boolean := nil; + + var first := true; + + foreach var currentRaw in rowsArray index inferRead do + begin + if inferRead >= inferLimit then break; + + var current := if currentRaw = nil then new string[0] else currentRaw; + + if first then + begin + if hasHeader then + begin + headers := Copy(current); + originalColCount := headers.Length; + end + else + begin + originalColCount := current.Length; + headers := ArrGen(originalColCount, i -> 'C' + i.ToString); + first := false; + end; + + map := new integer[originalColCount]; + var idx := 0; + for var j := 0 to originalColCount - 1 do + if (ignoreSet <> nil) and (headers[j] in ignoreSet) then + map[j] := -1 + else + begin + map[j] := idx; + idx += 1; + end; + + newColCount := idx; + + var newHeaders := new string[newColCount]; + for var j := 0 to originalColCount - 1 do + if map[j] <> -1 then + newHeaders[map[j]] := headers[j]; + + headers := newHeaders; + + canBool := new boolean[newColCount]; + canInt := new boolean[newColCount]; + canFloat := new boolean[newColCount]; + canDateTime := new boolean[newColCount]; + + for var j := 0 to newColCount - 1 do + begin + canBool[j] := inferTypes; + canInt[j] := inferTypes; + canFloat[j] := inferTypes; + canDateTime[j] := inferTypes; + end; + + uniqueSet := new HashSet[newColCount]; + freqMap := new Dictionary[newColCount]; + nonMissingCount := new integer[newColCount]; + autoCat := new boolean[newColCount]; + + for var j := 0 to newColCount - 1 do + begin + uniqueSet[j] := new HashSet; + freqMap[j] := new Dictionary; + nonMissingCount[j] := 0; + autoCat[j] := false; + end; + + for var j := 0 to newColCount - 1 do + begin + var name := headers[j]; + + if (schema <> nil) and schema.ContainsKey(name) then + begin + canBool[j] := false; + canInt[j] := false; + canFloat[j] := false; + canDateTime[j] := false; + + case schema[name] of + ctBool: canBool[j] := true; + ctInt: canInt[j] := true; + ctFloat: canFloat[j] := true; + ctDateTime: canDateTime[j] := true; + ctStr: ; + end; + + continue; + end; + + if (forceStrSet <> nil) and (name in forceStrSet) then + begin + canBool[j] := false; + canInt[j] := false; + canFloat[j] := false; + canDateTime[j] := false; + continue; + end; + + if (catSet <> nil) and (name in catSet) then + begin + canBool[j] := false; + canInt[j] := false; + canFloat[j] := false; + canDateTime[j] := false; + continue; + end; + end; + + if headers = nil then + Error(ER_EMPTY_CSV); + + if hasHeader then + begin + first := false; + continue; + end; + + first := false; + end; + + if (current.Length <> originalColCount) and strict then + Error(ER_CSV_COLUMN_COUNT_MISMATCH, originalColCount, current.Length); + + for var j := 0 to originalColCount - 1 do + begin + var nj := map[j]; + if nj = -1 then continue; + + var s := if j < current.Length then current[j] else ''; + if s = nil then + s := ''; + if trimWhitespace then + s := s.Trim; + if s in missing then continue; + + nonMissingCount[nj] += 1; + + var name := headers[nj]; + + if inferCategorical then + begin + if uniqueSet[nj].Count <= maxCategoricalCardinality * 2 then + begin + uniqueSet[nj].Add(s); + if freqMap[nj].ContainsKey(s) then + freqMap[nj][s] += 1 + else + freqMap[nj][s] := 1; + end; + end; + + if not inferTypes then continue; + if (schema <> nil) and schema.ContainsKey(name) then continue; + if (forceStrSet <> nil) and (name in forceStrSet) then continue; + if (catSet <> nil) and (name in catSet) then continue; + + var sl := s; + + if not ((sl = 'true') or (sl = 'false') or (sl = 'True') or (sl = 'False') or + (sl = 'yes') or (sl = 'no') or (sl = 'Yes') or (sl = 'No')) then + canBool[nj] := false; + + var iv: integer; + if not TryStrToInt(s, iv) then + canInt[nj] := false; + + var fv: real; + if not TryStrToReal(s, fv) then + canFloat[nj] := false; + + var dtv: System.DateTime; + if not TryStrToDateTime(s, dtv) then + canDateTime[nj] := false; + end; + end; + + if headers = nil then + Error(ER_EMPTY_CSV); + + if inferCategorical then + for var j := 0 to newColCount - 1 do + begin + var name := headers[j]; + + if (schema <> nil) and schema.ContainsKey(name) then continue; + if (forceStrSet <> nil) and (name in forceStrSet) then continue; + if (catSet <> nil) and (name in catSet) then continue; + + if canBool[j] or canInt[j] or canFloat[j] or canDateTime[j] then continue; + + var uc := uniqueSet[j].Count; + if uc = 0 then continue; + + var n := nonMissingCount[j]; + if n = 0 then continue; + + var ratio := uc / n; + + var maxFreq := 0; + foreach var kv in freqMap[j] do + if kv.Value > maxFreq then + maxFreq := kv.Value; + + var isLowCardinality := uc <= maxCategoricalCardinality; + var isReasonableRatio := ratio <= maxCategoricalRatio; + + if isLowCardinality and + (isReasonableRatio or (uc <= 100)) and + (ratio < idThreshold) and + (maxFreq >= minFrequency) then + autoCat[j] := true; + end; + + var df := new DataFrame; + + var intData := new IntArray[newColCount]; + var floatData := new RealArray[newColCount]; + var strData := new StringArray[newColCount]; + var boolData := new BoolArray[newColCount]; + var dtData := new DateTimeArray[newColCount]; + var valid := new BoolArray[newColCount]; + + for var j := 0 to newColCount - 1 do + begin + valid[j] := new boolean[rowCount]; + + if canBool[j] then + boolData[j] := new boolean[rowCount] + else if canInt[j] then + intData[j] := new integer[rowCount] + else if canFloat[j] then + floatData[j] := new real[rowCount] + else if canDateTime[j] then + dtData[j] := new System.DateTime[rowCount] + else + strData[j] := new string[rowCount]; + end; + + var row := 0; + first := true; + foreach var currentRaw in rowsArray do + begin + var current := if currentRaw = nil then new string[0] else currentRaw; + + if first then + begin + first := false; + if hasHeader then continue; + end; + + if (current.Length <> originalColCount) and strict then + Error(ER_CSV_COLUMN_COUNT_MISMATCH, originalColCount, current.Length); + + for var j := 0 to originalColCount - 1 do + begin + var nj := map[j]; + if nj = -1 then continue; + + var s := if j < current.Length then current[j] else ''; + if s = nil then + s := ''; + if trimWhitespace then + s := s.Trim; + + if s in missing then + begin + valid[nj][row] := false; + continue; + end; + + if canBool[nj] then + begin + var bv: boolean; + if TryStrToBoolStrict(s, bv) then + begin + boolData[nj][row] := bv; + valid[nj][row] := true; + end + else + begin + if strict then + Error(ER_CSV_INVALID_BOOL, s, headers[nj]); + valid[nj][row] := false; + end; + end + else if canInt[nj] then + begin + var iv: integer; + if TryStrToInt(s, iv) then + begin + intData[nj][row] := iv; + valid[nj][row] := true; + end + else + begin + if strict then + raise new Exception($'Invalid int "{s}" in column {headers[nj]}'); + valid[nj][row] := false; + end; + end + else if canFloat[nj] then + begin + var fv: real; + if TryStrToReal(s, fv) then + begin + floatData[nj][row] := fv; + valid[nj][row] := true; + end + else + begin + if strict then + raise new Exception($'Invalid float "{s}" in column {headers[nj]}'); + valid[nj][row] := false; + end; + end + else if canDateTime[nj] then + begin + var dtv: System.DateTime; + if TryStrToDateTime(s, dtv) then + begin + dtData[nj][row] := dtv; + valid[nj][row] := true; + end + else + begin + if strict then + raise new Exception($'Invalid DateTime "{s}" in column {headers[nj]}'); + valid[nj][row] := false; + end; + end + else + begin + strData[nj][row] := s; + valid[nj][row] := true; + end; + end; + + row += 1; + end; + + for var j := 0 to newColCount - 1 do + begin + if canBool[j] then + df.AddBoolColumn(headers[j], boolData[j], valid[j]) + else if canInt[j] then + df.AddIntColumn(headers[j], intData[j], valid[j]) + else if canFloat[j] then + df.AddFloatColumn(headers[j], floatData[j], valid[j]) + else if canDateTime[j] then + df.AddDateTimeColumn(headers[j], dtData[j], valid[j]) + else + df.AddStrColumn(headers[j], strData[j], valid[j]); + end; + + var names := new string[newColCount]; + var types := new ColumnType[newColCount]; + var cats := new boolean[newColCount]; + + for var j := 0 to newColCount - 1 do + begin + names[j] := headers[j]; + + if canBool[j] then + types[j] := ctBool + else if canInt[j] then + types[j] := ctInt + else if canFloat[j] then + types[j] := ctFloat + else if canDateTime[j] then + types[j] := ctDateTime + else + types[j] := ctStr; + + cats[j] := ((catSet <> nil) and (headers[j] in catSet)) or autoCat[j]; + end; + + df.SetSchema(new DataFrameSchema(names, types, cats)); + + Result := df; +end; + static function CSVLoader.LoadFromLines( lines: sequence of string; @@ -6301,8 +8051,8 @@ begin if hasHeader then rowCount -= 1; - var canBool, canInt, canFloat: array of boolean; - (canBool, canInt, canFloat) := (nil, nil, nil); + var canBool, canInt, canFloat, canDateTime: array of boolean; + (canBool, canInt, canFloat, canDateTime) := (nil, nil, nil, nil); var inferLimit := sampleSize; if inferLimit <= 0 then @@ -6322,7 +8072,6 @@ begin var first := true; - // временные буферы для определения числа колонок var tmpStarts := new integer[64]; var tmpLens := new integer[64]; var actualCount: integer; @@ -6334,7 +8083,6 @@ begin if strict then Error(ER_CSV_UNCLOSED_QUOTE); - // число колонок var maxColumns := 256; var starts := new integer[maxColumns]; @@ -6358,7 +8106,6 @@ begin begin var s := line.Substring(starts[j] - 1, lens[j]); - // снять кавычки, если есть if (s.Length >= 2) and (s[1] = '"') and (s[s.Length] = '"') then s := s.Substring(1, s.Length - 2); @@ -6401,12 +8148,14 @@ begin canBool := new boolean[newColCount]; canInt := new boolean[newColCount]; canFloat := new boolean[newColCount]; + canDateTime := new boolean[newColCount]; for var j := 0 to newColCount - 1 do begin canBool[j] := inferTypes; canInt[j] := inferTypes; canFloat[j] := inferTypes; + canDateTime[j] := inferTypes; end; uniqueSet := new HashSet[newColCount]; @@ -6431,11 +8180,13 @@ begin canBool[j] := false; canInt[j] := false; canFloat[j] := false; + canDateTime[j] := false; case schema[name] of ctBool: canBool[j] := true; ctInt: canInt[j] := true; ctFloat: canFloat[j] := true; + ctDateTime: canDateTime[j] := true; ctStr: ; end; @@ -6447,6 +8198,7 @@ begin canBool[j] := false; canInt[j] := false; canFloat[j] := false; + canDateTime[j] := false; continue; end; @@ -6455,6 +8207,7 @@ begin canBool[j] := false; canInt[j] := false; canFloat[j] := false; + canDateTime[j] := false; continue; end; end; @@ -6475,7 +8228,6 @@ begin begin var s := line.Substring(starts[j]-1, lens[j]); - // снять кавычки, если есть if (s.Length >= 2) and (s[1] = '"') and (s[s.Length] = '"') then s := s.Substring(1, s.Length - 2); @@ -6536,6 +8288,10 @@ begin var fv: real; if not TryStrToReal(s, fv) then canFloat[nj] := false; + + var dtv: System.DateTime; + if not TryStrToDateTime(s, dtv) then + canDateTime[nj] := false; end; end; @@ -6551,7 +8307,7 @@ begin if (forceStrSet <> nil) and (name in forceStrSet) then continue; if (catSet <> nil) and (name in catSet) then continue; - if canBool[j] or canInt[j] or canFloat[j] then continue; + if canBool[j] or canInt[j] or canFloat[j] or canDateTime[j] then continue; var uc := uniqueSet[j].Count; if uc = 0 then continue; @@ -6582,6 +8338,7 @@ begin var floatData := new RealArray[newColCount]; var strData := new StringArray[newColCount]; var boolData := new BoolArray[newColCount]; + var dtData := new DateTimeArray[newColCount]; var valid := new BoolArray[newColCount]; for var j := 0 to newColCount - 1 do @@ -6594,6 +8351,8 @@ begin intData[j] := new integer[rowCount] else if canFloat[j] then floatData[j] := new real[rowCount] + else if canDateTime[j] then + dtData[j] := new System.DateTime[rowCount] else strData[j] := new string[rowCount]; end; @@ -6710,6 +8469,25 @@ begin valid[nj][row] := false; end; end + else if canDateTime[nj] then + begin + var dtv: System.DateTime; + var s := if lens[j] > 0 then line.Substring(starts[j]-1, lens[j]) else ''; + if trimWhitespace then + s := s.Trim; + + if TryStrToDateTime(s, dtv) then + begin + dtData[nj][row] := dtv; + valid[nj][row] := true; + end + else + begin + if strict then + raise new Exception($'Invalid DateTime "{s}" in column {headers[nj]}'); + valid[nj][row] := false; + end; + end else begin var s := if lens[j] > 0 then line.Substring(starts[j]-1, lens[j]) else ''; @@ -6731,6 +8509,8 @@ begin df.AddIntColumn(headers[j], intData[j], valid[j]) else if canFloat[j] then df.AddFloatColumn(headers[j], floatData[j], valid[j]) + else if canDateTime[j] then + df.AddDateTimeColumn(headers[j], dtData[j], valid[j]) else df.AddStrColumn(headers[j], strData[j], valid[j]); end; @@ -6749,10 +8529,11 @@ begin types[j] := ctInt else if canFloat[j] then types[j] := ctFloat + else if canDateTime[j] then + types[j] := ctDateTime else types[j] := ctStr; - // вот сюда переносим логику categorical cats[j] := ((catSet <> nil) and (headers[j] in catSet)) or autoCat[j]; end; @@ -6803,6 +8584,17 @@ begin end; end; +function FormatDateTimeForPrint(dt: System.DateTime; fmt: string): string; +begin + if fmt <> nil then + exit(dt.ToString(fmt, CultureInfo.InvariantCulture)); + + if dt.TimeOfDay = System.TimeSpan.Zero then + exit(dt.ToString('yyyy-MM-dd', CultureInfo.InvariantCulture)); + + Result := dt.ToString('yyyy-MM-dd HH:mm:ss', CultureInfo.InvariantCulture); +end; + static function CSVLoader.Load(filename: string; delimiter: char; hasHeader: boolean; encoding: Encoding; @@ -6901,6 +8693,7 @@ begin ctFloat: w.Write(cur.Float(i)); ctStr: w.Write(EscapeCsv(cur.Str(i), delimiter)); ctBool: w.Write(cur.Bool(i)); + ctDateTime: w.Write(cur.DateTime(i).ToString('s', CultureInfo.InvariantCulture)); end; end; diff --git a/bin/Lib/DataFrameABCCore.pas b/bin/Lib/DataFrameABCCore.pas index e61b28564..b4215f1f8 100644 --- a/bin/Lib/DataFrameABCCore.pas +++ b/bin/Lib/DataFrameABCCore.pas @@ -23,8 +23,9 @@ type IntArray = array of integer; RealArray = array of real; BoolArray = array of boolean; + DateTimeArray = array of System.DateTime; - ColumnType = (ctInt, ctFloat, ctStr, ctBool); + ColumnType = (ctInt, ctFloat, ctStr, ctBool, ctDateTime); /// Неизменяемое описание структуры столбцов DataFrame DataFrameSchema = sealed class @@ -111,6 +112,102 @@ type constructor Create(name: string; colType: ColumnType); //IsCategorical - только в Schema! end; + + /// Типизированное значение ячейки DataFrame. + /// Используется для доступа через row['Column']. + /// + /// Позволяет писать в фильтрах и вычислениях: + /// row['score'].IsValid + /// row['score'].Float + /// row['date'].DateTime + /// + /// Если значение отсутствует, IsValid = False, + /// а попытка взять Int/Float/Str/Bool/DateTime + /// приводит к исключению. + DataValue = sealed class + private + fName: string; + fColType: ColumnType; + fIsValid: boolean; + fInt: integer; + fFloat: real; + fStr: string; + fBool: boolean; + fDateTime: System.DateTime; + + class function IsNumericType(t: ColumnType): boolean; + class function Compare(a, b: DataValue): integer; + class function ArithmeticType(a, b: DataValue): ColumnType; + class function MakeNumeric(name: string; value: real; asInt: boolean := False): DataValue; + public + /// Имя столбца, из которого получено значение + property Name: string read fName; + /// Тип столбца, из которого получено значение + property ColType: ColumnType read fColType; + /// Показывает, что в ячейке есть корректное значение, а не NA + property IsValid: boolean read fIsValid; + + constructor Create(name: string; value: integer); + constructor Create(name: string; value: real); + constructor Create(name: string; value: string); + constructor Create(name: string; value: boolean); + constructor Create(name: string; value: System.DateTime); + class function NA(name: string; colType: ColumnType): DataValue; + + /// Возвращает значение как integer + function GetInt: integer; + /// Возвращает значение как real + function GetFloat: real; + /// Возвращает значение как string + function GetStr: string; + /// Возвращает значение как boolean + function GetBool: boolean; + /// Возвращает значение как DateTime + function GetDateTime: System.DateTime; + + /// Значение ячейки как integer + property Int: integer read GetInt; + /// Значение ячейки как real + property Float: real read GetFloat; + /// Значение ячейки как string + property Str: string read GetStr; + /// Значение ячейки как boolean + property Bool: boolean read GetBool; + /// Значение ячейки как DateTime + property DateTime: System.DateTime read GetDateTime; + + function ToInt: integer := GetInt; + function ToFloat: real := GetFloat; + function ToStr: string := GetStr; + function ToBool: boolean := GetBool; + function ToDateTime: System.DateTime := GetDateTime; + + function ToString: string; override; + + static function operator implicit(v: integer): DataValue := new DataValue('', v); + static function operator implicit(v: real): DataValue := new DataValue('', v); + static function operator implicit(v: string): DataValue := new DataValue('', v); + static function operator implicit(v: boolean): DataValue := new DataValue('', v); + static function operator implicit(v: System.DateTime): DataValue := new DataValue('', v); + + {static function operator implicit(v: DataValue): integer; + static function operator implicit(v: DataValue): real; + static function operator implicit(v: DataValue): string; + static function operator implicit(v: DataValue): boolean; + static function operator implicit(v: DataValue): System.DateTime;} + + static function operator =(a, b: DataValue): boolean; + static function operator <>(a, b: DataValue): boolean := not (a = b); + static function operator <(a, b: DataValue): boolean := Compare(a, b) < 0; + static function operator <=(a, b: DataValue): boolean := Compare(a, b) <= 0; + static function operator >(a, b: DataValue): boolean := Compare(a, b) > 0; + static function operator >=(a, b: DataValue): boolean := Compare(a, b) >= 0; + + static function operator +(a, b: DataValue): DataValue; + static function operator -(a, b: DataValue): DataValue; + static function operator *(a, b: DataValue): DataValue; + static function operator /(a, b: DataValue): DataValue; + end; DataFrameCursor = class; @@ -183,12 +280,26 @@ type /// Возвращает количество строк в столбце function RowCount: integer; override := Data.Length; end; + + /// Столбец значений DateTime + DateTimeColumn = class(Column) + Data: array of System.DateTime; + public + constructor Create; begin end; + constructor Create(name: string); + constructor Create(name: string; values: array of System.DateTime; + valid: array of boolean := nil); + function TryGetNumericValue(i: integer; var value: real): boolean; override; + /// Возвращает количество строк в столбце + function RowCount: integer; override := Data.Length; + end; // Accessor типы для курсора IntAccessor = function(pos: integer): integer; FloatAccessor = function(pos: integer): real; StrAccessor = function(pos: integer): string; BoolAccessor = function(pos: integer): boolean; + DateTimeAccessor = function(pos: integer): System.DateTime; ValidAccessor = function(pos: integer): boolean; // Структуры для Join @@ -218,7 +329,10 @@ type floatAcc: array of FloatAccessor; strAcc: array of StrAccessor; boolAcc: array of BoolAccessor; + dtAcc: array of DateTimeAccessor; validAcc: array of ValidAccessor; + function GetValue(i: integer): DataValue; + function GetValue(name: string): DataValue; public /// Создает курсор для указанных столбцов constructor Create(cols: array of Column; schema: DataFrameSchema); @@ -238,6 +352,8 @@ type function Str(i: integer): string; /// Возвращает булево значение из столбца по индексу function Bool(i: integer): boolean; + /// Возвращает значение DateTime из столбца по индексу + function DateTime(i: integer): System.DateTime; /// Возвращает целочисленное значение из столбца по имени function Int(name: string): integer; /// Возвращает вещественное значение из столбца по имени @@ -246,6 +362,17 @@ type function Str(name: string): string; /// Возвращает булево значение из столбца по имени function Bool(name: string): boolean; + /// Возвращает значение DateTime из столбца по имени + function DateTime(name: string): System.DateTime; + /// Возвращает типизированное значение ячейки по индексу. + /// Пример: row.Value(0).IsValid + function Value(i: integer): DataValue := GetValue(i); + /// Возвращает типизированное значение ячейки по имени столбца. + /// Это основной способ проверки и чтения значения: + /// row['score'].IsValid + /// row['score'].Float + /// row['name'].Str + property Item[name: string]: DataValue read GetValue; default; /// Проверяет валидность значения в столбце по индексу function IsValid(i: integer): boolean; /// Проверяет валидность значения в столбце по имени @@ -286,6 +413,8 @@ const 'Столбец не является Str!!Column is not Str'; ER_COLUMN_NOT_BOOL = 'Столбец не является Bool!!Column is not Bool'; + ER_COLUMN_NOT_DATETIME = + 'Столбец не является DateTime!!Column is not DateTime'; ER_VALUE_IS_NA = 'Значение в столбце "{0}" равно NA!!Value in column "{0}" is NA'; ER_DUPLICATE_COLUMN_NAME = @@ -318,7 +447,23 @@ const 'Индекс строки {0} вне диапазона [0..{1})!!' + 'Row index {0} out of range [0..{1})'; ER_INVALID_ISVALID_LENGTH = - 'Длина IsValid должна совпадать с длиной Data!!IsValid length must match Data length'; + 'Длина IsValid должна совпадать с длиной Data!!IsValid length must match Data length'; + ER_DATAVALUE_NOT_INT = + 'Значение "{0}" не является Int!!Value "{0}" is not Int'; + ER_DATAVALUE_NOT_FLOAT = + 'Значение "{0}" не является Float!!Value "{0}" is not Float'; + ER_DATAVALUE_NOT_STR = + 'Значение "{0}" не является Str!!Value "{0}" is not Str'; + ER_DATAVALUE_NOT_BOOL = + 'Значение "{0}" не является Bool!!Value "{0}" is not Bool'; + ER_DATAVALUE_NOT_DATETIME = + 'Значение "{0}" не является DateTime!!Value "{0}" is not DateTime'; + ER_DATAVALUE_COMPARE_TYPES_MISMATCH = + 'Нельзя сравнивать значения типов {0} и {1}!!Cannot compare values of types {0} and {1}'; + ER_DATAVALUE_ARITHMETIC_TYPES_MISMATCH = + 'Нельзя выполнять арифметические операции для типов {0} и {1}!!Cannot apply arithmetic to types {0} and {1}'; + ER_DATAVALUE_DIVISION_BY_ZERO = + 'Деление на ноль!!Division by zero'; //----------------------------- // Сервисные функции @@ -348,6 +493,25 @@ begin Error(ER_COLUMN_NOT_BOOL); end; +function NotDateTime(pos: integer): System.DateTime; +begin + Result := default(System.DateTime); + Error(ER_COLUMN_NOT_DATETIME); +end; + +function ColumnTypeName(t: ColumnType): string; +begin + case t of + ctInt: Result := 'Int'; + ctFloat: Result := 'Float'; + ctStr: Result := 'Str'; + ctBool: Result := 'Bool'; + ctDateTime: Result := 'DateTime'; + else + Result := t.ToString; + end; +end; + //----------------------------- // DataFrameSchema //----------------------------- @@ -409,6 +573,275 @@ begin fColType := colType; end; +class function DataValue.IsNumericType(t: ColumnType): boolean; +begin + Result := t in [ctInt, ctFloat]; +end; + +constructor DataValue.Create(name: string; value: integer); +begin + fName := name; + fColType := ctInt; + fIsValid := true; + fInt := value; + fFloat := value; +end; + +constructor DataValue.Create(name: string; value: real); +begin + fName := name; + fColType := ctFloat; + fIsValid := true; + fFloat := value; +end; + +constructor DataValue.Create(name: string; value: string); +begin + fName := name; + fColType := ctStr; + fIsValid := true; + fStr := value; +end; + +constructor DataValue.Create(name: string; value: boolean); +begin + fName := name; + fColType := ctBool; + fIsValid := true; + fBool := value; +end; + +constructor DataValue.Create(name: string; value: System.DateTime); +begin + fName := name; + fColType := ctDateTime; + fIsValid := true; + fDateTime := value; +end; + +class function DataValue.NA(name: string; colType: ColumnType): DataValue; +begin + Result := new DataValue(name, 0); + Result.fColType := colType; + Result.fIsValid := false; +end; + +function DataValue.GetInt: integer; +begin + if not fIsValid then + Error(ER_VALUE_IS_NA, fName); + if fColType <> ctInt then + Error(ER_DATAVALUE_NOT_INT, fName); + Result := fInt; +end; + +function DataValue.GetFloat: real; +begin + if not fIsValid then + Error(ER_VALUE_IS_NA, fName); + if not IsNumericType(fColType) then + Error(ER_DATAVALUE_NOT_FLOAT, fName); + if fColType = ctInt then + Result := fInt + else + Result := fFloat; +end; + +function DataValue.GetStr: string; +begin + if not fIsValid then + Error(ER_VALUE_IS_NA, fName); + if fColType <> ctStr then + Error(ER_DATAVALUE_NOT_STR, fName); + Result := fStr; +end; + +function DataValue.GetBool: boolean; +begin + if not fIsValid then + Error(ER_VALUE_IS_NA, fName); + if fColType <> ctBool then + Error(ER_DATAVALUE_NOT_BOOL, fName); + Result := fBool; +end; + +function DataValue.GetDateTime: System.DateTime; +begin + if not fIsValid then + Error(ER_VALUE_IS_NA, fName); + if fColType <> ctDateTime then + Error(ER_DATAVALUE_NOT_DATETIME, fName); + Result := fDateTime; +end; + +function DataValue.ToString: string; +begin + if not fIsValid then + exit('NA'); + + case fColType of + ctInt: Result := fInt.ToString; + ctFloat: Result := fFloat.ToString; + ctStr: Result := fStr; + ctBool: Result := fBool.ToString; + ctDateTime: Result := fDateTime.ToString('yyyy-MM-dd HH:mm:ss'); + else + Result := ''; + end; +end; + +class function DataValue.Compare(a, b: DataValue): integer; +begin + if System.Object.ReferenceEquals(a, nil) then + ArgumentNullError('a'); + if System.Object.ReferenceEquals(b, nil) then + ArgumentNullError('b'); + if not a.fIsValid then + Error(ER_VALUE_IS_NA, a.fName); + if not b.fIsValid then + Error(ER_VALUE_IS_NA, b.fName); + + if IsNumericType(a.fColType) and IsNumericType(b.fColType) then + begin + var av := a.Float; + var bv := b.Float; + if av < bv then + Result := -1 + else if av > bv then + Result := 1 + else + Result := 0; + exit; + end; + + if a.fColType <> b.fColType then + Error(ER_DATAVALUE_COMPARE_TYPES_MISMATCH, ColumnTypeName(a.fColType), ColumnTypeName(b.fColType)); + + case a.fColType of + ctStr: + if a.fStr < b.fStr then Result := -1 else if a.fStr > b.fStr then Result := 1 else Result := 0; + ctBool: + if integer(a.fBool) < integer(b.fBool) then Result := -1 else if integer(a.fBool) > integer(b.fBool) then Result := 1 else Result := 0; + ctDateTime: + if a.fDateTime < b.fDateTime then Result := -1 else if a.fDateTime > b.fDateTime then Result := 1 else Result := 0; + else + Result := 0; + end; +end; + +class function DataValue.ArithmeticType(a, b: DataValue): ColumnType; +begin + if System.Object.ReferenceEquals(a, nil) then + ArgumentNullError('a'); + if System.Object.ReferenceEquals(b, nil) then + ArgumentNullError('b'); + if not a.fIsValid then + Error(ER_VALUE_IS_NA, a.fName); + if not b.fIsValid then + Error(ER_VALUE_IS_NA, b.fName); + + if not (IsNumericType(a.fColType) and IsNumericType(b.fColType)) then + Error(ER_DATAVALUE_ARITHMETIC_TYPES_MISMATCH, ColumnTypeName(a.fColType), ColumnTypeName(b.fColType)); + + if (a.fColType = ctInt) and (b.fColType = ctInt) then + Result := ctInt + else + Result := ctFloat; +end; + +class function DataValue.MakeNumeric(name: string; value: real; asInt: boolean): DataValue; +begin + if asInt then + Result := new DataValue(name, integer(value)) + else + Result := new DataValue(name, value); +end; + +{static function DataValue.operator implicit(v: DataValue): integer; +begin + if System.Object.ReferenceEquals(v, nil) then + ArgumentNullError('v'); + Result := v.Int; +end; + +static function DataValue.operator implicit(v: DataValue): real; +begin + if System.Object.ReferenceEquals(v, nil) then + ArgumentNullError('v'); + Result := v.Float; +end; + +static function DataValue.operator implicit(v: DataValue): string; +begin + if System.Object.ReferenceEquals(v, nil) then + ArgumentNullError('v'); + Result := v.Str; +end; + +static function DataValue.operator implicit(v: DataValue): boolean; +begin + if System.Object.ReferenceEquals(v, nil) then + ArgumentNullError('v'); + Result := v.Bool; +end; + +static function DataValue.operator implicit(v: DataValue): System.DateTime; +begin + if System.Object.ReferenceEquals(v, nil) then + ArgumentNullError('v'); + Result := v.DateTime; +end;} + +static function DataValue.operator =(a, b: DataValue): boolean; +begin + if System.Object.ReferenceEquals(a, nil) then + exit(System.Object.ReferenceEquals(b, nil)); + if System.Object.ReferenceEquals(b, nil) then + exit(false); + Result := Compare(a, b) = 0; +end; + +static function DataValue.operator +(a, b: DataValue): DataValue; +begin + if System.Object.ReferenceEquals(a, nil) then + ArgumentNullError('a'); + if System.Object.ReferenceEquals(b, nil) then + ArgumentNullError('b'); + if not a.fIsValid then + Error(ER_VALUE_IS_NA, a.fName); + if not b.fIsValid then + Error(ER_VALUE_IS_NA, b.fName); + + if (a.fColType = ctStr) and (b.fColType = ctStr) then + begin + Result := new DataValue(a.Name, a.fStr + b.fStr); + exit; + end; + + var t := ArithmeticType(a, b); + Result := MakeNumeric(a.Name, a.Float + b.Float, t = ctInt); +end; + +static function DataValue.operator -(a, b: DataValue): DataValue; +begin + var t := ArithmeticType(a, b); + Result := MakeNumeric(a.Name, a.Float - b.Float, t = ctInt); +end; + +static function DataValue.operator *(a, b: DataValue): DataValue; +begin + var t := ArithmeticType(a, b); + Result := MakeNumeric(a.Name, a.Float * b.Float, t = ctInt); +end; + +static function DataValue.operator /(a, b: DataValue): DataValue; +begin + ArithmeticType(a, b); + if b.Float = 0 then + Error(ER_DATAVALUE_DIVISION_BY_ZERO); + Result := new DataValue(a.Name, a.Float / b.Float); +end; + procedure DataFrameSchema.Print; begin if fNames.Length = 0 then @@ -430,6 +863,7 @@ begin ctFloat: t := 'float'; ctStr: t := 'string'; ctBool: t := 'bool'; + ctDateTime: t := 'datetime'; end; if fCategoricalFlags[i] then @@ -766,6 +1200,39 @@ begin exit(True); end; +constructor DateTimeColumn.Create(name: string; values: array of System.DateTime; valid: array of boolean); +begin + inherited Create; + Info := new ColumnInfo(name, ctDateTime); + + var n := Length(values); + Self.Data := if n = 0 then [] else values; + + if valid = nil then + IsValid := [True] * n + else + begin + if Length(valid) <> n then + Error(ER_INVALID_ISVALID_LENGTH); + + IsValid := valid; + end; +end; + +constructor DateTimeColumn.Create(name: string); +begin + inherited Create; + Info := new ColumnInfo(name, ctDateTime); + + Data := new System.DateTime[0]; + IsValid := new boolean[0]; +end; + +function DateTimeColumn.TryGetNumericValue(i: integer; var value: real): boolean; +begin + exit(False); +end; + //----------------------------- @@ -830,6 +1297,7 @@ begin ctFloat: rowCnt := FloatColumn(cols[0]).Data.Length; ctStr: rowCnt := StrColumn(cols[0]).Data.Length; ctBool: rowCnt := BoolColumn(cols[0]).Data.Length; + ctDateTime: rowCnt := DateTimeColumn(cols[0]).Data.Length; end; var n := cols.Length; @@ -839,6 +1307,7 @@ begin floatAcc := new FloatAccessor[n]; strAcc := new StrAccessor[n]; boolAcc := new BoolAccessor[n]; + dtAcc := new DateTimeAccessor[n]; validAcc := new ValidAccessor[n]; for var i := 0 to n - 1 do @@ -849,6 +1318,7 @@ begin floatAcc[i] := NotFloat; strAcc[i] := NotStr; boolAcc[i] := NotBool; + dtAcc[i] := NotDateTime; case fSchema.ColumnTypeAt(i) of ctInt: @@ -883,6 +1353,14 @@ begin boolAcc[i] := pos -> c.Data[pos]; end; + + ctDateTime: + begin + var c := DateTimeColumn(col); + validAcc[i] := pos -> c.IsValid[pos]; + + dtAcc[i] := pos -> c.Data[pos]; + end; else Error(ER_UNKNOWN_COLUMN_TYPE); end; @@ -932,6 +1410,13 @@ begin Error(ER_VALUE_IS_NA, fSchema.NameAt(i)); Result := boolAcc[i](pos); end; + +function DataFrameCursor.DateTime(i: integer): System.DateTime; +begin + if not IsValid(i) then + Error(ER_VALUE_IS_NA, fSchema.NameAt(i)); + Result := dtAcc[i](pos); +end; function DataFrameCursor.Int(name: string): integer; begin @@ -952,6 +1437,34 @@ function DataFrameCursor.Bool(name: string): boolean; begin Result := Bool(fSchema.IndexOf(name)); end; + +function DataFrameCursor.DateTime(name: string): System.DateTime; +begin + Result := DateTime(fSchema.IndexOf(name)); +end; + +function DataFrameCursor.GetValue(i: integer): DataValue; +begin + var name := fSchema.NameAt(i); + + if not IsValid(i) then + exit(DataValue.NA(name, fSchema.ColumnTypeAt(i))); + + case fSchema.ColumnTypeAt(i) of + ctInt: Result := new DataValue(name, Int(i)); + ctFloat: Result := new DataValue(name, Float(i)); + ctStr: Result := new DataValue(name, Str(i)); + ctBool: Result := new DataValue(name, Bool(i)); + ctDateTime: Result := new DataValue(name, DateTime(i)); + else + Error(ER_UNKNOWN_COLUMN_TYPE); + end; +end; + +function DataFrameCursor.GetValue(name: string): DataValue; +begin + Result := GetValue(fSchema.IndexOf(name)); +end; procedure DataFrameCursor.MoveTo(p: integer); begin diff --git a/bin/Lib/MLABC.pas b/bin/Lib/MLABC.pas index 14f2666ee..ccadc29c8 100644 --- a/bin/Lib/MLABC.pas +++ b/bin/Lib/MLABC.pas @@ -106,8 +106,10 @@ type DataFrame = DataFrameABC.DataFrame; DataFrameCursor = DataFrameABCCore.DataFrameCursor; + DataValue = DataFrameABCCore.DataValue; ColumnType = DataFrameABCCore.ColumnType; Column = DataFrameABCCore.Column; + DatePartKind = DataFrameABC.DatePartKind; Statistics = DataFrameABC.Statistics; CsvLoader = DataFrameABC.CsvLoader; @@ -163,6 +165,10 @@ type IPreprocessor = PreprocessorABC.IPreprocessor; OrdinalEncoder = PreprocessorABC.OrdinalEncoder; OneHotEncoder = PreprocessorABC.OneHotEncoder; + DateTimeComponentsEncoder = PreprocessorABC.DateTimeComponentsEncoder; + DateTimeUnit = PreprocessorABC.DateTimeUnit; + DateTimeOrdinalEncoder = PreprocessorABC.DateTimeOrdinalEncoder; + DateTimeCyclicEncoder = PreprocessorABC.DateTimeCyclicEncoder; ImputeStrategy = PreprocessorABC.ImputeStrategy; Imputer = PreprocessorABC.Imputer; @@ -191,6 +197,21 @@ const jkLeft = JoinKind.jkLeft; jkRight = JoinKind.jkRight; jkFull = JoinKind.jkFull; + + dpYear = DatePartKind.dpYear; + dpMonth = DatePartKind.dpMonth; + dpDay = DatePartKind.dpDay; + dpHour = DatePartKind.dpHour; + dpMinute = DatePartKind.dpMinute; + dpSecond = DatePartKind.dpSecond; + dpDayOfWeek = DatePartKind.dpDayOfWeek; + dpTimeOfDay = DatePartKind.dpTimeOfDay; + dpDate = DatePartKind.dpDate; + + dtuDays = DateTimeUnit.dtuDays; + dtuHours = DateTimeUnit.dtuHours; + dtuMinutes = DateTimeUnit.dtuMinutes; + dtuSeconds = DateTimeUnit.dtuSeconds; /// Кодирует строковые метки классов в целочисленные индексы. /// Каждому уникальному значению присваивается номер 0,1,2,... diff --git a/bin/Lib/MLPipelineABC.pas b/bin/Lib/MLPipelineABC.pas index 11c30097f..dbfd92b2e 100644 --- a/bin/Lib/MLPipelineABC.pas +++ b/bin/Lib/MLPipelineABC.pas @@ -309,8 +309,8 @@ const ER_ENCODELABELS_NOT_CATEGORICAL = 'Целевой столбец должен быть категориальным для задач классификации!!Target column must be categorical for classification tasks'; ER_CLASSIFICATION_TARGET_MUST_BE_CATEGORICAL_STR_OR_INT = - 'Целевой столбец "{0}" должен быть категориальным строковым или целочисленным для задач классификации!!' + - 'Target column "{0}" must be a categorical string or integer column for classification tasks'; + 'Целевой столбец "{0}" должен быть строковым, целочисленным или логическим для задач классификации!!' + + 'Target column "{0}" must be a string, integer, or boolean column for classification tasks'; ER_REGRESSION_TARGET_MUST_BE_NUMERIC = 'Целевой столбец "{0}" должен быть числовым для задач регрессии!!Target column "{0}" must be numeric for regression tasks'; ER_PREPROCESSOR_ROWCOUNT_CHANGED = @@ -990,10 +990,8 @@ begin case fTask of tkClassification: begin - if not df.IsCategorical(fTarget) then - ArgumentError(ER_ENCODELABELS_NOT_CATEGORICAL, fTarget); var ct := df.GetColumnType(fTarget); - if not (ct in [ColumnType.ctStr, ColumnType.ctInt]) then + if not (ct in [ColumnType.ctStr, ColumnType.ctInt, ColumnType.ctBool]) then ArgumentError(ER_CLASSIFICATION_TARGET_MUST_BE_CATEGORICAL_STR_OR_INT, fTarget); end; diff --git a/bin/Lib/PreprocessorABC.pas b/bin/Lib/PreprocessorABC.pas index 5a5b633a5..82878842b 100644 --- a/bin/Lib/PreprocessorABC.pas +++ b/bin/Lib/PreprocessorABC.pas @@ -123,6 +123,92 @@ type function Clone: IPreprocessor; end; + DateTimeUnit = (dtuDays, dtuHours, dtuMinutes, dtuSeconds); + +/// Разворачивает DateTime-столбец в набор компонентных признаков. +/// По умолчанию создаёт признаки year, month, day. +/// Исходный DateTime-столбец заменяется новыми столбцами. + DateTimeComponentsEncoder = class(IPreprocessor, IColumnBoundStep, IColumnExpander) + private + col: string; + parts: array of DatePartKind; + expandedColumns: array of string; + fitted: boolean; + function DefaultParts: array of DatePartKind; + class function PartSuffix(part: DatePartKind): string; + public + /// Создаёт encoder со стандартным набором компонент: year, month, day. + constructor Create(column: string); + /// Создаёт encoder с явным списком компонент. + constructor Create(column: string; dateParts: array of DatePartKind); + + function Fit(df: DataFrame): IPreprocessor; + function Transform(df: DataFrame): DataFrame; + function FitTransform(df: DataFrame): DataFrame; + function ToString: string; override; + + property ColumnName: string read col; + + function GetExpandedColumns(sourceColumn: string): array of string; + function Clone: IPreprocessor; + end; + +/// Кодирует DateTime-столбец в один вещественный столбец, +/// содержащий расстояние от минимального значения столбца в выбранных единицах. +/// На Fit запоминается fOrigin = Min(column), на Transform вычисляется +/// (dt - fOrigin).TotalDays / TotalHours / TotalMinutes / TotalSeconds. +/// Исходный столбец заменяется вещественным столбцом с новым именем. + DateTimeOrdinalEncoder = class(IPreprocessor, IColumnBoundStep) + private + col: string; + newCol: string; + unit_: DateTimeUnit; + fOrigin: System.DateTime; + fitted: boolean; + class function DefaultColumnName(column: string; unit_: DateTimeUnit): string; + public + constructor Create(column: string); + constructor Create(column, newColumn: string); + constructor Create(column, newColumn: string; unit_: DateTimeUnit); + + function Fit(df: DataFrame): IPreprocessor; + function Transform(df: DataFrame): DataFrame; + function FitTransform(df: DataFrame): DataFrame; + function ToString: string; override; + + property ColumnName: string read col; + + function Clone: IPreprocessor; + end; + +/// Кодирует циклическую компоненту DateTime через пару sin/cos. +/// Например, для dpHour создаёт created_at_cyc_hour_sin и created_at_cyc_hour_cos. +/// Исходный DateTime-столбец заменяется двумя вещественными столбцами. + DateTimeCyclicEncoder = class(IPreprocessor, IColumnBoundStep, IColumnExpander) + private + col: string; + part: DatePartKind; + sinCol: string; + cosCol: string; + fitted: boolean; + class function PartSuffix(part: DatePartKind): string; + class function IsSupportedPart(part: DatePartKind): boolean; + class function PartPeriod(part: DatePartKind): integer; + class function PartValue(dt: System.DateTime; part: DatePartKind): real; + public + constructor Create(column: string; part: DatePartKind); + + function Fit(df: DataFrame): IPreprocessor; + function Transform(df: DataFrame): DataFrame; + function FitTransform(df: DataFrame): DataFrame; + function ToString: string; override; + + property ColumnName: string read col; + + function GetExpandedColumns(sourceColumn: string): array of string; + function Clone: IPreprocessor; + end; + ImputeStrategy = (isMean, isConstant, isMedian); /// Заполняет пропущенные значения (NA) в столбцах DataFrame @@ -228,6 +314,45 @@ const 'Конфликт имён колонок: {0}!!Column name collision: {0}'; ER_ONEHOT_DUPLICATE_COLUMN = 'Дублирующаяся сгенерированная колонка: {0}!!Duplicate generated column: {0}'; + ER_DTENC_NO_COLUMN = + 'DateTimeComponentsEncoder: столбец не указан!!DateTimeComponentsEncoder: column not specified'; + ER_DTENC_NOT_DATETIME = + 'DateTimeComponentsEncoder: столбец "{0}" не является DateTime!!' + + 'DateTimeComponentsEncoder: column "{0}" is not DateTime'; + ER_DTENC_EMPTY_PARTS = + 'DateTimeComponentsEncoder: список компонент пуст!!DateTimeComponentsEncoder: date parts list is empty'; + ER_DTENC_DUPLICATE_PART = + 'DateTimeComponentsEncoder: компонент даты повторяется: {0}!!DateTimeComponentsEncoder: duplicate date part: {0}'; + ER_DTENC_NAME_EQUALS_SOURCE = + 'DateTimeComponentsEncoder: сгенерированная колонка совпадает с исходной: {0}!!DateTimeComponentsEncoder: generated column equals source column: {0}'; + ER_DTENC_COLUMN_COLLISION = + 'DateTimeComponentsEncoder: конфликт имён колонок: {0}!!DateTimeComponentsEncoder: column name collision: {0}'; + ER_DTENC_DUPLICATE_COLUMN = + 'DateTimeComponentsEncoder: дублирующаяся сгенерированная колонка: {0}!!DateTimeComponentsEncoder: duplicate generated column: {0}'; + ER_DTORD_NO_COLUMN = + 'DateTimeOrdinalEncoder: столбец не указан!!DateTimeOrdinalEncoder: column not specified'; + ER_DTORD_NO_NEW_COLUMN = + 'DateTimeOrdinalEncoder: имя новой колонки не указано!!DateTimeOrdinalEncoder: new column name not specified'; + ER_DTORD_NOT_DATETIME = + 'DateTimeOrdinalEncoder: столбец "{0}" не является DateTime!!' + + 'DateTimeOrdinalEncoder: column "{0}" is not DateTime'; + ER_DTORD_NO_VALID_VALUES = + 'DateTimeOrdinalEncoder: столбец "{0}" не содержит допустимых значений!!' + + 'DateTimeOrdinalEncoder: column "{0}" has no valid values'; + ER_DTORD_COLUMN_COLLISION = + 'DateTimeOrdinalEncoder: конфликт имён колонок: {0}!!DateTimeOrdinalEncoder: column name collision: {0}'; + ER_DTCYC_NO_COLUMN = + 'DateTimeCyclicEncoder: столбец не указан!!DateTimeCyclicEncoder: column not specified'; + ER_DTCYC_NOT_DATETIME = + 'DateTimeCyclicEncoder: столбец "{0}" не является DateTime!!' + + 'DateTimeCyclicEncoder: column "{0}" is not DateTime'; + ER_DTCYC_UNSUPPORTED_PART = + 'DateTimeCyclicEncoder: часть даты {0} не поддерживается для циклического кодирования!!' + + 'DateTimeCyclicEncoder: date part {0} is not supported for cyclic encoding'; + ER_DTCYC_COLUMN_COLLISION = + 'DateTimeCyclicEncoder: конфликт имён колонок: {0}!!DateTimeCyclicEncoder: column name collision: {0}'; + ER_DTCYC_DUPLICATE_COLUMN = + 'DateTimeCyclicEncoder: дублирующаяся сгенерированная колонка: {0}!!DateTimeCyclicEncoder: duplicate generated column: {0}'; //----------------------------- @@ -532,6 +657,548 @@ begin Result := new OneHotEncoder(col); end; +//----------------------------- +// DateTimeComponentsEncoder +//----------------------------- + +function DateTimeComponentsEncoder.DefaultParts: array of DatePartKind; +begin + Result := [DatePartKind.dpYear, DatePartKind.dpMonth, DatePartKind.dpDay]; +end; + +class function DateTimeComponentsEncoder.PartSuffix(part: DatePartKind): string; +begin + case part of + dpYear: Result := 'year'; + dpMonth: Result := 'month'; + dpDay: Result := 'day'; + dpHour: Result := 'hour'; + dpMinute: Result := 'minute'; + dpSecond: Result := 'second'; + dpDayOfWeek: Result := 'dayofweek'; + dpDate: Result := 'date'; + end; +end; + +constructor DateTimeComponentsEncoder.Create(column: string); +begin + if column = '' then + ArgumentError(ER_DTENC_NO_COLUMN); + + col := column; + parts := DefaultParts; + fitted := false; +end; + +constructor DateTimeComponentsEncoder.Create(column: string; dateParts: array of DatePartKind); +begin + if column = '' then + ArgumentError(ER_DTENC_NO_COLUMN); + if (dateParts = nil) or (dateParts.Length = 0) then + ArgumentError(ER_DTENC_EMPTY_PARTS); + + col := column; + parts := Copy(dateParts); + fitted := false; +end; + +function DateTimeComponentsEncoder.Fit(df: DataFrame): IPreprocessor; +begin + if df = nil then + ArgumentNullError(ER_ARG_NULL, 'df'); + + var idx := df.Schema.IndexOf(col); + if idx < 0 then + Error(ER_COLUMN_NOT_FOUND, col); + + if df.Schema.ColumnTypeAt(idx) <> ColumnType.ctDateTime then + Error(ER_DTENC_NOT_DATETIME, col); + + var usedParts := new HashSet; + for var i := 0 to parts.Length - 1 do + if usedParts.Contains(parts[i]) then + Error(ER_DTENC_DUPLICATE_PART, parts[i]) + else + usedParts.Add(parts[i]); + + expandedColumns := new string[parts.Length]; + var usedNames := new HashSet; + + for var i := 0 to parts.Length - 1 do + begin + var newName := col + '_' + PartSuffix(parts[i]); + expandedColumns[i] := newName; + + if newName = col then + Error(ER_DTENC_NAME_EQUALS_SOURCE, newName); + + if df.HasColumn(newName) then + Error(ER_DTENC_COLUMN_COLLISION, newName); + + if usedNames.Contains(newName) then + Error(ER_DTENC_DUPLICATE_COLUMN, newName); + + usedNames.Add(newName); + end; + + fitted := true; + Result := Self; +end; + +function DateTimeComponentsEncoder.Transform(df: DataFrame): DataFrame; +begin + if not fitted then + NotFittedError(ER_FIT_NOT_CALLED); + + var srcIdx := df.Schema.IndexOf(col); + if srcIdx < 0 then + ArgumentError(ER_COLUMN_NOT_FOUND, col); + if df.Schema.ColumnTypeAt(srcIdx) <> ColumnType.ctDateTime then + Error(ER_DTENC_NOT_DATETIME, col); + + var rowCount := df.RowCount; + var srcCol := DateTimeColumn(df.GetColumn(srcIdx)); + var srcData := srcCol.Data; + var srcValid := srcCol.IsValid; + + var res := new DataFrame; + var names := new List; + var types := new List; + var cats := new List; + + AppendAllColumnsExcept(res, df, srcIdx, names, types, cats); + + for var j := 0 to parts.Length - 1 do + begin + var part := parts[j]; + var newName := expandedColumns[j]; + + if part = dpDate then + begin + var data := new System.DateTime[rowCount]; + var valid := new boolean[rowCount]; + + for var row := 0 to rowCount - 1 do + if (srcValid <> nil) and not srcValid[row] then + begin + data[row] := default(System.DateTime); + valid[row] := False; + end + else + begin + data[row] := srcData[row].Date; + valid[row] := True; + end; + + res.AddDateTimeColumn(newName, data, valid); + names.Add(newName); + types.Add(ColumnType.ctDateTime); + cats.Add(False); + end + else + begin + var data := new integer[rowCount]; + var valid := new boolean[rowCount]; + + for var row := 0 to rowCount - 1 do + if (srcValid <> nil) and not srcValid[row] then + begin + data[row] := 0; + valid[row] := False; + end + else + begin + case part of + dpYear: data[row] := srcData[row].Year; + dpMonth: data[row] := srcData[row].Month; + dpDay: data[row] := srcData[row].Day; + dpHour: data[row] := srcData[row].Hour; + dpMinute: data[row] := srcData[row].Minute; + dpSecond: data[row] := srcData[row].Second; + dpDayOfWeek: data[row] := integer(srcData[row].DayOfWeek); + end; + valid[row] := True; + end; + + res.AddIntColumn(newName, data, valid); + names.Add(newName); + types.Add(ColumnType.ctInt); + cats.Add(False); + end; + end; + + res.SetSchema(new DataFrameSchema( + names.ToArray, + types.ToArray, + cats.ToArray + )); + + Result := res; +end; + +function DateTimeComponentsEncoder.FitTransform(df: DataFrame): DataFrame; +begin + Fit(df); + Result := Transform(df); +end; + +function DateTimeComponentsEncoder.ToString: string; +begin + Result := 'DateTimeComponentsEncoder(column=' + col + ')'; +end; + +function DateTimeComponentsEncoder.GetExpandedColumns(sourceColumn: string): array of string; +begin + if not fitted then + NotFittedError(ER_FIT_NOT_CALLED); + + if sourceColumn <> col then + exit(nil); + + Result := Copy(expandedColumns); +end; + +function DateTimeComponentsEncoder.Clone: IPreprocessor; +begin + Result := new DateTimeComponentsEncoder(col, parts); +end; + +//----------------------------- +// DateTimeOrdinalEncoder +//----------------------------- + +class function DateTimeOrdinalEncoder.DefaultColumnName(column: string; unit_: DateTimeUnit): string; +begin + case unit_ of + dtuDays: Result := column + '_days'; + dtuHours: Result := column + '_hours'; + dtuMinutes: Result := column + '_minutes'; + dtuSeconds: Result := column + '_seconds'; + end; +end; + +constructor DateTimeOrdinalEncoder.Create(column: string); +begin + Create(column, DefaultColumnName(column, DateTimeUnit.dtuDays), DateTimeUnit.dtuDays); +end; + +constructor DateTimeOrdinalEncoder.Create(column, newColumn: string); +begin + Create(column, newColumn, DateTimeUnit.dtuDays); +end; + +constructor DateTimeOrdinalEncoder.Create(column, newColumn: string; unit_: DateTimeUnit); +begin + if column = '' then + ArgumentError(ER_DTORD_NO_COLUMN); + if newColumn = '' then + ArgumentError(ER_DTORD_NO_NEW_COLUMN); + + col := column; + newCol := newColumn; + self.unit_ := unit_; + fitted := false; +end; + +function DateTimeOrdinalEncoder.Fit(df: DataFrame): IPreprocessor; +begin + if df = nil then + ArgumentNullError(ER_ARG_NULL, 'df'); + + var idx := df.Schema.IndexOf(col); + if idx < 0 then + Error(ER_COLUMN_NOT_FOUND, col); + + if df.Schema.ColumnTypeAt(idx) <> ColumnType.ctDateTime then + Error(ER_DTORD_NOT_DATETIME, col); + + if (newCol <> col) and df.HasColumn(newCol) then + Error(ER_DTORD_COLUMN_COLLISION, newCol); + + var cur := df.GetCursor; + var hasValue := false; + var minValue := default(System.DateTime); + + while cur.MoveNext do + if cur.IsValid(idx) then + begin + var dt := cur.DateTime(idx); + if not hasValue or (dt < minValue) then + begin + minValue := dt; + hasValue := true; + end; + end; + + if not hasValue then + Error(ER_DTORD_NO_VALID_VALUES, col); + + fOrigin := minValue; + fitted := true; + Result := Self; +end; + +function DateTimeOrdinalEncoder.Transform(df: DataFrame): DataFrame; +begin + if not fitted then + NotFittedError(ER_FIT_NOT_CALLED); + + var idx := df.Schema.IndexOf(col); + if idx < 0 then + ArgumentError(ER_COLUMN_NOT_FOUND, col); + if df.Schema.ColumnTypeAt(idx) <> ColumnType.ctDateTime then + Error(ER_DTORD_NOT_DATETIME, col); + + var srcCol := DateTimeColumn(df.GetColumn(idx)); + var rowCount := df.RowCount; + var data := new real[rowCount]; + var valid := new boolean[rowCount]; + + for var row := 0 to rowCount - 1 do + if (srcCol.IsValid <> nil) and not srcCol.IsValid[row] then + begin + data[row] := 0.0; + valid[row] := False; + end + else + begin + var delta := srcCol.Data[row] - fOrigin; + case unit_ of + dtuDays: data[row] := delta.TotalDays; + dtuHours: data[row] := delta.TotalHours; + dtuMinutes: data[row] := delta.TotalMinutes; + dtuSeconds: data[row] := delta.TotalSeconds; + end; + valid[row] := True; + end; + + var res := new DataFrame; + var names := new List; + var types := new List; + var cats := new List; + + for var i := 0 to df.ColumnCount - 1 do + if i <> idx then + begin + res.AddColumnAlias(df.GetColumn(i)); + names.Add(df.Schema.NameAt(i)); + types.Add(df.Schema.ColumnTypeAt(i)); + cats.Add(df.Schema.IsCategoricalAt(i)); + end + else + begin + res.AddFloatColumn(newCol, data, valid); + names.Add(newCol); + types.Add(ColumnType.ctFloat); + cats.Add(False); + end; + + res.SetSchema(new DataFrameSchema( + names.ToArray, + types.ToArray, + cats.ToArray + )); + + Result := res; +end; + +function DateTimeOrdinalEncoder.FitTransform(df: DataFrame): DataFrame; +begin + Fit(df); + Result := Transform(df); +end; + +function DateTimeOrdinalEncoder.ToString: string; +begin + Result := 'DateTimeOrdinalEncoder(column=' + col + ', newColumn=' + newCol + ', unit=' + unit_.ToString + ')'; +end; + +function DateTimeOrdinalEncoder.Clone: IPreprocessor; +begin + Result := new DateTimeOrdinalEncoder(col, newCol, unit_); +end; + +//----------------------------- +// DateTimeCyclicEncoder +//----------------------------- + +class function DateTimeCyclicEncoder.PartSuffix(part: DatePartKind): string; +begin + case part of + dpMonth: Result := 'month'; + dpDayOfWeek: Result := 'dayofweek'; + dpHour: Result := 'hour'; + dpMinute: Result := 'minute'; + dpSecond: Result := 'second'; + dpTimeOfDay: Result := 'timeofday'; + end; +end; + +class function DateTimeCyclicEncoder.IsSupportedPart(part: DatePartKind): boolean; +begin + Result := part in [dpMonth, dpDayOfWeek, dpHour, dpMinute, dpSecond, dpTimeOfDay]; +end; + +class function DateTimeCyclicEncoder.PartPeriod(part: DatePartKind): integer; +begin + case part of + dpMonth: Result := 12; + dpDayOfWeek: Result := 7; + dpHour: Result := 24; + dpMinute: Result := 60; + dpSecond: Result := 60; + dpTimeOfDay: Result := 86400; + else + Error(ER_DTCYC_UNSUPPORTED_PART, part); + end; +end; + +class function DateTimeCyclicEncoder.PartValue(dt: System.DateTime; part: DatePartKind): real; +begin + case part of + dpMonth: Result := dt.Month - 1; + dpDayOfWeek: Result := integer(dt.DayOfWeek); + dpHour: Result := dt.Hour; + dpMinute: Result := dt.Minute; + dpSecond: Result := dt.Second; + dpTimeOfDay: + Result := dt.Hour * 3600.0 + dt.Minute * 60.0 + dt.Second + dt.Millisecond / 1000.0; + else + Error(ER_DTCYC_UNSUPPORTED_PART, part); + end; +end; + +constructor DateTimeCyclicEncoder.Create(column: string; part: DatePartKind); +begin + if column = '' then + ArgumentError(ER_DTCYC_NO_COLUMN); + if not IsSupportedPart(part) then + Error(ER_DTCYC_UNSUPPORTED_PART, part); + + col := column; + self.part := part; + var suffix := PartSuffix(part); + sinCol := col + '_cyc_' + suffix + '_sin'; + cosCol := col + '_cyc_' + suffix + '_cos'; + fitted := false; +end; + +function DateTimeCyclicEncoder.Fit(df: DataFrame): IPreprocessor; +begin + if df = nil then + ArgumentNullError(ER_ARG_NULL, 'df'); + + var idx := df.Schema.IndexOf(col); + if idx < 0 then + Error(ER_COLUMN_NOT_FOUND, col); + + if df.Schema.ColumnTypeAt(idx) <> ColumnType.ctDateTime then + Error(ER_DTCYC_NOT_DATETIME, col); + + if df.HasColumn(sinCol) then + Error(ER_DTCYC_COLUMN_COLLISION, sinCol); + if df.HasColumn(cosCol) then + Error(ER_DTCYC_COLUMN_COLLISION, cosCol); + if sinCol = cosCol then + Error(ER_DTCYC_DUPLICATE_COLUMN, sinCol); + + fitted := true; + Result := Self; +end; + +function DateTimeCyclicEncoder.Transform(df: DataFrame): DataFrame; +begin + if not fitted then + NotFittedError(ER_FIT_NOT_CALLED); + + var srcIdx := df.Schema.IndexOf(col); + if srcIdx < 0 then + ArgumentError(ER_COLUMN_NOT_FOUND, col); + if df.Schema.ColumnTypeAt(srcIdx) <> ColumnType.ctDateTime then + Error(ER_DTCYC_NOT_DATETIME, col); + + var rowCount := df.RowCount; + var srcCol := DateTimeColumn(df.GetColumn(srcIdx)); + var srcData := srcCol.Data; + var srcValid := srcCol.IsValid; + + var period := PartPeriod(part); + var sinData := new real[rowCount]; + var cosData := new real[rowCount]; + var sinValid := new boolean[rowCount]; + var cosValid := new boolean[rowCount]; + + for var row := 0 to rowCount - 1 do + if (srcValid <> nil) and not srcValid[row] then + begin + sinData[row] := 0.0; + cosData[row] := 0.0; + sinValid[row] := False; + cosValid[row] := False; + end + else + begin + var value := PartValue(srcData[row], part); + var angle := 2 * PI * value / period; + sinData[row] := Sin(angle); + cosData[row] := Cos(angle); + sinValid[row] := True; + cosValid[row] := True; + end; + + var res := new DataFrame; + var names := new List; + var types := new List; + var cats := new List; + + AppendAllColumnsExcept(res, df, srcIdx, names, types, cats); + + res.AddFloatColumn(sinCol, sinData, sinValid); + names.Add(sinCol); + types.Add(ColumnType.ctFloat); + cats.Add(False); + + res.AddFloatColumn(cosCol, cosData, cosValid); + names.Add(cosCol); + types.Add(ColumnType.ctFloat); + cats.Add(False); + + res.SetSchema(new DataFrameSchema( + names.ToArray, + types.ToArray, + cats.ToArray + )); + + Result := res; +end; + +function DateTimeCyclicEncoder.FitTransform(df: DataFrame): DataFrame; +begin + Fit(df); + Result := Transform(df); +end; + +function DateTimeCyclicEncoder.ToString: string; +begin + Result := 'DateTimeCyclicEncoder(column=' + col + ', part=' + part.ToString + ')'; +end; + +function DateTimeCyclicEncoder.GetExpandedColumns(sourceColumn: string): array of string; +begin + if not fitted then + NotFittedError(ER_FIT_NOT_CALLED); + + if sourceColumn <> col then + exit(nil); + + Result := [sinCol, cosCol]; +end; + +function DateTimeCyclicEncoder.Clone: IPreprocessor; +begin + Result := new DateTimeCyclicEncoder(col, part); +end; + //----------------------------- // Imputer //-----------------------------