Новые ML примеры и тесты - в основном в DataFrame
EncodeLabels - убрана проверка категориальности, добавлен тип Bool В DataFrame добавлен тип DateTime Добавлен тип DataValue позволяющий писать df.Filter(r -> r['Col'] > 5)
This commit is contained in:
Mikhalkovich Stanislav
parent
d940541c65
commit
828484f15a
|
|
@ -15,7 +15,7 @@ internal static class RevisionClass
|
|||
public const string Major = "3";
|
||||
public const string Minor = "11";
|
||||
public const string Build = "1";
|
||||
public const string Revision = "3830";
|
||||
public const string Revision = "3833";
|
||||
|
||||
public const string MainVersion = Major + "." + Minor;
|
||||
public const string FullVersion = Major + "." + Minor + "." + Build + "." + Revision;
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
%COREVERSION%=1
|
||||
%REVISION%=3830
|
||||
%MINOR%=11
|
||||
%REVISION%=3833
|
||||
%COREVERSION%=1
|
||||
%MAJOR%=3
|
||||
|
|
|
|||
|
|
@ -9,5 +9,5 @@ begin
|
|||
Charlie,21,78
|
||||
''');
|
||||
|
||||
df.Filter(row -> row.Int('age') >= 21).Print;
|
||||
df.Filter(row -> row['age'] >= 21).Print;
|
||||
end.
|
||||
31
InstallerSamples/MachineLearning/02_DataFrame/df23.pas
Normal file
31
InstallerSamples/MachineLearning/02_DataFrame/df23.pas
Normal file
|
|
@ -0,0 +1,31 @@
|
|||
uses MLABC;
|
||||
|
||||
begin
|
||||
var text := '''
|
||||
id,created_at,name
|
||||
1,2024-01-15,Alice
|
||||
2,2024-01-16 12:30:00,Bob
|
||||
''';
|
||||
|
||||
var schema := new Dictionary<string, ColumnType>;
|
||||
schema['created_at'] := ColumnType.ctDateTime;
|
||||
|
||||
var df1 := CsvLoader.LoadFromLines(
|
||||
text.ToLines,
|
||||
schema := schema
|
||||
);
|
||||
|
||||
Println('Явная схема:');
|
||||
df1.Print;
|
||||
Println(df1.DateTime('created_at')[0].Year);
|
||||
|
||||
var df2 := CsvLoader.LoadFromLines(
|
||||
text.ToLines,
|
||||
inferTypes := True
|
||||
);
|
||||
|
||||
Println;
|
||||
Println('Автоопределение:');
|
||||
df2.Print(dateTimeFormat := 'dd.MM.yy');
|
||||
Println(df2.GetColumnType('created_at'));
|
||||
end.
|
||||
20
InstallerSamples/MachineLearning/02_DataFrame/df24.pas
Normal file
20
InstallerSamples/MachineLearning/02_DataFrame/df24.pas
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
uses MLABC;
|
||||
|
||||
begin
|
||||
var text := '''
|
||||
id,created_at,name
|
||||
1,15.01.2024,Alice
|
||||
2,16.01.2024 12:30:00,Bob
|
||||
''';
|
||||
|
||||
var df := CsvLoader.LoadFromLines(
|
||||
text.ToLines,
|
||||
inferTypes := True
|
||||
);
|
||||
|
||||
df.Print;
|
||||
Println;
|
||||
Println(df.GetColumnType('created_at'));
|
||||
Println(df.DateTime('created_at')[0].Year);
|
||||
Println(df.DateTime('created_at')[1].Hour);
|
||||
end.
|
||||
23
InstallerSamples/MachineLearning/02_DataFrame/df25.pas
Normal file
23
InstallerSamples/MachineLearning/02_DataFrame/df25.pas
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
uses MLABC;
|
||||
|
||||
begin
|
||||
var text := '''
|
||||
id,created_at,name
|
||||
1,16.01.2024 12:30:00,Bob
|
||||
2,15.01.2024,Alice
|
||||
3,17.01.2024 09:15:00,Charlie
|
||||
''';
|
||||
|
||||
var df := CsvLoader.LoadFromLines(
|
||||
text.ToLines,
|
||||
inferTypes := True
|
||||
);
|
||||
|
||||
var sorted := df.SortBy('created_at');
|
||||
|
||||
Println('Исходный DataFrame:');
|
||||
df.Print;
|
||||
Println;
|
||||
Println('После SortBy(created_at):');
|
||||
sorted.Print;
|
||||
end.
|
||||
23
InstallerSamples/MachineLearning/02_DataFrame/df26.pas
Normal file
23
InstallerSamples/MachineLearning/02_DataFrame/df26.pas
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
uses MLABC;
|
||||
|
||||
begin
|
||||
var text := '''
|
||||
id,created_at,name
|
||||
1,15.01.2024,Alice
|
||||
2,16.01.2024 12:30:00,Bob
|
||||
3,17.01.2024 09:15:00,Charlie
|
||||
''';
|
||||
|
||||
var df := CsvLoader.LoadFromLines(
|
||||
text.ToLines,
|
||||
inferTypes := True
|
||||
);
|
||||
|
||||
var filtered := df.Filter(cur -> cur.DateTime('created_at') >= DateTime.Create(2024, 1, 16));
|
||||
|
||||
Println('Исходный DataFrame:');
|
||||
df.Print;
|
||||
Println;
|
||||
Println('Дата >= 16.01.2024:');
|
||||
filtered.Print;
|
||||
end.
|
||||
24
InstallerSamples/MachineLearning/02_DataFrame/df27.pas
Normal file
24
InstallerSamples/MachineLearning/02_DataFrame/df27.pas
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
uses MLABC;
|
||||
|
||||
begin
|
||||
var text := '''
|
||||
id,created_at,name
|
||||
1,15.01.2024,Alice
|
||||
2,16.01.2024 12:30:00,Bob
|
||||
3,15.01.2024,Charlie
|
||||
4,16.01.2024 12:30:00,Diana
|
||||
''';
|
||||
|
||||
var df := CsvLoader.LoadFromLines(
|
||||
text.ToLines,
|
||||
inferTypes := True
|
||||
);
|
||||
|
||||
var grouped := df.GroupBy('created_at').Count;
|
||||
|
||||
Println('Исходный DataFrame:');
|
||||
df.Print;
|
||||
Println;
|
||||
Println('GroupBy(created_at).Count:');
|
||||
grouped.Print;
|
||||
end.
|
||||
29
InstallerSamples/MachineLearning/02_DataFrame/df28.pas
Normal file
29
InstallerSamples/MachineLearning/02_DataFrame/df28.pas
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
uses MLABC;
|
||||
|
||||
begin
|
||||
var text := '''
|
||||
id,created_at,name
|
||||
1,15.01.2023,Alice
|
||||
2,16.01.2024 12:30:00,Bob
|
||||
3,20.02.2023,Charlie
|
||||
4,05.03.2024 09:15:00,Diana
|
||||
''';
|
||||
|
||||
var df := CsvLoader.LoadFromLines(
|
||||
text.ToLines,
|
||||
inferTypes := True
|
||||
);
|
||||
|
||||
var df2 := df.WithColumnInt(
|
||||
'year',
|
||||
cur -> cur.DateTime('created_at').Year
|
||||
);
|
||||
|
||||
var grouped := df2.GroupBy('year').Count;
|
||||
|
||||
Println('DataFrame с колонкой year:');
|
||||
df2.Print;
|
||||
Println;
|
||||
Println('GroupBy(year).Count:');
|
||||
grouped.Print;
|
||||
end.
|
||||
24
InstallerSamples/MachineLearning/02_DataFrame/df29.pas
Normal file
24
InstallerSamples/MachineLearning/02_DataFrame/df29.pas
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
uses MLABC, DataFrameABC;
|
||||
|
||||
begin
|
||||
var text := '''
|
||||
id,created_at,name
|
||||
1,15.01.2023 10:20:30,Alice
|
||||
2,16.02.2024 12:30:00,Bob
|
||||
3,05.03.2024 09:15:45,Charlie
|
||||
''';
|
||||
|
||||
var df := CsvLoader.LoadFromLines(
|
||||
text.ToLines,
|
||||
inferTypes := True
|
||||
);
|
||||
|
||||
var df2 := df.WithDatePart('created_at', 'year', dpYear);
|
||||
var df3 := df2.WithDateParts('created_at', [
|
||||
('month', dpMonth),
|
||||
('day', dpDay),
|
||||
('date_only', dpDate)
|
||||
]);
|
||||
|
||||
df3.Print;
|
||||
end.
|
||||
15
InstallerSamples/MachineLearning/02_DataFrame/df30.pas
Normal file
15
InstallerSamples/MachineLearning/02_DataFrame/df30.pas
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
uses MLABC;
|
||||
|
||||
begin
|
||||
var df := DataFrame.FromCsvText('''
|
||||
product,count,pack_price
|
||||
milk,2,120
|
||||
bread,1,55
|
||||
milk,3,118
|
||||
tea,1,340
|
||||
''');
|
||||
|
||||
var total := df.Sum(r -> r['count'] * r['pack_price']);
|
||||
|
||||
Println('Общая стоимость упаковок:', total:0:2);
|
||||
end.
|
||||
18
InstallerSamples/MachineLearning/02_DataFrame/df31.pas
Normal file
18
InstallerSamples/MachineLearning/02_DataFrame/df31.pas
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
uses MLABC;
|
||||
|
||||
begin
|
||||
var df := DataFrame.FromCsvText('''
|
||||
name,district,operation,amount
|
||||
Alice,North,sale,120
|
||||
Bob,North,refund,30
|
||||
Clara,South,sale,210
|
||||
Dmitry,North,sale,180
|
||||
''');
|
||||
|
||||
var saleCount := df.Count(r ->
|
||||
(r['district'] = 'North') and
|
||||
(r['operation'] = 'sale')
|
||||
);
|
||||
|
||||
Println('Продаж в районе North:', saleCount);
|
||||
end.
|
||||
18
InstallerSamples/MachineLearning/02_DataFrame/df32.pas
Normal file
18
InstallerSamples/MachineLearning/02_DataFrame/df32.pas
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
uses MLABC;
|
||||
|
||||
begin
|
||||
var df := DataFrame.FromCsvText('''
|
||||
product,count,price
|
||||
milk,2,120
|
||||
bread,1,55
|
||||
milk,3,118
|
||||
tea,1,340
|
||||
coffee,2,260
|
||||
''');
|
||||
|
||||
Println('Средний чек по строке:', df.Mean(r -> r['count'] * r['price']):0:2);
|
||||
Println('Медианный чек по строке:', df.Median(r -> r['count'] * r['price']):0:2);
|
||||
Println('Минимальный чек по строке:', df.Min(r -> r['count'] * r['price']):0:2);
|
||||
Println('Максимальный чек по строке:', df.Max(r -> r['count'] * r['price']):0:2);
|
||||
Println('Стандартное отклонение:', df.Std(r -> r['count'] * r['price']):0:2);
|
||||
end.
|
||||
23
InstallerSamples/MachineLearning/02_DataFrame/df33.pas
Normal file
23
InstallerSamples/MachineLearning/02_DataFrame/df33.pas
Normal file
|
|
@ -0,0 +1,23 @@
|
|||
uses MLABC;
|
||||
|
||||
begin
|
||||
var df := DataFrame.FromCsvText('''
|
||||
Город
|
||||
Москва
|
||||
Казань
|
||||
Москва
|
||||
Омск
|
||||
Казань
|
||||
Москва
|
||||
''');
|
||||
|
||||
var values := df.Unique('Город');
|
||||
Println('Уникальные города:');
|
||||
foreach var v in values do
|
||||
Println(v.Str);
|
||||
|
||||
Println;
|
||||
Println('Число различных городов: ', df.NUnique('Город'));
|
||||
Println;
|
||||
df.ValueCounts('Город').Print;
|
||||
end.
|
||||
26
InstallerSamples/MachineLearning/02_DataFrame/df34.pas
Normal file
26
InstallerSamples/MachineLearning/02_DataFrame/df34.pas
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
uses MLABC;
|
||||
|
||||
begin
|
||||
var df := DataFrame.FromCsvText('''
|
||||
name,age,income
|
||||
Alice,20,100
|
||||
Bob,,120
|
||||
Clara,25,
|
||||
Dmitry,30,150
|
||||
''');
|
||||
|
||||
Println('Пропусков в столбце age:', df.MissingCount('age'));
|
||||
Println('Пропусков в столбце income:', df.MissingCount('income'));
|
||||
Println;
|
||||
|
||||
Println('Пропуски по столбцам:');
|
||||
df.MissingCounts.Print;
|
||||
Println;
|
||||
|
||||
Println('Строки без пропусков во всех столбцах:');
|
||||
df.DropMissing.Print;
|
||||
Println;
|
||||
|
||||
Println('Строки без пропусков в age и income:');
|
||||
df.DropMissing(['age', 'income']).Print;
|
||||
end.
|
||||
18
InstallerSamples/MachineLearning/02_DataFrame/df35.pas
Normal file
18
InstallerSamples/MachineLearning/02_DataFrame/df35.pas
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
uses MLABC;
|
||||
|
||||
begin
|
||||
var df := DataFrame.FromCsvText('''
|
||||
name,age,score
|
||||
Alice,10,1.5
|
||||
Bob,25,4.8
|
||||
Alice,10,1.5
|
||||
Clara,30,8.2
|
||||
''');
|
||||
|
||||
Println('Без полных дубликатов:');
|
||||
df.DropDuplicates.Print;
|
||||
Println;
|
||||
|
||||
Println('После Clip(score, 0, 5):');
|
||||
df.Clip('score', 0, 5).Print;
|
||||
end.
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
uses MLABC;
|
||||
|
||||
begin
|
||||
var df := DataFrame.FromCsvText('''
|
||||
id,created_at,name
|
||||
1,15.01.2024 10:20:30,Alice
|
||||
2,16.02.2024 12:30:00,Bob
|
||||
3,17.03.2025 09:15:45,Charlie
|
||||
''');
|
||||
|
||||
var enc1 := new DateTimeComponentsEncoder('created_at');
|
||||
var df1 := enc1.FitTransform(df);
|
||||
|
||||
Println('Стандартные компоненты:');
|
||||
df1.Print;
|
||||
Println;
|
||||
|
||||
var enc2 := new DateTimeComponentsEncoder('created_at', [dpYear, dpMonth, dpDay, dpHour]);
|
||||
var df2 := enc2.FitTransform(df);
|
||||
|
||||
Println('Пользовательский набор компонент:');
|
||||
df2.Print;
|
||||
end.
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
uses MLABC;
|
||||
|
||||
begin
|
||||
var df := DataFrame.FromCsvText('''
|
||||
id,created_at,name
|
||||
1,15.01.2024 00:00:00,Alice
|
||||
2,16.01.2024 12:00:00,Bob
|
||||
3,18.01.2024 06:00:00,Charlie
|
||||
''');
|
||||
|
||||
var enc1 := new DateTimeOrdinalEncoder('created_at');
|
||||
var df1 := enc1.FitTransform(df);
|
||||
|
||||
Println('Смещение в днях от минимальной даты:');
|
||||
df1.Print;
|
||||
Println;
|
||||
|
||||
var enc2 := new DateTimeOrdinalEncoder('created_at', 'created_at_hours', dtuHours);
|
||||
var df2 := enc2.FitTransform(df);
|
||||
|
||||
Println('Смещение в часах от минимальной даты:');
|
||||
df2.Print;
|
||||
end.
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
uses MLABC;
|
||||
|
||||
begin
|
||||
var df := DataFrame.FromCsvText('''
|
||||
id,created_at,name
|
||||
1,15.01.2024 00:00:00,Alice
|
||||
2,15.01.2024 06:00:00,Bob
|
||||
3,15.01.2024 12:00:00,Charlie
|
||||
4,15.01.2024 18:00:00,Diana
|
||||
5,15.01.2024 23:56:00,Eva
|
||||
''');
|
||||
|
||||
var enc := new DateTimeCyclicEncoder('created_at', dpTimeOfDay);
|
||||
var df2 := enc.FitTransform(df);
|
||||
|
||||
df2.Print;
|
||||
end.
|
||||
|
|
@ -1 +1 @@
|
|||
3.11.1.3830
|
||||
3.11.1.3833
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
!define VERSION '3.11.1.3830'
|
||||
!define VERSION '3.11.1.3833'
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ uses TestHelpers in '..\TestHelpers.pas';
|
|||
begin
|
||||
var df := new DataFrame;
|
||||
df.AddStrColumn('Target', Arr('cat', 'dog', 'cat', 'fox'));
|
||||
df := df.SetCategorical(['Target']);
|
||||
|
||||
var classes: array of string;
|
||||
var y := df.EncodeLabels('Target', classes);
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ uses TestHelpers in '..\TestHelpers.pas';
|
|||
begin
|
||||
var df := new DataFrame;
|
||||
df.AddIntColumn('Target', Arr(10, 20, 10, 30));
|
||||
df := df.SetCategorical(['Target']);
|
||||
|
||||
var classes: array of string;
|
||||
var y := df.EncodeLabels('Target', classes);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,59 @@
|
|||
uses MLABC;
|
||||
uses TestHelpers in '..\TestHelpers.pas';
|
||||
|
||||
begin
|
||||
var textIso := '''
|
||||
id,created_at,name
|
||||
1,2024-01-15,Alice
|
||||
2,2024-01-16T12:30:00,Bob
|
||||
''';
|
||||
|
||||
var textRu := '''
|
||||
id,created_at,name
|
||||
1,15.01.2024,Alice
|
||||
2,16.01.2024 12:30:00,Bob
|
||||
''';
|
||||
|
||||
var schema := new Dictionary<string, ColumnType>;
|
||||
schema['created_at'] := ColumnType.ctDateTime;
|
||||
|
||||
var dfSchema := CsvLoader.LoadFromLines(
|
||||
textIso.ToLines,
|
||||
schema := schema
|
||||
);
|
||||
|
||||
Check(dfSchema.GetColumnType('created_at') = ColumnType.ctDateTime, 'Schema DateTime type expected');
|
||||
Check(dfSchema.DateTime('created_at')[0] = new System.DateTime(2024, 1, 15), 'Schema DateTime value mismatch');
|
||||
|
||||
var dfInfer := CsvLoader.LoadFromLines(
|
||||
textIso.ToLines,
|
||||
inferTypes := true
|
||||
);
|
||||
|
||||
Check(dfInfer.GetColumnType('created_at') = ColumnType.ctDateTime, 'Inferred DateTime type expected');
|
||||
Check(dfInfer.DateTime('created_at')[1] = new System.DateTime(2024, 1, 16, 12, 30, 0), 'Inferred DateTime value mismatch');
|
||||
|
||||
var dfRu := CsvLoader.LoadFromLines(
|
||||
textRu.ToLines,
|
||||
inferTypes := true
|
||||
);
|
||||
|
||||
Check(dfRu.GetColumnType('created_at') = ColumnType.ctDateTime, 'Russian DateTime inference expected');
|
||||
Check(dfRu.DateTime('created_at')[0] = new System.DateTime(2024, 1, 15), 'Russian DateTime date mismatch');
|
||||
Check(dfRu.DateTime('created_at')[1] = new System.DateTime(2024, 1, 16, 12, 30, 0), 'Russian DateTime value mismatch');
|
||||
|
||||
var tmp := System.IO.Path.Combine(
|
||||
System.IO.Path.GetTempPath,
|
||||
'ml_datetime_' + System.Guid.NewGuid.ToString + '.csv'
|
||||
);
|
||||
|
||||
dfInfer.ToCsv(tmp);
|
||||
var dfRoundTrip := CsvLoader.Load(tmp);
|
||||
|
||||
Check(dfRoundTrip.GetColumnType('created_at') = ColumnType.ctDateTime, 'Round-trip DateTime type expected');
|
||||
Check(dfRoundTrip.DateTime('created_at')[1] = dfInfer.DateTime('created_at')[1], 'Round-trip DateTime value mismatch');
|
||||
CheckSchemaMatchesColumns(dfRoundTrip);
|
||||
|
||||
if System.IO.File.Exists(tmp) then
|
||||
System.IO.File.Delete(tmp);
|
||||
end.
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
uses MLABC;
|
||||
uses TestHelpers in '..\TestHelpers.pas';
|
||||
|
||||
begin
|
||||
var df := new DataFrame;
|
||||
df.AddDateTimeColumn('created_at', Arr(
|
||||
new System.DateTime(2024, 1, 15),
|
||||
new System.DateTime(2024, 1, 16, 12, 30, 0)
|
||||
));
|
||||
|
||||
var oldOut := System.Console.Out;
|
||||
var sw := new System.IO.StringWriter;
|
||||
System.Console.SetOut(sw);
|
||||
try
|
||||
df.Print;
|
||||
finally
|
||||
System.Console.SetOut(oldOut);
|
||||
end;
|
||||
|
||||
var s := sw.ToString;
|
||||
Check(s.Contains('2024-01-15'), 'Default Print must show date-only value');
|
||||
Check(s.Contains('2024-01-16 12:30:00'), 'Default Print must show date and time');
|
||||
Check(not s.Contains('2024-01-15 00:00:00'), 'Default Print must omit zero time');
|
||||
|
||||
sw := new System.IO.StringWriter;
|
||||
System.Console.SetOut(sw);
|
||||
try
|
||||
df.Print(dateTimeFormat := 'yyyy/MM/dd');
|
||||
finally
|
||||
System.Console.SetOut(oldOut);
|
||||
end;
|
||||
|
||||
s := sw.ToString;
|
||||
Check(s.Contains('2024/01/15'), 'Custom Print format must be applied');
|
||||
Check(s.Contains('2024/01/16'), 'Custom Print format must be applied to all DateTime values');
|
||||
end.
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
uses MLABC;
|
||||
uses TestHelpers in '..\TestHelpers.pas';
|
||||
|
||||
begin
|
||||
var df := new DataFrame;
|
||||
df.AddIntColumn('Id', Arr(1, 2, 3));
|
||||
df.AddDateTimeColumn('CreatedAt', Arr(
|
||||
new System.DateTime(2024, 1, 16, 12, 30, 0),
|
||||
new System.DateTime(2024, 1, 15),
|
||||
new System.DateTime(2024, 1, 17, 9, 15, 0)
|
||||
));
|
||||
|
||||
var res := df.SortBy('CreatedAt');
|
||||
|
||||
Check(res.RowCount = 3, 'SortBy(DateTime) row count mismatch');
|
||||
Check(res.Int('Id')[0] = 2, 'First sorted row mismatch');
|
||||
Check(res.Int('Id')[1] = 1, 'Second sorted row mismatch');
|
||||
Check(res.Int('Id')[2] = 3, 'Third sorted row mismatch');
|
||||
Check(res.DateTime('CreatedAt')[0] = new System.DateTime(2024, 1, 15), 'Sorted DateTime value mismatch');
|
||||
CheckSchemaMatchesColumns(res);
|
||||
end.
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
uses MLABC;
|
||||
uses TestHelpers in '..\TestHelpers.pas';
|
||||
|
||||
begin
|
||||
var df := new DataFrame;
|
||||
df.AddIntColumn('Id', Arr(1, 2, 3));
|
||||
df.AddDateTimeColumn('CreatedAt', Arr(
|
||||
new System.DateTime(2024, 1, 15),
|
||||
new System.DateTime(2024, 1, 16, 12, 30, 0),
|
||||
new System.DateTime(2024, 1, 17, 9, 15, 0)
|
||||
));
|
||||
df.AddStrColumn('Name', Arr('Alice', 'Bob', 'Charlie'));
|
||||
|
||||
var res := df.Filter(cur -> cur.DateTime('CreatedAt') >= new System.DateTime(2024, 1, 16));
|
||||
|
||||
Check(res.RowCount = 2, 'Filter(DateTime) row count mismatch');
|
||||
Check(res.Int('Id')[0] = 2, 'First filtered row mismatch');
|
||||
Check(res.Int('Id')[1] = 3, 'Second filtered row mismatch');
|
||||
Check(res.DateTime('CreatedAt')[0] = new System.DateTime(2024, 1, 16, 12, 30, 0), 'First filtered DateTime mismatch');
|
||||
CheckSchemaMatchesColumns(res);
|
||||
end.
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
uses MLABC, DataFrameABCCore;
|
||||
uses TestHelpers in '..\TestHelpers.pas';
|
||||
|
||||
begin
|
||||
var df := new DataFrame;
|
||||
df.AddIntColumn('Id', Arr(1, 2, 3, 4));
|
||||
df.AddDateTimeColumn('CreatedAt', Arr(
|
||||
new System.DateTime(2024, 1, 15),
|
||||
new System.DateTime(2024, 1, 16, 12, 30, 0),
|
||||
new System.DateTime(2024, 1, 15),
|
||||
new System.DateTime(2024, 1, 16, 12, 30, 0)
|
||||
));
|
||||
|
||||
var g := df.GroupBy('CreatedAt').Count;
|
||||
|
||||
Check(g.Schema.ColumnCount = 2, 'Unexpected GroupBy(DateTime) column count');
|
||||
Check(g.Schema.NameAt(0) = 'CreatedAt', 'DateTime key column name mismatch');
|
||||
Check(g.Schema.ColumnTypeAt(0) = ColumnType.ctDateTime, 'DateTime key column type must stay DateTime');
|
||||
Check(g.GetColumn(0).Info.Name = 'CreatedAt', 'Physical DateTime key column name mismatch');
|
||||
Check(g.GetColumn(0).Info.ColType = ColumnType.ctDateTime, 'Physical DateTime key column type must stay DateTime');
|
||||
Check(g.RowCount = 2, 'GroupBy(DateTime) distinct key count mismatch');
|
||||
Check(g.DateTime('CreatedAt')[0] = new System.DateTime(2024, 1, 15), 'First grouped DateTime key mismatch');
|
||||
Check(g.GetIntColumn('count')[0] = 2, 'First grouped count mismatch');
|
||||
Check(g.GetIntColumn('count')[1] = 2, 'Second grouped count mismatch');
|
||||
end.
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
uses MLABC, DataFrameABC, DataFrameABCCore;
|
||||
uses TestHelpers in '..\TestHelpers.pas';
|
||||
|
||||
begin
|
||||
var df := new DataFrame;
|
||||
df.AddIntColumn('Id', Arr(1, 2));
|
||||
df.AddDateTimeColumn('CreatedAt', Arr(
|
||||
new System.DateTime(2024, 1, 15, 10, 20, 30),
|
||||
new System.DateTime(2025, 2, 16, 12, 30, 40)
|
||||
));
|
||||
|
||||
var resYear := df.WithDatePart('CreatedAt', 'Year', dpYear);
|
||||
Check(resYear.GetColumnType('Year') = ColumnType.ctInt, 'Year part type mismatch');
|
||||
Check(resYear.Int('Year')[0] = 2024, 'First year mismatch');
|
||||
Check(resYear.Int('Year')[1] = 2025, 'Second year mismatch');
|
||||
|
||||
var resDate := df.WithDatePart('CreatedAt', 'OnlyDate', dpDate);
|
||||
Check(resDate.GetColumnType('OnlyDate') = ColumnType.ctDateTime, 'Date part type mismatch');
|
||||
Check(resDate.DateTime('OnlyDate')[0] = new System.DateTime(2024, 1, 15), 'First date part mismatch');
|
||||
Check(resDate.DateTime('OnlyDate')[1] = new System.DateTime(2025, 2, 16), 'Second date part mismatch');
|
||||
CheckSchemaMatchesColumns(resDate);
|
||||
end.
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
uses MLABC, DataFrameABC, DataFrameABCCore;
|
||||
uses TestHelpers in '..\TestHelpers.pas';
|
||||
|
||||
begin
|
||||
var df := new DataFrame;
|
||||
df.AddIntColumn('Id', Arr(1, 2));
|
||||
df.AddDateTimeColumn('CreatedAt', Arr(
|
||||
new System.DateTime(2024, 1, 15, 10, 20, 30),
|
||||
new System.DateTime(2025, 2, 16, 12, 30, 40)
|
||||
));
|
||||
|
||||
var res := df.WithDateParts('CreatedAt', [
|
||||
('Year', dpYear),
|
||||
('Month', dpMonth),
|
||||
('OnlyDate', dpDate)
|
||||
]);
|
||||
|
||||
Check(res.ColumnCount = 5, 'WithDateParts column count mismatch');
|
||||
Check(res.GetColumnType('Year') = ColumnType.ctInt, 'Year type mismatch');
|
||||
Check(res.GetColumnType('Month') = ColumnType.ctInt, 'Month type mismatch');
|
||||
Check(res.GetColumnType('OnlyDate') = ColumnType.ctDateTime, 'OnlyDate type mismatch');
|
||||
Check(res.Int('Year')[0] = 2024, 'Year[0] mismatch');
|
||||
Check(res.Int('Month')[1] = 2, 'Month[1] mismatch');
|
||||
Check(res.DateTime('OnlyDate')[0] = new System.DateTime(2024, 1, 15), 'OnlyDate[0] mismatch');
|
||||
CheckSchemaMatchesColumns(res);
|
||||
end.
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
uses MLABC, DataFrameABC;
|
||||
uses TestHelpers in '..\TestHelpers.pas';
|
||||
|
||||
begin
|
||||
var df := CsvLoader.LoadFromLines('''
|
||||
id,created_at,name
|
||||
1,15.01.2024 10:20:30,Alice
|
||||
2,16.02.2025 12:30:40,Bob
|
||||
'''.ToLines, inferTypes := True);
|
||||
|
||||
var enc := new DateTimeComponentsEncoder('created_at');
|
||||
var res := enc.FitTransform(df);
|
||||
|
||||
Check(not res.HasColumn('created_at'), 'Source DateTime column must be replaced');
|
||||
Check(res.HasColumn('created_at_year'), 'Year column missing');
|
||||
Check(res.HasColumn('created_at_month'), 'Month column missing');
|
||||
Check(res.HasColumn('created_at_day'), 'Day column missing');
|
||||
Check(res.GetColumnType('created_at_year') = ColumnType.ctInt, 'Year column type mismatch');
|
||||
Check(res.Int('created_at_year')[0] = 2024, 'First year mismatch');
|
||||
Check(res.Int('created_at_month')[1] = 2, 'Second month mismatch');
|
||||
Check(res.Int('created_at_day')[1] = 16, 'Second day mismatch');
|
||||
CheckSchemaMatchesColumns(res);
|
||||
end.
|
||||
|
|
@ -0,0 +1,23 @@
|
|||
uses MLABC, DataFrameABC;
|
||||
uses TestHelpers in '..\TestHelpers.pas';
|
||||
|
||||
begin
|
||||
var df := CsvLoader.LoadFromLines('''
|
||||
id,created_at,name
|
||||
1,15.01.2024 10:20:30,Alice
|
||||
2,16.02.2025 12:30:40,Bob
|
||||
'''.ToLines, inferTypes := True);
|
||||
|
||||
var enc := new DateTimeComponentsEncoder('created_at', [dpDate, dpHour, dpDayOfWeek]);
|
||||
var res := enc.FitTransform(df);
|
||||
|
||||
Check(not res.HasColumn('created_at'), 'Source DateTime column must be replaced');
|
||||
Check(res.HasColumn('created_at_date'), 'Date component column missing');
|
||||
Check(res.HasColumn('created_at_hour'), 'Hour component column missing');
|
||||
Check(res.HasColumn('created_at_dayofweek'), 'DayOfWeek component column missing');
|
||||
Check(res.GetColumnType('created_at_date') = ColumnType.ctDateTime, 'Date component type mismatch');
|
||||
Check(res.GetColumnType('created_at_hour') = ColumnType.ctInt, 'Hour component type mismatch');
|
||||
Check(res.DateTime('created_at_date')[0] = new System.DateTime(2024, 1, 15), 'Date component value mismatch');
|
||||
Check(res.Int('created_at_hour')[1] = 12, 'Hour component value mismatch');
|
||||
CheckSchemaMatchesColumns(res);
|
||||
end.
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
uses MLABC;
|
||||
uses TestHelpers in '..\TestHelpers.pas';
|
||||
|
||||
begin
|
||||
var df := CsvLoader.LoadFromLines('''
|
||||
id,created_at,name
|
||||
1,15.01.2024 00:00:00,Alice
|
||||
2,16.01.2024 12:00:00,Bob
|
||||
3,18.01.2024 06:00:00,Charlie
|
||||
'''.ToLines, inferTypes := True);
|
||||
|
||||
var encDays := new DateTimeOrdinalEncoder('created_at');
|
||||
var daysDf := encDays.FitTransform(df);
|
||||
|
||||
Check(not daysDf.HasColumn('created_at'), 'Source DateTime column must be replaced');
|
||||
Check(daysDf.GetColumnType('created_at_days') = ColumnType.ctFloat, 'Ordinal DateTime column must become float');
|
||||
Check(Abs(daysDf.Float('created_at_days')[0] - 0.0) < 1e-9, 'First day offset mismatch');
|
||||
Check(Abs(daysDf.Float('created_at_days')[1] - 1.5) < 1e-9, 'Second day offset mismatch');
|
||||
Check(Abs(daysDf.Float('created_at_days')[2] - 3.25) < 1e-9, 'Third day offset mismatch');
|
||||
|
||||
var encHours := new DateTimeOrdinalEncoder('created_at', 'created_at_hours', dtuHours);
|
||||
var hoursDf := encHours.FitTransform(df);
|
||||
|
||||
Check(hoursDf.GetColumnType('created_at_hours') = ColumnType.ctFloat, 'Hour ordinal column type mismatch');
|
||||
Check(Abs(hoursDf.Float('created_at_hours')[0] - 0.0) < 1e-9, 'First hour offset mismatch');
|
||||
Check(Abs(hoursDf.Float('created_at_hours')[1] - 36.0) < 1e-9, 'Second hour offset mismatch');
|
||||
Check(Abs(hoursDf.Float('created_at_hours')[2] - 78.0) < 1e-9, 'Third hour offset mismatch');
|
||||
CheckSchemaMatchesColumns(hoursDf);
|
||||
end.
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
uses MLABC;
|
||||
uses TestHelpers in '..\TestHelpers.pas';
|
||||
|
||||
begin
|
||||
var df := CsvLoader.LoadFromLines('''
|
||||
id,created_at,name
|
||||
1,15.01.2024 00:00:00,Alice
|
||||
2,15.01.2024 06:00:00,Bob
|
||||
3,15.01.2024 12:00:00,Charlie
|
||||
4,15.01.2024 18:00:00,Diana
|
||||
5,15.01.2024 23:59:59,Eva
|
||||
'''.ToLines, inferTypes := True);
|
||||
|
||||
var enc := new DateTimeCyclicEncoder('created_at', dpTimeOfDay);
|
||||
var res := enc.FitTransform(df);
|
||||
|
||||
Check(not res.HasColumn('created_at'), 'Source DateTime column must be replaced');
|
||||
Check(res.HasColumn('created_at_cyc_timeofday_sin'), 'TimeOfDay sin column missing');
|
||||
Check(res.HasColumn('created_at_cyc_timeofday_cos'), 'TimeOfDay cos column missing');
|
||||
Check(res.GetColumnType('created_at_cyc_timeofday_sin') = ColumnType.ctFloat, 'TimeOfDay sin type mismatch');
|
||||
Check(res.GetColumnType('created_at_cyc_timeofday_cos') = ColumnType.ctFloat, 'TimeOfDay cos type mismatch');
|
||||
|
||||
Check(Abs(res.Float('created_at_cyc_timeofday_sin')[0] - 0.0) < 1e-9, '00:00 sin mismatch');
|
||||
Check(Abs(res.Float('created_at_cyc_timeofday_cos')[0] - 1.0) < 1e-9, '00:00 cos mismatch');
|
||||
|
||||
Check(Abs(res.Float('created_at_cyc_timeofday_sin')[1] - 1.0) < 1e-9, '06:00 sin mismatch');
|
||||
Check(Abs(res.Float('created_at_cyc_timeofday_cos')[1] - 0.0) < 1e-9, '06:00 cos mismatch');
|
||||
|
||||
Check(Abs(res.Float('created_at_cyc_timeofday_sin')[2] - 0.0) < 1e-9, '12:00 sin mismatch');
|
||||
Check(Abs(res.Float('created_at_cyc_timeofday_cos')[2] + 1.0) < 1e-9, '12:00 cos mismatch');
|
||||
|
||||
Check(Abs(res.Float('created_at_cyc_timeofday_sin')[3] + 1.0) < 1e-9, '18:00 sin mismatch');
|
||||
Check(Abs(res.Float('created_at_cyc_timeofday_cos')[3] - 0.0) < 1e-9, '18:00 cos mismatch');
|
||||
|
||||
Check(Abs(res.Float('created_at_cyc_timeofday_sin')[4]) < 1e-3, '23:59:59 sin must be close to 0');
|
||||
Check(Abs(res.Float('created_at_cyc_timeofday_cos')[4] - 1.0) < 1e-6, '23:59:59 cos must be close to 1');
|
||||
CheckSchemaMatchesColumns(res);
|
||||
end.
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
uses MLABC, DataFrameABC, DataFrameABCCore;
|
||||
uses TestHelpers in '..\TestHelpers.pas';
|
||||
|
||||
begin
|
||||
var df := DataFrame.FromCsvText('''
|
||||
city
|
||||
Moscow
|
||||
Kazan
|
||||
Moscow
|
||||
Omsk
|
||||
Kazan
|
||||
Moscow
|
||||
''');
|
||||
|
||||
var values := df.Unique('city');
|
||||
Check(values.Length = 3, 'Unique length mismatch');
|
||||
Check(values[0].Str = 'Moscow', 'Unique first value mismatch');
|
||||
Check(values[1].Str = 'Kazan', 'Unique second value mismatch');
|
||||
Check(values[2].Str = 'Omsk', 'Unique third value mismatch');
|
||||
|
||||
Check(df.NUnique('city') = 3, 'NUnique mismatch');
|
||||
|
||||
var vc := df.ValueCounts('city');
|
||||
Check(vc.RowCount = 3, 'ValueCounts row count mismatch');
|
||||
Check(vc.Str('city')[0] = 'Moscow', 'ValueCounts first key mismatch');
|
||||
Check(vc.Int('Count')[0] = 3, 'ValueCounts first count mismatch');
|
||||
Check(vc.Str('city')[1] = 'Kazan', 'ValueCounts second key mismatch');
|
||||
Check(vc.Int('Count')[1] = 2, 'ValueCounts second count mismatch');
|
||||
Check(vc.Str('city')[2] = 'Omsk', 'ValueCounts third key mismatch');
|
||||
Check(vc.Int('Count')[2] = 1, 'ValueCounts third count mismatch');
|
||||
CheckSchemaMatchesColumns(vc);
|
||||
end.
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
uses MLABC, DataFrameABC, DataFrameABCCore;
|
||||
uses TestHelpers in '..\TestHelpers.pas';
|
||||
|
||||
begin
|
||||
var df := DataFrame.FromCsvText('''
|
||||
name,age,income
|
||||
Alice,20,100
|
||||
Bob,,120
|
||||
Clara,25,
|
||||
Dmitry,30,150
|
||||
''');
|
||||
|
||||
Check(df.MissingCount('age') = 1, 'MissingCount(age) mismatch');
|
||||
Check(df.MissingCount('income') = 1, 'MissingCount(income) mismatch');
|
||||
|
||||
var mc := df.MissingCounts;
|
||||
Check(mc.RowCount = 3, 'MissingCounts row count mismatch');
|
||||
Check(mc.Str('Column')[0] = 'name', 'MissingCounts first column mismatch');
|
||||
Check(mc.Int('MissingCount')[0] = 0, 'MissingCounts first count mismatch');
|
||||
Check(mc.Str('Column')[1] = 'age', 'MissingCounts second column mismatch');
|
||||
Check(mc.Int('MissingCount')[1] = 1, 'MissingCounts second count mismatch');
|
||||
Check(mc.Str('Column')[2] = 'income', 'MissingCounts third column mismatch');
|
||||
Check(mc.Int('MissingCount')[2] = 1, 'MissingCounts third count mismatch');
|
||||
|
||||
var dropped := df.DropMissing(['age', 'income']);
|
||||
Check(dropped.RowCount = 2, 'DropMissing selected row count mismatch');
|
||||
Check(dropped.Str('name')[0] = 'Alice', 'DropMissing first row mismatch');
|
||||
Check(dropped.Str('name')[1] = 'Dmitry', 'DropMissing second row mismatch');
|
||||
CheckSchemaMatchesColumns(dropped);
|
||||
end.
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
uses MLABC, DataFrameABC, DataFrameABCCore;
|
||||
uses TestHelpers in '..\TestHelpers.pas';
|
||||
|
||||
begin
|
||||
var df := DataFrame.FromCsvText('''
|
||||
name,age,score
|
||||
Alice,10,1.5
|
||||
Bob,25,4.8
|
||||
Alice,10,1.5
|
||||
Clara,30,8.2
|
||||
''');
|
||||
|
||||
var dedup := df.DropDuplicates;
|
||||
Check(dedup.RowCount = 3, 'DropDuplicates row count mismatch');
|
||||
Check(dedup.Str('name')[0] = 'Alice', 'DropDuplicates first row mismatch');
|
||||
Check(dedup.Str('name')[1] = 'Bob', 'DropDuplicates second row mismatch');
|
||||
Check(dedup.Str('name')[2] = 'Clara', 'DropDuplicates third row mismatch');
|
||||
CheckSchemaMatchesColumns(dedup);
|
||||
|
||||
var clipped := df.Clip('score', 0, 5);
|
||||
Check(clipped.RowCount = 4, 'Clip row count mismatch');
|
||||
Check(Abs(clipped.Float('score')[0] - 1.5) < 1e-9, 'Clip first value mismatch');
|
||||
Check(Abs(clipped.Float('score')[1] - 4.8) < 1e-9, 'Clip second value mismatch');
|
||||
Check(Abs(clipped.Float('score')[3] - 5.0) < 1e-9, 'Clip clipped value mismatch');
|
||||
CheckSchemaMatchesColumns(clipped);
|
||||
end.
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
uses MLABC, DataFrameABC, DataFrameABCCore;
|
||||
uses TestHelpers in '..\TestHelpers.pas';
|
||||
|
||||
begin
|
||||
var df := DataFrame.FromCsvText('''
|
||||
name,age,income
|
||||
Alice,20,100
|
||||
Bob,,120
|
||||
Clara,25,
|
||||
Dmitry,30,150
|
||||
''');
|
||||
|
||||
var dropped := df.DropMissing;
|
||||
Check(dropped.RowCount = 2, 'DropMissing all row count mismatch');
|
||||
Check(dropped.Str('name')[0] = 'Alice', 'DropMissing all first row mismatch');
|
||||
Check(dropped.Str('name')[1] = 'Dmitry', 'DropMissing all second row mismatch');
|
||||
CheckSchemaMatchesColumns(dropped);
|
||||
end.
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
uses MLABC, DataFrameABC, DataFrameABCCore;
|
||||
uses TestHelpers in '..\TestHelpers.pas';
|
||||
|
||||
begin
|
||||
var df := DataFrame.FromCsvText('''
|
||||
city
|
||||
Moscow
|
||||
Kazan
|
||||
Moscow
|
||||
Kazan
|
||||
Omsk
|
||||
''');
|
||||
|
||||
var vc := df.ValueCounts('city');
|
||||
Check(vc.RowCount = 3, 'ValueCounts tie row count mismatch');
|
||||
Check(vc.Str('city')[0] = 'Moscow', 'ValueCounts tie first key mismatch');
|
||||
Check(vc.Int('Count')[0] = 2, 'ValueCounts tie first count mismatch');
|
||||
Check(vc.Str('city')[1] = 'Kazan', 'ValueCounts tie second key mismatch');
|
||||
Check(vc.Int('Count')[1] = 2, 'ValueCounts tie second count mismatch');
|
||||
Check(vc.Str('city')[2] = 'Omsk', 'ValueCounts tie third key mismatch');
|
||||
Check(vc.Int('Count')[2] = 1, 'ValueCounts tie third count mismatch');
|
||||
|
||||
var df2 := DataFrame.FromCsvText('''
|
||||
Count
|
||||
1
|
||||
2
|
||||
1
|
||||
''');
|
||||
|
||||
var vc2 := df2.ValueCounts('Count');
|
||||
Check(vc2.HasColumn('Count'), 'ValueCounts must keep source column name');
|
||||
Check(vc2.HasColumn('Frequency'), 'ValueCounts must rename counter column on conflict');
|
||||
Check(vc2.Int('Count')[0] = 1, 'ValueCounts conflict first key mismatch');
|
||||
Check(vc2.Int('Frequency')[0] = 2, 'ValueCounts conflict first frequency mismatch');
|
||||
CheckSchemaMatchesColumns(vc2);
|
||||
end.
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
uses MLABC, DataFrameABC, DataFrameABCCore;
|
||||
uses TestHelpers in '..\TestHelpers.pas';
|
||||
|
||||
begin
|
||||
var df := DataFrame.FromCsvText('''
|
||||
age
|
||||
-5
|
||||
10
|
||||
30
|
||||
''');
|
||||
|
||||
var clipped := df.Clip('age', 0, 20);
|
||||
Check(clipped.RowCount = 3, 'Clip int row count mismatch');
|
||||
Check(clipped.Int('age')[0] = 0, 'Clip int first value mismatch');
|
||||
Check(clipped.Int('age')[1] = 10, 'Clip int second value mismatch');
|
||||
Check(clipped.Int('age')[2] = 20, 'Clip int third value mismatch');
|
||||
CheckSchemaMatchesColumns(clipped);
|
||||
|
||||
CheckRaises(procedure -> begin
|
||||
var tmp := df.Clip('age', 0.5, 20);
|
||||
end, 'Clip must reject non-integer bounds for int column');
|
||||
|
||||
CheckRaises(procedure -> begin
|
||||
var tmp := df.Clip('age', 10, 0);
|
||||
end, 'Clip must reject lower > upper');
|
||||
end.
|
||||
|
|
@ -12,7 +12,14 @@ begin
|
|||
new LogisticRegression
|
||||
);
|
||||
|
||||
CheckRaises(procedure -> begin pipe.Fit(df); end,
|
||||
'Classification pipeline must require categorical target');
|
||||
pipe.Fit(df);
|
||||
|
||||
var pred := pipe.Predict(df);
|
||||
var labels := pipe.PredictLabels(df);
|
||||
var classes := pipe.GetClassLabels;
|
||||
|
||||
Check(pred.Length = df.RowCount, 'Pipeline prediction length mismatch');
|
||||
Check(labels.Length = df.RowCount, 'Pipeline label prediction length mismatch');
|
||||
Check(classes.Length = 2, 'Pipeline class count mismatch');
|
||||
end.
|
||||
|
||||
|
|
|
|||
|
|
@ -134,9 +134,6 @@ begin
|
|||
if not Self.HasColumn(target) then
|
||||
ArgumentError(ER_COLUMN_NOT_FOUND, target);
|
||||
|
||||
if not Self.IsCategorical(target) then
|
||||
ArgumentError(ER_ENCODELABELS_NOT_CATEGORICAL, target);
|
||||
|
||||
var col := Self[target];
|
||||
for var i := 0 to Self.RowCount - 1 do
|
||||
if not col.IsValid[i] then
|
||||
|
|
@ -157,6 +154,13 @@ begin
|
|||
exit;
|
||||
end;
|
||||
|
||||
if Self.GetColumnType(target) = ColumnType.ctBool then
|
||||
begin
|
||||
var labels := Self.GetBoolColumn(target).ToArray.Select(x -> x.ToString).ToArray;
|
||||
Result := EncodeLabels(labels);
|
||||
exit;
|
||||
end;
|
||||
|
||||
ArgumentError(ER_ENCODELABELS_UNSUPPORTED_TYPE, target);
|
||||
end;
|
||||
|
||||
|
|
@ -166,7 +170,7 @@ end;
|
|||
/// Каждому уникальному значению присваивается номер 0,1,2,...
|
||||
/// Порядок кодирования соответствует порядку первого появления значений в столбце.
|
||||
/// В параметр classes возвращается массив уникальных значений в порядке кодирования.
|
||||
/// Работает только для категориальных столбцов типов string и integer.
|
||||
/// Работает для столбцов типов string, integer и boolean.
|
||||
/// Для целочисленных категориальных столбцов значения преобразуются в строки.
|
||||
/// Используется при подготовке данных для задач классификации и визуализации.
|
||||
function EncodeLabels(Self: DataFrame; target: string; var classes: array of string): array of integer; extensionmethod;
|
||||
|
|
@ -180,9 +184,6 @@ begin
|
|||
if not Self.HasColumn(target) then
|
||||
ArgumentError(ER_COLUMN_NOT_FOUND, target);
|
||||
|
||||
if not Self.IsCategorical(target) then
|
||||
ArgumentError(ER_ENCODELABELS_NOT_CATEGORICAL, target);
|
||||
|
||||
var col := Self[target];
|
||||
for var i := 0 to Self.RowCount - 1 do
|
||||
if not col.IsValid[i] then
|
||||
|
|
@ -204,6 +205,11 @@ begin
|
|||
// если API требует string classes:
|
||||
classes := intClasses.Select(x -> x.ToString).ToArray;
|
||||
end;
|
||||
ColumnType.ctBool:
|
||||
begin
|
||||
var labels := Self.GetBoolColumn(target).ToArray.Select(x -> x.ToString).ToArray;
|
||||
Result := EncodeLabels(labels, classes);
|
||||
end;
|
||||
else
|
||||
ArgumentError(ER_ENCODELABELS_UNSUPPORTED_TYPE, target);
|
||||
end;
|
||||
|
|
@ -228,9 +234,6 @@ begin
|
|||
if not Self.HasColumn(target) then
|
||||
ArgumentError(ER_COLUMN_NOT_FOUND, target);
|
||||
|
||||
if not Self.IsCategorical(target) then
|
||||
ArgumentError(ER_ENCODELABELS_NOT_CATEGORICAL, target);
|
||||
|
||||
var col := Self[target];
|
||||
for var i := 0 to Self.RowCount - 1 do
|
||||
if not col.IsValid[i] then
|
||||
|
|
@ -247,6 +250,16 @@ begin
|
|||
var data := Self.GetIntColumn(target).ToArray;
|
||||
var strData := new string[data.Length];
|
||||
|
||||
for var i := 0 to data.Length - 1 do
|
||||
strData[i] := data[i].ToString;
|
||||
|
||||
Result := TransformLabels(strData, classes);
|
||||
end;
|
||||
ColumnType.ctBool:
|
||||
begin
|
||||
var data := Self.GetBoolColumn(target).ToArray;
|
||||
var strData := new string[data.Length];
|
||||
|
||||
for var i := 0 to data.Length - 1 do
|
||||
strData[i] := data[i].ToString;
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -23,8 +23,9 @@ type
|
|||
IntArray = array of integer;
|
||||
RealArray = array of real;
|
||||
BoolArray = array of boolean;
|
||||
DateTimeArray = array of System.DateTime;
|
||||
|
||||
ColumnType = (ctInt, ctFloat, ctStr, ctBool);
|
||||
ColumnType = (ctInt, ctFloat, ctStr, ctBool, ctDateTime);
|
||||
|
||||
/// Неизменяемое описание структуры столбцов DataFrame
|
||||
DataFrameSchema = sealed class
|
||||
|
|
@ -111,6 +112,102 @@ type
|
|||
constructor Create(name: string; colType: ColumnType);
|
||||
//IsCategorical - только в Schema!
|
||||
end;
|
||||
|
||||
/// Типизированное значение ячейки DataFrame.
|
||||
/// Используется для доступа через row['Column'].
|
||||
///
|
||||
/// Позволяет писать в фильтрах и вычислениях:
|
||||
/// row['score'].IsValid
|
||||
/// row['score'].Float
|
||||
/// row['date'].DateTime
|
||||
///
|
||||
/// Если значение отсутствует, IsValid = False,
|
||||
/// а попытка взять Int/Float/Str/Bool/DateTime
|
||||
/// приводит к исключению.
|
||||
DataValue = sealed class
|
||||
private
|
||||
fName: string;
|
||||
fColType: ColumnType;
|
||||
fIsValid: boolean;
|
||||
fInt: integer;
|
||||
fFloat: real;
|
||||
fStr: string;
|
||||
fBool: boolean;
|
||||
fDateTime: System.DateTime;
|
||||
|
||||
class function IsNumericType(t: ColumnType): boolean;
|
||||
class function Compare(a, b: DataValue): integer;
|
||||
class function ArithmeticType(a, b: DataValue): ColumnType;
|
||||
class function MakeNumeric(name: string; value: real; asInt: boolean := False): DataValue;
|
||||
public
|
||||
/// Имя столбца, из которого получено значение
|
||||
property Name: string read fName;
|
||||
/// Тип столбца, из которого получено значение
|
||||
property ColType: ColumnType read fColType;
|
||||
/// Показывает, что в ячейке есть корректное значение, а не NA
|
||||
property IsValid: boolean read fIsValid;
|
||||
|
||||
constructor Create(name: string; value: integer);
|
||||
constructor Create(name: string; value: real);
|
||||
constructor Create(name: string; value: string);
|
||||
constructor Create(name: string; value: boolean);
|
||||
constructor Create(name: string; value: System.DateTime);
|
||||
class function NA(name: string; colType: ColumnType): DataValue;
|
||||
|
||||
/// Возвращает значение как integer
|
||||
function GetInt: integer;
|
||||
/// Возвращает значение как real
|
||||
function GetFloat: real;
|
||||
/// Возвращает значение как string
|
||||
function GetStr: string;
|
||||
/// Возвращает значение как boolean
|
||||
function GetBool: boolean;
|
||||
/// Возвращает значение как DateTime
|
||||
function GetDateTime: System.DateTime;
|
||||
|
||||
/// Значение ячейки как integer
|
||||
property Int: integer read GetInt;
|
||||
/// Значение ячейки как real
|
||||
property Float: real read GetFloat;
|
||||
/// Значение ячейки как string
|
||||
property Str: string read GetStr;
|
||||
/// Значение ячейки как boolean
|
||||
property Bool: boolean read GetBool;
|
||||
/// Значение ячейки как DateTime
|
||||
property DateTime: System.DateTime read GetDateTime;
|
||||
|
||||
function ToInt: integer := GetInt;
|
||||
function ToFloat: real := GetFloat;
|
||||
function ToStr: string := GetStr;
|
||||
function ToBool: boolean := GetBool;
|
||||
function ToDateTime: System.DateTime := GetDateTime;
|
||||
|
||||
function ToString: string; override;
|
||||
|
||||
static function operator implicit(v: integer): DataValue := new DataValue('', v);
|
||||
static function operator implicit(v: real): DataValue := new DataValue('', v);
|
||||
static function operator implicit(v: string): DataValue := new DataValue('', v);
|
||||
static function operator implicit(v: boolean): DataValue := new DataValue('', v);
|
||||
static function operator implicit(v: System.DateTime): DataValue := new DataValue('', v);
|
||||
|
||||
{static function operator implicit(v: DataValue): integer;
|
||||
static function operator implicit(v: DataValue): real;
|
||||
static function operator implicit(v: DataValue): string;
|
||||
static function operator implicit(v: DataValue): boolean;
|
||||
static function operator implicit(v: DataValue): System.DateTime;}
|
||||
|
||||
static function operator =(a, b: DataValue): boolean;
|
||||
static function operator <>(a, b: DataValue): boolean := not (a = b);
|
||||
static function operator <(a, b: DataValue): boolean := Compare(a, b) < 0;
|
||||
static function operator <=(a, b: DataValue): boolean := Compare(a, b) <= 0;
|
||||
static function operator >(a, b: DataValue): boolean := Compare(a, b) > 0;
|
||||
static function operator >=(a, b: DataValue): boolean := Compare(a, b) >= 0;
|
||||
|
||||
static function operator +(a, b: DataValue): DataValue;
|
||||
static function operator -(a, b: DataValue): DataValue;
|
||||
static function operator *(a, b: DataValue): DataValue;
|
||||
static function operator /(a, b: DataValue): DataValue;
|
||||
end;
|
||||
|
||||
DataFrameCursor = class;
|
||||
|
||||
|
|
@ -183,12 +280,26 @@ type
|
|||
/// Возвращает количество строк в столбце
|
||||
function RowCount: integer; override := Data.Length;
|
||||
end;
|
||||
|
||||
/// Столбец значений DateTime
|
||||
DateTimeColumn = class(Column)
|
||||
Data: array of System.DateTime;
|
||||
public
|
||||
constructor Create; begin end;
|
||||
constructor Create(name: string);
|
||||
constructor Create(name: string; values: array of System.DateTime;
|
||||
valid: array of boolean := nil);
|
||||
function TryGetNumericValue(i: integer; var value: real): boolean; override;
|
||||
/// Возвращает количество строк в столбце
|
||||
function RowCount: integer; override := Data.Length;
|
||||
end;
|
||||
|
||||
// Accessor типы для курсора
|
||||
IntAccessor = function(pos: integer): integer;
|
||||
FloatAccessor = function(pos: integer): real;
|
||||
StrAccessor = function(pos: integer): string;
|
||||
BoolAccessor = function(pos: integer): boolean;
|
||||
DateTimeAccessor = function(pos: integer): System.DateTime;
|
||||
ValidAccessor = function(pos: integer): boolean;
|
||||
|
||||
// Структуры для Join
|
||||
|
|
@ -218,7 +329,10 @@ type
|
|||
floatAcc: array of FloatAccessor;
|
||||
strAcc: array of StrAccessor;
|
||||
boolAcc: array of BoolAccessor;
|
||||
dtAcc: array of DateTimeAccessor;
|
||||
validAcc: array of ValidAccessor;
|
||||
function GetValue(i: integer): DataValue;
|
||||
function GetValue(name: string): DataValue;
|
||||
public
|
||||
/// Создает курсор для указанных столбцов
|
||||
constructor Create(cols: array of Column; schema: DataFrameSchema);
|
||||
|
|
@ -238,6 +352,8 @@ type
|
|||
function Str(i: integer): string;
|
||||
/// Возвращает булево значение из столбца по индексу
|
||||
function Bool(i: integer): boolean;
|
||||
/// Возвращает значение DateTime из столбца по индексу
|
||||
function DateTime(i: integer): System.DateTime;
|
||||
/// Возвращает целочисленное значение из столбца по имени
|
||||
function Int(name: string): integer;
|
||||
/// Возвращает вещественное значение из столбца по имени
|
||||
|
|
@ -246,6 +362,17 @@ type
|
|||
function Str(name: string): string;
|
||||
/// Возвращает булево значение из столбца по имени
|
||||
function Bool(name: string): boolean;
|
||||
/// Возвращает значение DateTime из столбца по имени
|
||||
function DateTime(name: string): System.DateTime;
|
||||
/// Возвращает типизированное значение ячейки по индексу.
|
||||
/// Пример: row.Value(0).IsValid
|
||||
function Value(i: integer): DataValue := GetValue(i);
|
||||
/// Возвращает типизированное значение ячейки по имени столбца.
|
||||
/// Это основной способ проверки и чтения значения:
|
||||
/// row['score'].IsValid
|
||||
/// row['score'].Float
|
||||
/// row['name'].Str
|
||||
property Item[name: string]: DataValue read GetValue; default;
|
||||
/// Проверяет валидность значения в столбце по индексу
|
||||
function IsValid(i: integer): boolean;
|
||||
/// Проверяет валидность значения в столбце по имени
|
||||
|
|
@ -286,6 +413,8 @@ const
|
|||
'Столбец не является Str!!Column is not Str';
|
||||
ER_COLUMN_NOT_BOOL =
|
||||
'Столбец не является Bool!!Column is not Bool';
|
||||
ER_COLUMN_NOT_DATETIME =
|
||||
'Столбец не является DateTime!!Column is not DateTime';
|
||||
ER_VALUE_IS_NA =
|
||||
'Значение в столбце "{0}" равно NA!!Value in column "{0}" is NA';
|
||||
ER_DUPLICATE_COLUMN_NAME =
|
||||
|
|
@ -318,7 +447,23 @@ const
|
|||
'Индекс строки {0} вне диапазона [0..{1})!!' +
|
||||
'Row index {0} out of range [0..{1})';
|
||||
ER_INVALID_ISVALID_LENGTH =
|
||||
'Длина IsValid должна совпадать с длиной Data!!IsValid length must match Data length';
|
||||
'Длина IsValid должна совпадать с длиной Data!!IsValid length must match Data length';
|
||||
ER_DATAVALUE_NOT_INT =
|
||||
'Значение "{0}" не является Int!!Value "{0}" is not Int';
|
||||
ER_DATAVALUE_NOT_FLOAT =
|
||||
'Значение "{0}" не является Float!!Value "{0}" is not Float';
|
||||
ER_DATAVALUE_NOT_STR =
|
||||
'Значение "{0}" не является Str!!Value "{0}" is not Str';
|
||||
ER_DATAVALUE_NOT_BOOL =
|
||||
'Значение "{0}" не является Bool!!Value "{0}" is not Bool';
|
||||
ER_DATAVALUE_NOT_DATETIME =
|
||||
'Значение "{0}" не является DateTime!!Value "{0}" is not DateTime';
|
||||
ER_DATAVALUE_COMPARE_TYPES_MISMATCH =
|
||||
'Нельзя сравнивать значения типов {0} и {1}!!Cannot compare values of types {0} and {1}';
|
||||
ER_DATAVALUE_ARITHMETIC_TYPES_MISMATCH =
|
||||
'Нельзя выполнять арифметические операции для типов {0} и {1}!!Cannot apply arithmetic to types {0} and {1}';
|
||||
ER_DATAVALUE_DIVISION_BY_ZERO =
|
||||
'Деление на ноль!!Division by zero';
|
||||
|
||||
//-----------------------------
|
||||
// Сервисные функции
|
||||
|
|
@ -348,6 +493,25 @@ begin
|
|||
Error(ER_COLUMN_NOT_BOOL);
|
||||
end;
|
||||
|
||||
function NotDateTime(pos: integer): System.DateTime;
|
||||
begin
|
||||
Result := default(System.DateTime);
|
||||
Error(ER_COLUMN_NOT_DATETIME);
|
||||
end;
|
||||
|
||||
function ColumnTypeName(t: ColumnType): string;
|
||||
begin
|
||||
case t of
|
||||
ctInt: Result := 'Int';
|
||||
ctFloat: Result := 'Float';
|
||||
ctStr: Result := 'Str';
|
||||
ctBool: Result := 'Bool';
|
||||
ctDateTime: Result := 'DateTime';
|
||||
else
|
||||
Result := t.ToString;
|
||||
end;
|
||||
end;
|
||||
|
||||
//-----------------------------
|
||||
// DataFrameSchema
|
||||
//-----------------------------
|
||||
|
|
@ -409,6 +573,275 @@ begin
|
|||
fColType := colType;
|
||||
end;
|
||||
|
||||
class function DataValue.IsNumericType(t: ColumnType): boolean;
|
||||
begin
|
||||
Result := t in [ctInt, ctFloat];
|
||||
end;
|
||||
|
||||
constructor DataValue.Create(name: string; value: integer);
|
||||
begin
|
||||
fName := name;
|
||||
fColType := ctInt;
|
||||
fIsValid := true;
|
||||
fInt := value;
|
||||
fFloat := value;
|
||||
end;
|
||||
|
||||
constructor DataValue.Create(name: string; value: real);
|
||||
begin
|
||||
fName := name;
|
||||
fColType := ctFloat;
|
||||
fIsValid := true;
|
||||
fFloat := value;
|
||||
end;
|
||||
|
||||
constructor DataValue.Create(name: string; value: string);
|
||||
begin
|
||||
fName := name;
|
||||
fColType := ctStr;
|
||||
fIsValid := true;
|
||||
fStr := value;
|
||||
end;
|
||||
|
||||
constructor DataValue.Create(name: string; value: boolean);
|
||||
begin
|
||||
fName := name;
|
||||
fColType := ctBool;
|
||||
fIsValid := true;
|
||||
fBool := value;
|
||||
end;
|
||||
|
||||
constructor DataValue.Create(name: string; value: System.DateTime);
|
||||
begin
|
||||
fName := name;
|
||||
fColType := ctDateTime;
|
||||
fIsValid := true;
|
||||
fDateTime := value;
|
||||
end;
|
||||
|
||||
class function DataValue.NA(name: string; colType: ColumnType): DataValue;
|
||||
begin
|
||||
Result := new DataValue(name, 0);
|
||||
Result.fColType := colType;
|
||||
Result.fIsValid := false;
|
||||
end;
|
||||
|
||||
function DataValue.GetInt: integer;
|
||||
begin
|
||||
if not fIsValid then
|
||||
Error(ER_VALUE_IS_NA, fName);
|
||||
if fColType <> ctInt then
|
||||
Error(ER_DATAVALUE_NOT_INT, fName);
|
||||
Result := fInt;
|
||||
end;
|
||||
|
||||
function DataValue.GetFloat: real;
|
||||
begin
|
||||
if not fIsValid then
|
||||
Error(ER_VALUE_IS_NA, fName);
|
||||
if not IsNumericType(fColType) then
|
||||
Error(ER_DATAVALUE_NOT_FLOAT, fName);
|
||||
if fColType = ctInt then
|
||||
Result := fInt
|
||||
else
|
||||
Result := fFloat;
|
||||
end;
|
||||
|
||||
function DataValue.GetStr: string;
|
||||
begin
|
||||
if not fIsValid then
|
||||
Error(ER_VALUE_IS_NA, fName);
|
||||
if fColType <> ctStr then
|
||||
Error(ER_DATAVALUE_NOT_STR, fName);
|
||||
Result := fStr;
|
||||
end;
|
||||
|
||||
function DataValue.GetBool: boolean;
|
||||
begin
|
||||
if not fIsValid then
|
||||
Error(ER_VALUE_IS_NA, fName);
|
||||
if fColType <> ctBool then
|
||||
Error(ER_DATAVALUE_NOT_BOOL, fName);
|
||||
Result := fBool;
|
||||
end;
|
||||
|
||||
function DataValue.GetDateTime: System.DateTime;
|
||||
begin
|
||||
if not fIsValid then
|
||||
Error(ER_VALUE_IS_NA, fName);
|
||||
if fColType <> ctDateTime then
|
||||
Error(ER_DATAVALUE_NOT_DATETIME, fName);
|
||||
Result := fDateTime;
|
||||
end;
|
||||
|
||||
function DataValue.ToString: string;
|
||||
begin
|
||||
if not fIsValid then
|
||||
exit('NA');
|
||||
|
||||
case fColType of
|
||||
ctInt: Result := fInt.ToString;
|
||||
ctFloat: Result := fFloat.ToString;
|
||||
ctStr: Result := fStr;
|
||||
ctBool: Result := fBool.ToString;
|
||||
ctDateTime: Result := fDateTime.ToString('yyyy-MM-dd HH:mm:ss');
|
||||
else
|
||||
Result := '';
|
||||
end;
|
||||
end;
|
||||
|
||||
class function DataValue.Compare(a, b: DataValue): integer;
|
||||
begin
|
||||
if System.Object.ReferenceEquals(a, nil) then
|
||||
ArgumentNullError('a');
|
||||
if System.Object.ReferenceEquals(b, nil) then
|
||||
ArgumentNullError('b');
|
||||
if not a.fIsValid then
|
||||
Error(ER_VALUE_IS_NA, a.fName);
|
||||
if not b.fIsValid then
|
||||
Error(ER_VALUE_IS_NA, b.fName);
|
||||
|
||||
if IsNumericType(a.fColType) and IsNumericType(b.fColType) then
|
||||
begin
|
||||
var av := a.Float;
|
||||
var bv := b.Float;
|
||||
if av < bv then
|
||||
Result := -1
|
||||
else if av > bv then
|
||||
Result := 1
|
||||
else
|
||||
Result := 0;
|
||||
exit;
|
||||
end;
|
||||
|
||||
if a.fColType <> b.fColType then
|
||||
Error(ER_DATAVALUE_COMPARE_TYPES_MISMATCH, ColumnTypeName(a.fColType), ColumnTypeName(b.fColType));
|
||||
|
||||
case a.fColType of
|
||||
ctStr:
|
||||
if a.fStr < b.fStr then Result := -1 else if a.fStr > b.fStr then Result := 1 else Result := 0;
|
||||
ctBool:
|
||||
if integer(a.fBool) < integer(b.fBool) then Result := -1 else if integer(a.fBool) > integer(b.fBool) then Result := 1 else Result := 0;
|
||||
ctDateTime:
|
||||
if a.fDateTime < b.fDateTime then Result := -1 else if a.fDateTime > b.fDateTime then Result := 1 else Result := 0;
|
||||
else
|
||||
Result := 0;
|
||||
end;
|
||||
end;
|
||||
|
||||
class function DataValue.ArithmeticType(a, b: DataValue): ColumnType;
|
||||
begin
|
||||
if System.Object.ReferenceEquals(a, nil) then
|
||||
ArgumentNullError('a');
|
||||
if System.Object.ReferenceEquals(b, nil) then
|
||||
ArgumentNullError('b');
|
||||
if not a.fIsValid then
|
||||
Error(ER_VALUE_IS_NA, a.fName);
|
||||
if not b.fIsValid then
|
||||
Error(ER_VALUE_IS_NA, b.fName);
|
||||
|
||||
if not (IsNumericType(a.fColType) and IsNumericType(b.fColType)) then
|
||||
Error(ER_DATAVALUE_ARITHMETIC_TYPES_MISMATCH, ColumnTypeName(a.fColType), ColumnTypeName(b.fColType));
|
||||
|
||||
if (a.fColType = ctInt) and (b.fColType = ctInt) then
|
||||
Result := ctInt
|
||||
else
|
||||
Result := ctFloat;
|
||||
end;
|
||||
|
||||
class function DataValue.MakeNumeric(name: string; value: real; asInt: boolean): DataValue;
|
||||
begin
|
||||
if asInt then
|
||||
Result := new DataValue(name, integer(value))
|
||||
else
|
||||
Result := new DataValue(name, value);
|
||||
end;
|
||||
|
||||
{static function DataValue.operator implicit(v: DataValue): integer;
|
||||
begin
|
||||
if System.Object.ReferenceEquals(v, nil) then
|
||||
ArgumentNullError('v');
|
||||
Result := v.Int;
|
||||
end;
|
||||
|
||||
static function DataValue.operator implicit(v: DataValue): real;
|
||||
begin
|
||||
if System.Object.ReferenceEquals(v, nil) then
|
||||
ArgumentNullError('v');
|
||||
Result := v.Float;
|
||||
end;
|
||||
|
||||
static function DataValue.operator implicit(v: DataValue): string;
|
||||
begin
|
||||
if System.Object.ReferenceEquals(v, nil) then
|
||||
ArgumentNullError('v');
|
||||
Result := v.Str;
|
||||
end;
|
||||
|
||||
static function DataValue.operator implicit(v: DataValue): boolean;
|
||||
begin
|
||||
if System.Object.ReferenceEquals(v, nil) then
|
||||
ArgumentNullError('v');
|
||||
Result := v.Bool;
|
||||
end;
|
||||
|
||||
static function DataValue.operator implicit(v: DataValue): System.DateTime;
|
||||
begin
|
||||
if System.Object.ReferenceEquals(v, nil) then
|
||||
ArgumentNullError('v');
|
||||
Result := v.DateTime;
|
||||
end;}
|
||||
|
||||
static function DataValue.operator =(a, b: DataValue): boolean;
|
||||
begin
|
||||
if System.Object.ReferenceEquals(a, nil) then
|
||||
exit(System.Object.ReferenceEquals(b, nil));
|
||||
if System.Object.ReferenceEquals(b, nil) then
|
||||
exit(false);
|
||||
Result := Compare(a, b) = 0;
|
||||
end;
|
||||
|
||||
static function DataValue.operator +(a, b: DataValue): DataValue;
|
||||
begin
|
||||
if System.Object.ReferenceEquals(a, nil) then
|
||||
ArgumentNullError('a');
|
||||
if System.Object.ReferenceEquals(b, nil) then
|
||||
ArgumentNullError('b');
|
||||
if not a.fIsValid then
|
||||
Error(ER_VALUE_IS_NA, a.fName);
|
||||
if not b.fIsValid then
|
||||
Error(ER_VALUE_IS_NA, b.fName);
|
||||
|
||||
if (a.fColType = ctStr) and (b.fColType = ctStr) then
|
||||
begin
|
||||
Result := new DataValue(a.Name, a.fStr + b.fStr);
|
||||
exit;
|
||||
end;
|
||||
|
||||
var t := ArithmeticType(a, b);
|
||||
Result := MakeNumeric(a.Name, a.Float + b.Float, t = ctInt);
|
||||
end;
|
||||
|
||||
static function DataValue.operator -(a, b: DataValue): DataValue;
|
||||
begin
|
||||
var t := ArithmeticType(a, b);
|
||||
Result := MakeNumeric(a.Name, a.Float - b.Float, t = ctInt);
|
||||
end;
|
||||
|
||||
static function DataValue.operator *(a, b: DataValue): DataValue;
|
||||
begin
|
||||
var t := ArithmeticType(a, b);
|
||||
Result := MakeNumeric(a.Name, a.Float * b.Float, t = ctInt);
|
||||
end;
|
||||
|
||||
static function DataValue.operator /(a, b: DataValue): DataValue;
|
||||
begin
|
||||
ArithmeticType(a, b);
|
||||
if b.Float = 0 then
|
||||
Error(ER_DATAVALUE_DIVISION_BY_ZERO);
|
||||
Result := new DataValue(a.Name, a.Float / b.Float);
|
||||
end;
|
||||
|
||||
procedure DataFrameSchema.Print;
|
||||
begin
|
||||
if fNames.Length = 0 then
|
||||
|
|
@ -430,6 +863,7 @@ begin
|
|||
ctFloat: t := 'float';
|
||||
ctStr: t := 'string';
|
||||
ctBool: t := 'bool';
|
||||
ctDateTime: t := 'datetime';
|
||||
end;
|
||||
|
||||
if fCategoricalFlags[i] then
|
||||
|
|
@ -766,6 +1200,39 @@ begin
|
|||
exit(True);
|
||||
end;
|
||||
|
||||
constructor DateTimeColumn.Create(name: string; values: array of System.DateTime; valid: array of boolean);
|
||||
begin
|
||||
inherited Create;
|
||||
Info := new ColumnInfo(name, ctDateTime);
|
||||
|
||||
var n := Length(values);
|
||||
Self.Data := if n = 0 then [] else values;
|
||||
|
||||
if valid = nil then
|
||||
IsValid := [True] * n
|
||||
else
|
||||
begin
|
||||
if Length(valid) <> n then
|
||||
Error(ER_INVALID_ISVALID_LENGTH);
|
||||
|
||||
IsValid := valid;
|
||||
end;
|
||||
end;
|
||||
|
||||
constructor DateTimeColumn.Create(name: string);
|
||||
begin
|
||||
inherited Create;
|
||||
Info := new ColumnInfo(name, ctDateTime);
|
||||
|
||||
Data := new System.DateTime[0];
|
||||
IsValid := new boolean[0];
|
||||
end;
|
||||
|
||||
function DateTimeColumn.TryGetNumericValue(i: integer; var value: real): boolean;
|
||||
begin
|
||||
exit(False);
|
||||
end;
|
||||
|
||||
|
||||
|
||||
//-----------------------------
|
||||
|
|
@ -830,6 +1297,7 @@ begin
|
|||
ctFloat: rowCnt := FloatColumn(cols[0]).Data.Length;
|
||||
ctStr: rowCnt := StrColumn(cols[0]).Data.Length;
|
||||
ctBool: rowCnt := BoolColumn(cols[0]).Data.Length;
|
||||
ctDateTime: rowCnt := DateTimeColumn(cols[0]).Data.Length;
|
||||
end;
|
||||
|
||||
var n := cols.Length;
|
||||
|
|
@ -839,6 +1307,7 @@ begin
|
|||
floatAcc := new FloatAccessor[n];
|
||||
strAcc := new StrAccessor[n];
|
||||
boolAcc := new BoolAccessor[n];
|
||||
dtAcc := new DateTimeAccessor[n];
|
||||
validAcc := new ValidAccessor[n];
|
||||
|
||||
for var i := 0 to n - 1 do
|
||||
|
|
@ -849,6 +1318,7 @@ begin
|
|||
floatAcc[i] := NotFloat;
|
||||
strAcc[i] := NotStr;
|
||||
boolAcc[i] := NotBool;
|
||||
dtAcc[i] := NotDateTime;
|
||||
|
||||
case fSchema.ColumnTypeAt(i) of
|
||||
ctInt:
|
||||
|
|
@ -883,6 +1353,14 @@ begin
|
|||
|
||||
boolAcc[i] := pos -> c.Data[pos];
|
||||
end;
|
||||
|
||||
ctDateTime:
|
||||
begin
|
||||
var c := DateTimeColumn(col);
|
||||
validAcc[i] := pos -> c.IsValid[pos];
|
||||
|
||||
dtAcc[i] := pos -> c.Data[pos];
|
||||
end;
|
||||
|
||||
else Error(ER_UNKNOWN_COLUMN_TYPE);
|
||||
end;
|
||||
|
|
@ -932,6 +1410,13 @@ begin
|
|||
Error(ER_VALUE_IS_NA, fSchema.NameAt(i));
|
||||
Result := boolAcc[i](pos);
|
||||
end;
|
||||
|
||||
function DataFrameCursor.DateTime(i: integer): System.DateTime;
|
||||
begin
|
||||
if not IsValid(i) then
|
||||
Error(ER_VALUE_IS_NA, fSchema.NameAt(i));
|
||||
Result := dtAcc[i](pos);
|
||||
end;
|
||||
|
||||
function DataFrameCursor.Int(name: string): integer;
|
||||
begin
|
||||
|
|
@ -952,6 +1437,34 @@ function DataFrameCursor.Bool(name: string): boolean;
|
|||
begin
|
||||
Result := Bool(fSchema.IndexOf(name));
|
||||
end;
|
||||
|
||||
function DataFrameCursor.DateTime(name: string): System.DateTime;
|
||||
begin
|
||||
Result := DateTime(fSchema.IndexOf(name));
|
||||
end;
|
||||
|
||||
function DataFrameCursor.GetValue(i: integer): DataValue;
|
||||
begin
|
||||
var name := fSchema.NameAt(i);
|
||||
|
||||
if not IsValid(i) then
|
||||
exit(DataValue.NA(name, fSchema.ColumnTypeAt(i)));
|
||||
|
||||
case fSchema.ColumnTypeAt(i) of
|
||||
ctInt: Result := new DataValue(name, Int(i));
|
||||
ctFloat: Result := new DataValue(name, Float(i));
|
||||
ctStr: Result := new DataValue(name, Str(i));
|
||||
ctBool: Result := new DataValue(name, Bool(i));
|
||||
ctDateTime: Result := new DataValue(name, DateTime(i));
|
||||
else
|
||||
Error(ER_UNKNOWN_COLUMN_TYPE);
|
||||
end;
|
||||
end;
|
||||
|
||||
function DataFrameCursor.GetValue(name: string): DataValue;
|
||||
begin
|
||||
Result := GetValue(fSchema.IndexOf(name));
|
||||
end;
|
||||
|
||||
procedure DataFrameCursor.MoveTo(p: integer);
|
||||
begin
|
||||
|
|
|
|||
|
|
@ -106,8 +106,10 @@ type
|
|||
|
||||
DataFrame = DataFrameABC.DataFrame;
|
||||
DataFrameCursor = DataFrameABCCore.DataFrameCursor;
|
||||
DataValue = DataFrameABCCore.DataValue;
|
||||
ColumnType = DataFrameABCCore.ColumnType;
|
||||
Column = DataFrameABCCore.Column;
|
||||
DatePartKind = DataFrameABC.DatePartKind;
|
||||
|
||||
Statistics = DataFrameABC.Statistics;
|
||||
CsvLoader = DataFrameABC.CsvLoader;
|
||||
|
|
@ -163,6 +165,10 @@ type
|
|||
IPreprocessor = PreprocessorABC.IPreprocessor;
|
||||
OrdinalEncoder = PreprocessorABC.OrdinalEncoder;
|
||||
OneHotEncoder = PreprocessorABC.OneHotEncoder;
|
||||
DateTimeComponentsEncoder = PreprocessorABC.DateTimeComponentsEncoder;
|
||||
DateTimeUnit = PreprocessorABC.DateTimeUnit;
|
||||
DateTimeOrdinalEncoder = PreprocessorABC.DateTimeOrdinalEncoder;
|
||||
DateTimeCyclicEncoder = PreprocessorABC.DateTimeCyclicEncoder;
|
||||
ImputeStrategy = PreprocessorABC.ImputeStrategy;
|
||||
Imputer = PreprocessorABC.Imputer;
|
||||
|
||||
|
|
@ -191,6 +197,21 @@ const
|
|||
jkLeft = JoinKind.jkLeft;
|
||||
jkRight = JoinKind.jkRight;
|
||||
jkFull = JoinKind.jkFull;
|
||||
|
||||
dpYear = DatePartKind.dpYear;
|
||||
dpMonth = DatePartKind.dpMonth;
|
||||
dpDay = DatePartKind.dpDay;
|
||||
dpHour = DatePartKind.dpHour;
|
||||
dpMinute = DatePartKind.dpMinute;
|
||||
dpSecond = DatePartKind.dpSecond;
|
||||
dpDayOfWeek = DatePartKind.dpDayOfWeek;
|
||||
dpTimeOfDay = DatePartKind.dpTimeOfDay;
|
||||
dpDate = DatePartKind.dpDate;
|
||||
|
||||
dtuDays = DateTimeUnit.dtuDays;
|
||||
dtuHours = DateTimeUnit.dtuHours;
|
||||
dtuMinutes = DateTimeUnit.dtuMinutes;
|
||||
dtuSeconds = DateTimeUnit.dtuSeconds;
|
||||
|
||||
/// Кодирует строковые метки классов в целочисленные индексы.
|
||||
/// Каждому уникальному значению присваивается номер 0,1,2,...
|
||||
|
|
|
|||
|
|
@ -309,8 +309,8 @@ const
|
|||
ER_ENCODELABELS_NOT_CATEGORICAL =
|
||||
'Целевой столбец должен быть категориальным для задач классификации!!Target column must be categorical for classification tasks';
|
||||
ER_CLASSIFICATION_TARGET_MUST_BE_CATEGORICAL_STR_OR_INT =
|
||||
'Целевой столбец "{0}" должен быть категориальным строковым или целочисленным для задач классификации!!' +
|
||||
'Target column "{0}" must be a categorical string or integer column for classification tasks';
|
||||
'Целевой столбец "{0}" должен быть строковым, целочисленным или логическим для задач классификации!!' +
|
||||
'Target column "{0}" must be a string, integer, or boolean column for classification tasks';
|
||||
ER_REGRESSION_TARGET_MUST_BE_NUMERIC =
|
||||
'Целевой столбец "{0}" должен быть числовым для задач регрессии!!Target column "{0}" must be numeric for regression tasks';
|
||||
ER_PREPROCESSOR_ROWCOUNT_CHANGED =
|
||||
|
|
@ -990,10 +990,8 @@ begin
|
|||
case fTask of
|
||||
tkClassification:
|
||||
begin
|
||||
if not df.IsCategorical(fTarget) then
|
||||
ArgumentError(ER_ENCODELABELS_NOT_CATEGORICAL, fTarget);
|
||||
var ct := df.GetColumnType(fTarget);
|
||||
if not (ct in [ColumnType.ctStr, ColumnType.ctInt]) then
|
||||
if not (ct in [ColumnType.ctStr, ColumnType.ctInt, ColumnType.ctBool]) then
|
||||
ArgumentError(ER_CLASSIFICATION_TARGET_MUST_BE_CATEGORICAL_STR_OR_INT, fTarget);
|
||||
end;
|
||||
|
||||
|
|
|
|||
|
|
@ -123,6 +123,92 @@ type
|
|||
function Clone: IPreprocessor;
|
||||
end;
|
||||
|
||||
DateTimeUnit = (dtuDays, dtuHours, dtuMinutes, dtuSeconds);
|
||||
|
||||
/// Разворачивает DateTime-столбец в набор компонентных признаков.
|
||||
/// По умолчанию создаёт признаки year, month, day.
|
||||
/// Исходный DateTime-столбец заменяется новыми столбцами.
|
||||
DateTimeComponentsEncoder = class(IPreprocessor, IColumnBoundStep, IColumnExpander)
|
||||
private
|
||||
col: string;
|
||||
parts: array of DatePartKind;
|
||||
expandedColumns: array of string;
|
||||
fitted: boolean;
|
||||
function DefaultParts: array of DatePartKind;
|
||||
class function PartSuffix(part: DatePartKind): string;
|
||||
public
|
||||
/// Создаёт encoder со стандартным набором компонент: year, month, day.
|
||||
constructor Create(column: string);
|
||||
/// Создаёт encoder с явным списком компонент.
|
||||
constructor Create(column: string; dateParts: array of DatePartKind);
|
||||
|
||||
function Fit(df: DataFrame): IPreprocessor;
|
||||
function Transform(df: DataFrame): DataFrame;
|
||||
function FitTransform(df: DataFrame): DataFrame;
|
||||
function ToString: string; override;
|
||||
|
||||
property ColumnName: string read col;
|
||||
|
||||
function GetExpandedColumns(sourceColumn: string): array of string;
|
||||
function Clone: IPreprocessor;
|
||||
end;
|
||||
|
||||
/// Кодирует DateTime-столбец в один вещественный столбец,
|
||||
/// содержащий расстояние от минимального значения столбца в выбранных единицах.
|
||||
/// На Fit запоминается fOrigin = Min(column), на Transform вычисляется
|
||||
/// (dt - fOrigin).TotalDays / TotalHours / TotalMinutes / TotalSeconds.
|
||||
/// Исходный столбец заменяется вещественным столбцом с новым именем.
|
||||
DateTimeOrdinalEncoder = class(IPreprocessor, IColumnBoundStep)
|
||||
private
|
||||
col: string;
|
||||
newCol: string;
|
||||
unit_: DateTimeUnit;
|
||||
fOrigin: System.DateTime;
|
||||
fitted: boolean;
|
||||
class function DefaultColumnName(column: string; unit_: DateTimeUnit): string;
|
||||
public
|
||||
constructor Create(column: string);
|
||||
constructor Create(column, newColumn: string);
|
||||
constructor Create(column, newColumn: string; unit_: DateTimeUnit);
|
||||
|
||||
function Fit(df: DataFrame): IPreprocessor;
|
||||
function Transform(df: DataFrame): DataFrame;
|
||||
function FitTransform(df: DataFrame): DataFrame;
|
||||
function ToString: string; override;
|
||||
|
||||
property ColumnName: string read col;
|
||||
|
||||
function Clone: IPreprocessor;
|
||||
end;
|
||||
|
||||
/// Кодирует циклическую компоненту DateTime через пару sin/cos.
|
||||
/// Например, для dpHour создаёт created_at_cyc_hour_sin и created_at_cyc_hour_cos.
|
||||
/// Исходный DateTime-столбец заменяется двумя вещественными столбцами.
|
||||
DateTimeCyclicEncoder = class(IPreprocessor, IColumnBoundStep, IColumnExpander)
|
||||
private
|
||||
col: string;
|
||||
part: DatePartKind;
|
||||
sinCol: string;
|
||||
cosCol: string;
|
||||
fitted: boolean;
|
||||
class function PartSuffix(part: DatePartKind): string;
|
||||
class function IsSupportedPart(part: DatePartKind): boolean;
|
||||
class function PartPeriod(part: DatePartKind): integer;
|
||||
class function PartValue(dt: System.DateTime; part: DatePartKind): real;
|
||||
public
|
||||
constructor Create(column: string; part: DatePartKind);
|
||||
|
||||
function Fit(df: DataFrame): IPreprocessor;
|
||||
function Transform(df: DataFrame): DataFrame;
|
||||
function FitTransform(df: DataFrame): DataFrame;
|
||||
function ToString: string; override;
|
||||
|
||||
property ColumnName: string read col;
|
||||
|
||||
function GetExpandedColumns(sourceColumn: string): array of string;
|
||||
function Clone: IPreprocessor;
|
||||
end;
|
||||
|
||||
ImputeStrategy = (isMean, isConstant, isMedian);
|
||||
|
||||
/// Заполняет пропущенные значения (NA) в столбцах DataFrame
|
||||
|
|
@ -228,6 +314,45 @@ const
|
|||
'Конфликт имён колонок: {0}!!Column name collision: {0}';
|
||||
ER_ONEHOT_DUPLICATE_COLUMN =
|
||||
'Дублирующаяся сгенерированная колонка: {0}!!Duplicate generated column: {0}';
|
||||
ER_DTENC_NO_COLUMN =
|
||||
'DateTimeComponentsEncoder: столбец не указан!!DateTimeComponentsEncoder: column not specified';
|
||||
ER_DTENC_NOT_DATETIME =
|
||||
'DateTimeComponentsEncoder: столбец "{0}" не является DateTime!!' +
|
||||
'DateTimeComponentsEncoder: column "{0}" is not DateTime';
|
||||
ER_DTENC_EMPTY_PARTS =
|
||||
'DateTimeComponentsEncoder: список компонент пуст!!DateTimeComponentsEncoder: date parts list is empty';
|
||||
ER_DTENC_DUPLICATE_PART =
|
||||
'DateTimeComponentsEncoder: компонент даты повторяется: {0}!!DateTimeComponentsEncoder: duplicate date part: {0}';
|
||||
ER_DTENC_NAME_EQUALS_SOURCE =
|
||||
'DateTimeComponentsEncoder: сгенерированная колонка совпадает с исходной: {0}!!DateTimeComponentsEncoder: generated column equals source column: {0}';
|
||||
ER_DTENC_COLUMN_COLLISION =
|
||||
'DateTimeComponentsEncoder: конфликт имён колонок: {0}!!DateTimeComponentsEncoder: column name collision: {0}';
|
||||
ER_DTENC_DUPLICATE_COLUMN =
|
||||
'DateTimeComponentsEncoder: дублирующаяся сгенерированная колонка: {0}!!DateTimeComponentsEncoder: duplicate generated column: {0}';
|
||||
ER_DTORD_NO_COLUMN =
|
||||
'DateTimeOrdinalEncoder: столбец не указан!!DateTimeOrdinalEncoder: column not specified';
|
||||
ER_DTORD_NO_NEW_COLUMN =
|
||||
'DateTimeOrdinalEncoder: имя новой колонки не указано!!DateTimeOrdinalEncoder: new column name not specified';
|
||||
ER_DTORD_NOT_DATETIME =
|
||||
'DateTimeOrdinalEncoder: столбец "{0}" не является DateTime!!' +
|
||||
'DateTimeOrdinalEncoder: column "{0}" is not DateTime';
|
||||
ER_DTORD_NO_VALID_VALUES =
|
||||
'DateTimeOrdinalEncoder: столбец "{0}" не содержит допустимых значений!!' +
|
||||
'DateTimeOrdinalEncoder: column "{0}" has no valid values';
|
||||
ER_DTORD_COLUMN_COLLISION =
|
||||
'DateTimeOrdinalEncoder: конфликт имён колонок: {0}!!DateTimeOrdinalEncoder: column name collision: {0}';
|
||||
ER_DTCYC_NO_COLUMN =
|
||||
'DateTimeCyclicEncoder: столбец не указан!!DateTimeCyclicEncoder: column not specified';
|
||||
ER_DTCYC_NOT_DATETIME =
|
||||
'DateTimeCyclicEncoder: столбец "{0}" не является DateTime!!' +
|
||||
'DateTimeCyclicEncoder: column "{0}" is not DateTime';
|
||||
ER_DTCYC_UNSUPPORTED_PART =
|
||||
'DateTimeCyclicEncoder: часть даты {0} не поддерживается для циклического кодирования!!' +
|
||||
'DateTimeCyclicEncoder: date part {0} is not supported for cyclic encoding';
|
||||
ER_DTCYC_COLUMN_COLLISION =
|
||||
'DateTimeCyclicEncoder: конфликт имён колонок: {0}!!DateTimeCyclicEncoder: column name collision: {0}';
|
||||
ER_DTCYC_DUPLICATE_COLUMN =
|
||||
'DateTimeCyclicEncoder: дублирующаяся сгенерированная колонка: {0}!!DateTimeCyclicEncoder: duplicate generated column: {0}';
|
||||
|
||||
|
||||
//-----------------------------
|
||||
|
|
@ -532,6 +657,548 @@ begin
|
|||
Result := new OneHotEncoder(col);
|
||||
end;
|
||||
|
||||
//-----------------------------
|
||||
// DateTimeComponentsEncoder
|
||||
//-----------------------------
|
||||
|
||||
function DateTimeComponentsEncoder.DefaultParts: array of DatePartKind;
|
||||
begin
|
||||
Result := [DatePartKind.dpYear, DatePartKind.dpMonth, DatePartKind.dpDay];
|
||||
end;
|
||||
|
||||
class function DateTimeComponentsEncoder.PartSuffix(part: DatePartKind): string;
|
||||
begin
|
||||
case part of
|
||||
dpYear: Result := 'year';
|
||||
dpMonth: Result := 'month';
|
||||
dpDay: Result := 'day';
|
||||
dpHour: Result := 'hour';
|
||||
dpMinute: Result := 'minute';
|
||||
dpSecond: Result := 'second';
|
||||
dpDayOfWeek: Result := 'dayofweek';
|
||||
dpDate: Result := 'date';
|
||||
end;
|
||||
end;
|
||||
|
||||
constructor DateTimeComponentsEncoder.Create(column: string);
|
||||
begin
|
||||
if column = '' then
|
||||
ArgumentError(ER_DTENC_NO_COLUMN);
|
||||
|
||||
col := column;
|
||||
parts := DefaultParts;
|
||||
fitted := false;
|
||||
end;
|
||||
|
||||
constructor DateTimeComponentsEncoder.Create(column: string; dateParts: array of DatePartKind);
|
||||
begin
|
||||
if column = '' then
|
||||
ArgumentError(ER_DTENC_NO_COLUMN);
|
||||
if (dateParts = nil) or (dateParts.Length = 0) then
|
||||
ArgumentError(ER_DTENC_EMPTY_PARTS);
|
||||
|
||||
col := column;
|
||||
parts := Copy(dateParts);
|
||||
fitted := false;
|
||||
end;
|
||||
|
||||
function DateTimeComponentsEncoder.Fit(df: DataFrame): IPreprocessor;
|
||||
begin
|
||||
if df = nil then
|
||||
ArgumentNullError(ER_ARG_NULL, 'df');
|
||||
|
||||
var idx := df.Schema.IndexOf(col);
|
||||
if idx < 0 then
|
||||
Error(ER_COLUMN_NOT_FOUND, col);
|
||||
|
||||
if df.Schema.ColumnTypeAt(idx) <> ColumnType.ctDateTime then
|
||||
Error(ER_DTENC_NOT_DATETIME, col);
|
||||
|
||||
var usedParts := new HashSet<DatePartKind>;
|
||||
for var i := 0 to parts.Length - 1 do
|
||||
if usedParts.Contains(parts[i]) then
|
||||
Error(ER_DTENC_DUPLICATE_PART, parts[i])
|
||||
else
|
||||
usedParts.Add(parts[i]);
|
||||
|
||||
expandedColumns := new string[parts.Length];
|
||||
var usedNames := new HashSet<string>;
|
||||
|
||||
for var i := 0 to parts.Length - 1 do
|
||||
begin
|
||||
var newName := col + '_' + PartSuffix(parts[i]);
|
||||
expandedColumns[i] := newName;
|
||||
|
||||
if newName = col then
|
||||
Error(ER_DTENC_NAME_EQUALS_SOURCE, newName);
|
||||
|
||||
if df.HasColumn(newName) then
|
||||
Error(ER_DTENC_COLUMN_COLLISION, newName);
|
||||
|
||||
if usedNames.Contains(newName) then
|
||||
Error(ER_DTENC_DUPLICATE_COLUMN, newName);
|
||||
|
||||
usedNames.Add(newName);
|
||||
end;
|
||||
|
||||
fitted := true;
|
||||
Result := Self;
|
||||
end;
|
||||
|
||||
function DateTimeComponentsEncoder.Transform(df: DataFrame): DataFrame;
|
||||
begin
|
||||
if not fitted then
|
||||
NotFittedError(ER_FIT_NOT_CALLED);
|
||||
|
||||
var srcIdx := df.Schema.IndexOf(col);
|
||||
if srcIdx < 0 then
|
||||
ArgumentError(ER_COLUMN_NOT_FOUND, col);
|
||||
if df.Schema.ColumnTypeAt(srcIdx) <> ColumnType.ctDateTime then
|
||||
Error(ER_DTENC_NOT_DATETIME, col);
|
||||
|
||||
var rowCount := df.RowCount;
|
||||
var srcCol := DateTimeColumn(df.GetColumn(srcIdx));
|
||||
var srcData := srcCol.Data;
|
||||
var srcValid := srcCol.IsValid;
|
||||
|
||||
var res := new DataFrame;
|
||||
var names := new List<string>;
|
||||
var types := new List<ColumnType>;
|
||||
var cats := new List<boolean>;
|
||||
|
||||
AppendAllColumnsExcept(res, df, srcIdx, names, types, cats);
|
||||
|
||||
for var j := 0 to parts.Length - 1 do
|
||||
begin
|
||||
var part := parts[j];
|
||||
var newName := expandedColumns[j];
|
||||
|
||||
if part = dpDate then
|
||||
begin
|
||||
var data := new System.DateTime[rowCount];
|
||||
var valid := new boolean[rowCount];
|
||||
|
||||
for var row := 0 to rowCount - 1 do
|
||||
if (srcValid <> nil) and not srcValid[row] then
|
||||
begin
|
||||
data[row] := default(System.DateTime);
|
||||
valid[row] := False;
|
||||
end
|
||||
else
|
||||
begin
|
||||
data[row] := srcData[row].Date;
|
||||
valid[row] := True;
|
||||
end;
|
||||
|
||||
res.AddDateTimeColumn(newName, data, valid);
|
||||
names.Add(newName);
|
||||
types.Add(ColumnType.ctDateTime);
|
||||
cats.Add(False);
|
||||
end
|
||||
else
|
||||
begin
|
||||
var data := new integer[rowCount];
|
||||
var valid := new boolean[rowCount];
|
||||
|
||||
for var row := 0 to rowCount - 1 do
|
||||
if (srcValid <> nil) and not srcValid[row] then
|
||||
begin
|
||||
data[row] := 0;
|
||||
valid[row] := False;
|
||||
end
|
||||
else
|
||||
begin
|
||||
case part of
|
||||
dpYear: data[row] := srcData[row].Year;
|
||||
dpMonth: data[row] := srcData[row].Month;
|
||||
dpDay: data[row] := srcData[row].Day;
|
||||
dpHour: data[row] := srcData[row].Hour;
|
||||
dpMinute: data[row] := srcData[row].Minute;
|
||||
dpSecond: data[row] := srcData[row].Second;
|
||||
dpDayOfWeek: data[row] := integer(srcData[row].DayOfWeek);
|
||||
end;
|
||||
valid[row] := True;
|
||||
end;
|
||||
|
||||
res.AddIntColumn(newName, data, valid);
|
||||
names.Add(newName);
|
||||
types.Add(ColumnType.ctInt);
|
||||
cats.Add(False);
|
||||
end;
|
||||
end;
|
||||
|
||||
res.SetSchema(new DataFrameSchema(
|
||||
names.ToArray,
|
||||
types.ToArray,
|
||||
cats.ToArray
|
||||
));
|
||||
|
||||
Result := res;
|
||||
end;
|
||||
|
||||
function DateTimeComponentsEncoder.FitTransform(df: DataFrame): DataFrame;
|
||||
begin
|
||||
Fit(df);
|
||||
Result := Transform(df);
|
||||
end;
|
||||
|
||||
function DateTimeComponentsEncoder.ToString: string;
|
||||
begin
|
||||
Result := 'DateTimeComponentsEncoder(column=' + col + ')';
|
||||
end;
|
||||
|
||||
function DateTimeComponentsEncoder.GetExpandedColumns(sourceColumn: string): array of string;
|
||||
begin
|
||||
if not fitted then
|
||||
NotFittedError(ER_FIT_NOT_CALLED);
|
||||
|
||||
if sourceColumn <> col then
|
||||
exit(nil);
|
||||
|
||||
Result := Copy(expandedColumns);
|
||||
end;
|
||||
|
||||
function DateTimeComponentsEncoder.Clone: IPreprocessor;
|
||||
begin
|
||||
Result := new DateTimeComponentsEncoder(col, parts);
|
||||
end;
|
||||
|
||||
//-----------------------------
|
||||
// DateTimeOrdinalEncoder
|
||||
//-----------------------------
|
||||
|
||||
class function DateTimeOrdinalEncoder.DefaultColumnName(column: string; unit_: DateTimeUnit): string;
|
||||
begin
|
||||
case unit_ of
|
||||
dtuDays: Result := column + '_days';
|
||||
dtuHours: Result := column + '_hours';
|
||||
dtuMinutes: Result := column + '_minutes';
|
||||
dtuSeconds: Result := column + '_seconds';
|
||||
end;
|
||||
end;
|
||||
|
||||
constructor DateTimeOrdinalEncoder.Create(column: string);
|
||||
begin
|
||||
Create(column, DefaultColumnName(column, DateTimeUnit.dtuDays), DateTimeUnit.dtuDays);
|
||||
end;
|
||||
|
||||
constructor DateTimeOrdinalEncoder.Create(column, newColumn: string);
|
||||
begin
|
||||
Create(column, newColumn, DateTimeUnit.dtuDays);
|
||||
end;
|
||||
|
||||
constructor DateTimeOrdinalEncoder.Create(column, newColumn: string; unit_: DateTimeUnit);
|
||||
begin
|
||||
if column = '' then
|
||||
ArgumentError(ER_DTORD_NO_COLUMN);
|
||||
if newColumn = '' then
|
||||
ArgumentError(ER_DTORD_NO_NEW_COLUMN);
|
||||
|
||||
col := column;
|
||||
newCol := newColumn;
|
||||
self.unit_ := unit_;
|
||||
fitted := false;
|
||||
end;
|
||||
|
||||
function DateTimeOrdinalEncoder.Fit(df: DataFrame): IPreprocessor;
|
||||
begin
|
||||
if df = nil then
|
||||
ArgumentNullError(ER_ARG_NULL, 'df');
|
||||
|
||||
var idx := df.Schema.IndexOf(col);
|
||||
if idx < 0 then
|
||||
Error(ER_COLUMN_NOT_FOUND, col);
|
||||
|
||||
if df.Schema.ColumnTypeAt(idx) <> ColumnType.ctDateTime then
|
||||
Error(ER_DTORD_NOT_DATETIME, col);
|
||||
|
||||
if (newCol <> col) and df.HasColumn(newCol) then
|
||||
Error(ER_DTORD_COLUMN_COLLISION, newCol);
|
||||
|
||||
var cur := df.GetCursor;
|
||||
var hasValue := false;
|
||||
var minValue := default(System.DateTime);
|
||||
|
||||
while cur.MoveNext do
|
||||
if cur.IsValid(idx) then
|
||||
begin
|
||||
var dt := cur.DateTime(idx);
|
||||
if not hasValue or (dt < minValue) then
|
||||
begin
|
||||
minValue := dt;
|
||||
hasValue := true;
|
||||
end;
|
||||
end;
|
||||
|
||||
if not hasValue then
|
||||
Error(ER_DTORD_NO_VALID_VALUES, col);
|
||||
|
||||
fOrigin := minValue;
|
||||
fitted := true;
|
||||
Result := Self;
|
||||
end;
|
||||
|
||||
function DateTimeOrdinalEncoder.Transform(df: DataFrame): DataFrame;
|
||||
begin
|
||||
if not fitted then
|
||||
NotFittedError(ER_FIT_NOT_CALLED);
|
||||
|
||||
var idx := df.Schema.IndexOf(col);
|
||||
if idx < 0 then
|
||||
ArgumentError(ER_COLUMN_NOT_FOUND, col);
|
||||
if df.Schema.ColumnTypeAt(idx) <> ColumnType.ctDateTime then
|
||||
Error(ER_DTORD_NOT_DATETIME, col);
|
||||
|
||||
var srcCol := DateTimeColumn(df.GetColumn(idx));
|
||||
var rowCount := df.RowCount;
|
||||
var data := new real[rowCount];
|
||||
var valid := new boolean[rowCount];
|
||||
|
||||
for var row := 0 to rowCount - 1 do
|
||||
if (srcCol.IsValid <> nil) and not srcCol.IsValid[row] then
|
||||
begin
|
||||
data[row] := 0.0;
|
||||
valid[row] := False;
|
||||
end
|
||||
else
|
||||
begin
|
||||
var delta := srcCol.Data[row] - fOrigin;
|
||||
case unit_ of
|
||||
dtuDays: data[row] := delta.TotalDays;
|
||||
dtuHours: data[row] := delta.TotalHours;
|
||||
dtuMinutes: data[row] := delta.TotalMinutes;
|
||||
dtuSeconds: data[row] := delta.TotalSeconds;
|
||||
end;
|
||||
valid[row] := True;
|
||||
end;
|
||||
|
||||
var res := new DataFrame;
|
||||
var names := new List<string>;
|
||||
var types := new List<ColumnType>;
|
||||
var cats := new List<boolean>;
|
||||
|
||||
for var i := 0 to df.ColumnCount - 1 do
|
||||
if i <> idx then
|
||||
begin
|
||||
res.AddColumnAlias(df.GetColumn(i));
|
||||
names.Add(df.Schema.NameAt(i));
|
||||
types.Add(df.Schema.ColumnTypeAt(i));
|
||||
cats.Add(df.Schema.IsCategoricalAt(i));
|
||||
end
|
||||
else
|
||||
begin
|
||||
res.AddFloatColumn(newCol, data, valid);
|
||||
names.Add(newCol);
|
||||
types.Add(ColumnType.ctFloat);
|
||||
cats.Add(False);
|
||||
end;
|
||||
|
||||
res.SetSchema(new DataFrameSchema(
|
||||
names.ToArray,
|
||||
types.ToArray,
|
||||
cats.ToArray
|
||||
));
|
||||
|
||||
Result := res;
|
||||
end;
|
||||
|
||||
function DateTimeOrdinalEncoder.FitTransform(df: DataFrame): DataFrame;
|
||||
begin
|
||||
Fit(df);
|
||||
Result := Transform(df);
|
||||
end;
|
||||
|
||||
function DateTimeOrdinalEncoder.ToString: string;
|
||||
begin
|
||||
Result := 'DateTimeOrdinalEncoder(column=' + col + ', newColumn=' + newCol + ', unit=' + unit_.ToString + ')';
|
||||
end;
|
||||
|
||||
function DateTimeOrdinalEncoder.Clone: IPreprocessor;
|
||||
begin
|
||||
Result := new DateTimeOrdinalEncoder(col, newCol, unit_);
|
||||
end;
|
||||
|
||||
//-----------------------------
|
||||
// DateTimeCyclicEncoder
|
||||
//-----------------------------
|
||||
|
||||
class function DateTimeCyclicEncoder.PartSuffix(part: DatePartKind): string;
|
||||
begin
|
||||
case part of
|
||||
dpMonth: Result := 'month';
|
||||
dpDayOfWeek: Result := 'dayofweek';
|
||||
dpHour: Result := 'hour';
|
||||
dpMinute: Result := 'minute';
|
||||
dpSecond: Result := 'second';
|
||||
dpTimeOfDay: Result := 'timeofday';
|
||||
end;
|
||||
end;
|
||||
|
||||
class function DateTimeCyclicEncoder.IsSupportedPart(part: DatePartKind): boolean;
|
||||
begin
|
||||
Result := part in [dpMonth, dpDayOfWeek, dpHour, dpMinute, dpSecond, dpTimeOfDay];
|
||||
end;
|
||||
|
||||
class function DateTimeCyclicEncoder.PartPeriod(part: DatePartKind): integer;
|
||||
begin
|
||||
case part of
|
||||
dpMonth: Result := 12;
|
||||
dpDayOfWeek: Result := 7;
|
||||
dpHour: Result := 24;
|
||||
dpMinute: Result := 60;
|
||||
dpSecond: Result := 60;
|
||||
dpTimeOfDay: Result := 86400;
|
||||
else
|
||||
Error(ER_DTCYC_UNSUPPORTED_PART, part);
|
||||
end;
|
||||
end;
|
||||
|
||||
class function DateTimeCyclicEncoder.PartValue(dt: System.DateTime; part: DatePartKind): real;
|
||||
begin
|
||||
case part of
|
||||
dpMonth: Result := dt.Month - 1;
|
||||
dpDayOfWeek: Result := integer(dt.DayOfWeek);
|
||||
dpHour: Result := dt.Hour;
|
||||
dpMinute: Result := dt.Minute;
|
||||
dpSecond: Result := dt.Second;
|
||||
dpTimeOfDay:
|
||||
Result := dt.Hour * 3600.0 + dt.Minute * 60.0 + dt.Second + dt.Millisecond / 1000.0;
|
||||
else
|
||||
Error(ER_DTCYC_UNSUPPORTED_PART, part);
|
||||
end;
|
||||
end;
|
||||
|
||||
constructor DateTimeCyclicEncoder.Create(column: string; part: DatePartKind);
|
||||
begin
|
||||
if column = '' then
|
||||
ArgumentError(ER_DTCYC_NO_COLUMN);
|
||||
if not IsSupportedPart(part) then
|
||||
Error(ER_DTCYC_UNSUPPORTED_PART, part);
|
||||
|
||||
col := column;
|
||||
self.part := part;
|
||||
var suffix := PartSuffix(part);
|
||||
sinCol := col + '_cyc_' + suffix + '_sin';
|
||||
cosCol := col + '_cyc_' + suffix + '_cos';
|
||||
fitted := false;
|
||||
end;
|
||||
|
||||
function DateTimeCyclicEncoder.Fit(df: DataFrame): IPreprocessor;
|
||||
begin
|
||||
if df = nil then
|
||||
ArgumentNullError(ER_ARG_NULL, 'df');
|
||||
|
||||
var idx := df.Schema.IndexOf(col);
|
||||
if idx < 0 then
|
||||
Error(ER_COLUMN_NOT_FOUND, col);
|
||||
|
||||
if df.Schema.ColumnTypeAt(idx) <> ColumnType.ctDateTime then
|
||||
Error(ER_DTCYC_NOT_DATETIME, col);
|
||||
|
||||
if df.HasColumn(sinCol) then
|
||||
Error(ER_DTCYC_COLUMN_COLLISION, sinCol);
|
||||
if df.HasColumn(cosCol) then
|
||||
Error(ER_DTCYC_COLUMN_COLLISION, cosCol);
|
||||
if sinCol = cosCol then
|
||||
Error(ER_DTCYC_DUPLICATE_COLUMN, sinCol);
|
||||
|
||||
fitted := true;
|
||||
Result := Self;
|
||||
end;
|
||||
|
||||
function DateTimeCyclicEncoder.Transform(df: DataFrame): DataFrame;
|
||||
begin
|
||||
if not fitted then
|
||||
NotFittedError(ER_FIT_NOT_CALLED);
|
||||
|
||||
var srcIdx := df.Schema.IndexOf(col);
|
||||
if srcIdx < 0 then
|
||||
ArgumentError(ER_COLUMN_NOT_FOUND, col);
|
||||
if df.Schema.ColumnTypeAt(srcIdx) <> ColumnType.ctDateTime then
|
||||
Error(ER_DTCYC_NOT_DATETIME, col);
|
||||
|
||||
var rowCount := df.RowCount;
|
||||
var srcCol := DateTimeColumn(df.GetColumn(srcIdx));
|
||||
var srcData := srcCol.Data;
|
||||
var srcValid := srcCol.IsValid;
|
||||
|
||||
var period := PartPeriod(part);
|
||||
var sinData := new real[rowCount];
|
||||
var cosData := new real[rowCount];
|
||||
var sinValid := new boolean[rowCount];
|
||||
var cosValid := new boolean[rowCount];
|
||||
|
||||
for var row := 0 to rowCount - 1 do
|
||||
if (srcValid <> nil) and not srcValid[row] then
|
||||
begin
|
||||
sinData[row] := 0.0;
|
||||
cosData[row] := 0.0;
|
||||
sinValid[row] := False;
|
||||
cosValid[row] := False;
|
||||
end
|
||||
else
|
||||
begin
|
||||
var value := PartValue(srcData[row], part);
|
||||
var angle := 2 * PI * value / period;
|
||||
sinData[row] := Sin(angle);
|
||||
cosData[row] := Cos(angle);
|
||||
sinValid[row] := True;
|
||||
cosValid[row] := True;
|
||||
end;
|
||||
|
||||
var res := new DataFrame;
|
||||
var names := new List<string>;
|
||||
var types := new List<ColumnType>;
|
||||
var cats := new List<boolean>;
|
||||
|
||||
AppendAllColumnsExcept(res, df, srcIdx, names, types, cats);
|
||||
|
||||
res.AddFloatColumn(sinCol, sinData, sinValid);
|
||||
names.Add(sinCol);
|
||||
types.Add(ColumnType.ctFloat);
|
||||
cats.Add(False);
|
||||
|
||||
res.AddFloatColumn(cosCol, cosData, cosValid);
|
||||
names.Add(cosCol);
|
||||
types.Add(ColumnType.ctFloat);
|
||||
cats.Add(False);
|
||||
|
||||
res.SetSchema(new DataFrameSchema(
|
||||
names.ToArray,
|
||||
types.ToArray,
|
||||
cats.ToArray
|
||||
));
|
||||
|
||||
Result := res;
|
||||
end;
|
||||
|
||||
function DateTimeCyclicEncoder.FitTransform(df: DataFrame): DataFrame;
|
||||
begin
|
||||
Fit(df);
|
||||
Result := Transform(df);
|
||||
end;
|
||||
|
||||
function DateTimeCyclicEncoder.ToString: string;
|
||||
begin
|
||||
Result := 'DateTimeCyclicEncoder(column=' + col + ', part=' + part.ToString + ')';
|
||||
end;
|
||||
|
||||
function DateTimeCyclicEncoder.GetExpandedColumns(sourceColumn: string): array of string;
|
||||
begin
|
||||
if not fitted then
|
||||
NotFittedError(ER_FIT_NOT_CALLED);
|
||||
|
||||
if sourceColumn <> col then
|
||||
exit(nil);
|
||||
|
||||
Result := [sinCol, cosCol];
|
||||
end;
|
||||
|
||||
function DateTimeCyclicEncoder.Clone: IPreprocessor;
|
||||
begin
|
||||
Result := new DateTimeCyclicEncoder(col, part);
|
||||
end;
|
||||
|
||||
//-----------------------------
|
||||
// Imputer
|
||||
//-----------------------------
|
||||
|
|
|
|||
Loading…
Reference in a new issue