diff --git a/bin/Lib/DataFrameABC.pas b/bin/Lib/DataFrameABC.pas index 06485efe5..4d04dc992 100644 --- a/bin/Lib/DataFrameABC.pas +++ b/bin/Lib/DataFrameABC.pas @@ -4319,6 +4319,194 @@ begin Result := res; end; +procedure UpdateGroupStatsHelper( + g: integer; + idxs: List; + valid: array of boolean; + dataInt: array of integer; + dataFloat: array of real; + isInt: boolean; + var counts: array of integer; + var sums: array of real; + var mins: array of real; + var maxs: array of real +); +begin + for var j := 0 to idxs.Count-1 do + begin + var i := idxs[j]; + if not valid[i] then continue; + + var v := if isInt then dataInt[i] else dataFloat[i]; + + counts[g] += 1; + sums[g] += v; + + if v < mins[g] then mins[g] := v; + if v > maxs[g] then maxs[g] := v; + end; +end; + +procedure UpdateGroupStdHelper( + g: integer; + idxs: List; + valid: array of boolean; + dataInt: array of integer; + dataFloat: array of real; + isInt: boolean; + means: array of real; + var sumsq: array of real +); +begin + for var j := 0 to idxs.Count-1 do + begin + var i := idxs[j]; + if not valid[i] then continue; + + var v := if isInt then dataInt[i] else dataFloat[i]; + var d := v - means[g]; + + sumsq[g] += d*d; + end; +end; + +procedure ComputeAllStatsHelper( + groups: Dictionary>; + n: integer; + valid: array of boolean; + dataInt: array of integer; + dataFloat: array of real; + isInt: boolean; + var counts: array of integer; + var means: array of real; + var stds: array of real; + var mins: array of real; + var maxs: array of real +); +begin + var keys := groups.Keys.ToArray; + + var sums := new real[n]; + var sumsq := new real[n]; + + for var i := 0 to n-1 do + begin + mins[i] := real.MaxValue; + maxs[i] := real.MinValue; + end; + + // PASS 1 + for var g := 0 to n-1 do + begin + var idxs := groups[keys[g]]; + UpdateGroupStatsHelper(g, idxs, valid, dataInt, dataFloat, isInt, + counts, sums, mins, maxs); + end; + + // mean + for var i := 0 to n-1 do + if counts[i] > 0 then + means[i] := sums[i] / counts[i]; + + // PASS 2 + for var g := 0 to n-1 do + begin + var idxs := groups[keys[g]]; + UpdateGroupStdHelper(g, idxs, valid, dataInt, dataFloat, isInt, + means, sumsq); + end; + + for var i := 0 to n-1 do + if counts[i] > 1 then + stds[i] := Sqrt(sumsq[i] / (counts[i] - 1)); +end; + +procedure ComputeAllStatsHelper( + groups: Dictionary>; + n: integer; + valid: array of boolean; + dataInt: array of integer; + dataFloat: array of real; + isInt: boolean; + var counts: array of integer; + var means: array of real; + var stds: array of real; + var mins: array of real; + var maxs: array of real +); +begin + var keys := groups.Keys.ToArray; + + var sums := new real[n]; + var sumsq := new real[n]; + + for var i := 0 to n - 1 do + begin + mins[i] := real.MaxValue; + maxs[i] := real.MinValue; + end; + + // PASS 1 + for var g := 0 to n - 1 do + begin + var idxs := groups[keys[g]]; + UpdateGroupStatsHelper(g, idxs, valid, dataInt, dataFloat, isInt, + counts, sums, mins, maxs); + end; + + // mean + for var i := 0 to n - 1 do + if counts[i] > 0 then + means[i] := sums[i] / counts[i]; + + // PASS 2 + for var g := 0 to n - 1 do + begin + var idxs := groups[keys[g]]; + UpdateGroupStdHelper(g, idxs, valid, dataInt, dataFloat, isInt, + means, sumsq); + end; + + for var i := 0 to n - 1 do + if counts[i] > 1 then + stds[i] := Sqrt(sumsq[i] / (counts[i] - 1)); +end; + +procedure AddDescribeStatsColumnsHelper( + res: DataFrame; + counts: array of integer; + means, stds, mins, maxs: array of real; + names: List; + types: List; + cats: List +); +begin + res.AddIntColumn('count', counts, nil); + names.Add('count'); + types.Add(ctInt); + cats.Add(false); + + res.AddFloatColumn('mean', means, nil); + names.Add('mean'); + types.Add(ctFloat); + cats.Add(false); + + res.AddFloatColumn('std', stds, nil); + names.Add('std'); + types.Add(ctFloat); + cats.Add(false); + + res.AddFloatColumn('min', mins, nil); + names.Add('min'); + types.Add(ctFloat); + cats.Add(false); + + res.AddFloatColumn('max', maxs, nil); + names.Add('max'); + types.Add(ctFloat); + cats.Add(false); +end; + function GroupByContext.Describe(colName: string): DataFrame; begin var colIndex := source.ColumnIndex(colName); @@ -4338,60 +4526,50 @@ begin if singleKey then begin - var keys := groups1.Keys.ToArray; + var n := groups1.Count; - var counts := new integer[keys.Length]; - var means := new real[keys.Length]; - var stds := new real[keys.Length]; - var mins := new real[keys.Length]; - var maxs := new real[keys.Length]; + var counts := new integer[n]; + var means := new real[n]; + var stds := new real[n]; + var mins := new real[n]; + var maxs := new real[n]; - // --- твой код без изменений --- - // pass1 + pass2 остаются как есть + ComputeAllStatsHelper(groups1, n, valid, dataInt, dataFloat, isInt, + counts, means, stds, mins, maxs); + + // ключи в правильном порядке + var keys := groups1.Select(kvp -> kvp.Key).ToArray; var col := source.columns[keyColumn]; var keyName := col.Info.Name; if col.Info.ColType = ctInt then - begin - res.AddIntColumn(keyName, keys.Select(k -> integer(k)).ToArray, nil); - types.Add(ctInt); - end + res.AddIntColumn(keyName, keys.Select(k -> integer(k)).ToArray, nil) else - begin res.AddStrColumn(keyName, keys.Select(k -> string(k)).ToArray, nil); - types.Add(ctStr); - end; names.Add(keyName); + types.Add(col.Info.ColType); cats.Add(true); - res.AddIntColumn('count', counts, nil); - names.Add('count'); types.Add(ctInt); cats.Add(false); - - res.AddFloatColumn('mean', means, nil); - names.Add('mean'); types.Add(ctFloat); cats.Add(false); - - res.AddFloatColumn('std', stds, nil); - names.Add('std'); types.Add(ctFloat); cats.Add(false); - - res.AddFloatColumn('min', mins, nil); - names.Add('min'); types.Add(ctFloat); cats.Add(false); - - res.AddFloatColumn('max', maxs, nil); - names.Add('max'); types.Add(ctFloat); cats.Add(false); + AddDescribeStatsColumnsHelper(res, counts, means, stds, mins, maxs, + names, types, cats); end else begin - var keys := groupsN.Keys.ToArray; + var n := groupsN.Count; - var counts := new integer[keys.Length]; - var means := new real[keys.Length]; - var stds := new real[keys.Length]; - var mins := new real[keys.Length]; - var maxs := new real[keys.Length]; + var counts := new integer[n]; + var means := new real[n]; + var stds := new real[n]; + var mins := new real[n]; + var maxs := new real[n]; - // --- твой код pass1 + pass2 --- + ComputeAllStatsHelper(groupsN, n, valid, dataInt, dataFloat, isInt, + counts, means, stds, mins, maxs); + + // ключи в правильном порядке + var keys := groupsN.Select(kvp -> kvp.Key).ToArray; for var k := 0 to keyColumns.Length - 1 do begin @@ -4400,37 +4578,19 @@ begin var keyName := col.Info.Name; if col.Info.ColType = ctInt then - begin - res.AddIntColumn(keyName, keys.Select(key -> integer(key[k])).ToArray, nil); - types.Add(ctInt); - end + res.AddIntColumn(keyName, keys.Select(key -> integer(key[k])).ToArray, nil) else - begin res.AddStrColumn(keyName, keys.Select(key -> string(key[k])).ToArray, nil); - types.Add(ctStr); - end; names.Add(keyName); + types.Add(col.Info.ColType); cats.Add(true); end; - res.AddIntColumn('count', counts, nil); - names.Add('count'); types.Add(ctInt); cats.Add(false); - - res.AddFloatColumn('mean', means, nil); - names.Add('mean'); types.Add(ctFloat); cats.Add(false); - - res.AddFloatColumn('std', stds, nil); - names.Add('std'); types.Add(ctFloat); cats.Add(false); - - res.AddFloatColumn('min', mins, nil); - names.Add('min'); types.Add(ctFloat); cats.Add(false); - - res.AddFloatColumn('max', maxs, nil); - names.Add('max'); types.Add(ctFloat); cats.Add(false); + AddDescribeStatsColumnsHelper(res, counts, means, stds, mins, maxs, + names, types, cats); end; - // 🔥 САМОЕ ВАЖНОЕ res.SetSchema(new DataFrameSchema( names.ToArray, types.ToArray, @@ -4440,6 +4600,7 @@ begin Result := res; end; +// TODO: Optimize DescribeAll to single-pass implementation function GroupByContext.DescribeAll: DataFrame; begin var res := new DataFrame; @@ -4451,7 +4612,7 @@ begin // 1) ключи if singleKey then begin - var keys := groups1.Keys.ToArray; + var keys := groups1.Select(kvp -> kvp.Key).ToArray; var col := source.columns[keyColumn]; var keyName := col.Info.Name; @@ -4471,7 +4632,7 @@ begin end else begin - var keys := groupsN.Keys.ToArray; + var keys := groupsN.Select(kvp -> kvp.Key).ToArray; for var k := 0 to keyColumns.Length - 1 do begin